Unverified Commit 570ecf42 authored by Jin Hai's avatar Jin Hai Committed by GitHub
Browse files

Add cfg fun (#1501) 



* save work

Signed-off-by: default avatarlichengming <chengming.li@zilliz.com>

* wal config&gtest

Signed-off-by: default avatarlichengming <chengming.li@zilliz.com>

* fix dco and lint error

Signed-off-by: default avatarlichengming <chengming.li@zilliz.com>

* try to fix lint error

Signed-off-by: default avatarlichengming <chengming.li@zilliz.com>

* try to fix clang-format issue

Signed-off-by: default avatarlichengming <chengming.li@zilliz.com>

* update change log

Signed-off-by: default avatarlichengming <chengming.li@zilliz.com>

* ajdjust the code by yudong's advice

Signed-off-by: default avatarlichengming <chengming.li@zilliz.com>

* fix error

Signed-off-by: default avatarlichengming <chengming.li@zilliz.com>

* recommit pr 1501,local gtest is ok

Signed-off-by: default avatarlichengming <chengming.li@zilliz.com>
parent 8c7ef47f

CHANGELOG.md

+2 −1
Original line number Diff line number Diff line
@@ -32,6 +32,7 @@ Please mark all change in change log and use the issue from GitHub 
-   \#1499 Fix duplicated ID number issue

-   \#1491 Server crashed during adding vectors  

-   \#1504 Avoid possible race condition between delete and search 

-   \#1510 Add set interfaces for WAL configurations



## Feature

-   \#216 Add CLI to get server info
@@ -44,7 +45,7 @@ Please mark all change in change log and use the issue from GitHub 
-   \#813 Add push mode for prometheus monitor

-   \#815 Support MinIO storage

-   \#823 Support binary vector tanimoto/jaccard/hamming metric

-   \#830 - Support WAL(write-ahead logging)

-   \#830 Support WAL(write-ahead logging)

-   \#853 Support HNSW

-   \#861 Support DeleteById / SearchByID / GetVectorById / Flush

-   \#910 Change Milvus c++ standard to c++17

core/src/server/Config.cpp

+64 −3
Original line number Diff line number Diff line
@@ -284,6 +284,12 @@ Config::ResetDefaultConfig() { 
    CONFIG_CHECK(SetEngineConfigUseBlasThreshold(CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT));

    CONFIG_CHECK(SetEngineConfigOmpThreadNum(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT));

    CONFIG_CHECK(SetEngineConfigUseAVX512(CONFIG_ENGINE_USE_AVX512_DEFAULT));



    /* wal config */

    CONFIG_CHECK(SetWalConfigEnable(CONFIG_WAL_ENABLE_DEFAULT));

    CONFIG_CHECK(SetWalConfigRecoveryErrorIgnore(CONFIG_WAL_RECOVERY_ERROR_IGNORE_DEFAULT));

    CONFIG_CHECK(SetWalConfigBufferSize(CONFIG_WAL_BUFFER_SIZE_DEFAULT));

    CONFIG_CHECK(SetWalConfigWalPath(CONFIG_WAL_WAL_PATH_DEFAULT));

#ifdef MILVUS_GPU_VERSION

    CONFIG_CHECK(SetEngineConfigGpuSearchThreshold(CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT));

#endif
@@ -402,6 +408,16 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key 
#endif

    } else if (parent_key == CONFIG_TRACING) {

        return Status(SERVER_UNSUPPORTED_ERROR, "Not support set tracing_config currently");

    } else if (parent_key == CONFIG_WAL) {

        if (child_key == CONFIG_WAL_ENABLE) {

            status = SetWalConfigEnable(value);

        } else if (child_key == CONFIG_WAL_RECOVERY_ERROR_IGNORE) {

            status = SetWalConfigRecoveryErrorIgnore(value);

        } else if (child_key == CONFIG_WAL_BUFFER_SIZE) {

            status = SetWalConfigBufferSize(value);

        } else if (child_key == CONFIG_WAL_WAL_PATH) {

            status = SetWalConfigWalPath(value);

        }

    }



    if (status.ok()) {
@@ -1201,7 +1217,10 @@ Config::CheckGpuResourceConfigBuildIndexResources(const std::vector<std::string> 
/* wal config */

Status

Config::CheckWalConfigEnable(const std::string& value) {

    if (!ValidationUtil::ValidateStringIsBool(value).ok()) {

    auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok();

    fiu_do_on("check_config_wal_enable_fail", exist_error = true);



    if (exist_error) {

        std::string msg = "Invalid wal config: " + value + ". Possible reason: wal_config.enable is not a boolean.";

        return Status(SERVER_INVALID_ARGUMENT, msg);

    }
@@ -1210,7 +1229,10 @@ Config::CheckWalConfigEnable(const std::string& value) { 


Status

Config::CheckWalConfigRecoveryErrorIgnore(const std::string& value) {

    if (!ValidationUtil::ValidateStringIsBool(value).ok()) {

    auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok();

    fiu_do_on("check_config_wal_recovery_error_ignore_fail", exist_error = true);



    if (exist_error) {

        std::string msg =

            "Invalid wal config: " + value + ". Possible reason: wal_config.recovery_error_ignore is not a boolean.";

        return Status(SERVER_INVALID_ARGUMENT, msg);
@@ -1220,7 +1242,10 @@ Config::CheckWalConfigRecoveryErrorIgnore(const std::string& value) { 


Status

Config::CheckWalConfigBufferSize(const std::string& value) {

    if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {

    auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok();

    fiu_do_on("check_config_wal_buffer_size_fail", exist_error = true);



    if (exist_error) {

        std::string msg = "Invalid wal buffer size: " + value +

                          ". Possible reason: wal_config.buffer_size is not a positive integer.";

        return Status(SERVER_INVALID_ARGUMENT, msg);
@@ -1228,6 +1253,16 @@ Config::CheckWalConfigBufferSize(const std::string& value) { 
    return Status::OK();

}



Status

Config::CheckWalConfigWalPath(const std::string& value) {

    fiu_return_on("check_wal_path_fail", Status(SERVER_INVALID_ARGUMENT, ""));

    if (value.empty()) {

        return Status(SERVER_INVALID_ARGUMENT, "wal_config.wal_path is empty!");

    }



    return ValidationUtil::ValidateStoragePath(value);

}



////////////////////////////////////////////////////////////////////////////////

ConfigNode&

Config::GetConfigRoot() {
@@ -1688,6 +1723,7 @@ Config::GetWalConfigBufferSize(int64_t& buffer_size) { 
Status

Config::GetWalConfigWalPath(std::string& wal_path) {

    wal_path = GetConfigStr(CONFIG_WAL, CONFIG_WAL_WAL_PATH, CONFIG_WAL_WAL_PATH_DEFAULT);

    CONFIG_CHECK(CheckWalConfigWalPath(wal_path));

    return Status::OK();

}
@@ -1865,6 +1901,31 @@ Config::SetEngineConfigUseAVX512(const std::string& value) { 
    return SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_USE_AVX512, value);

}



/* wal config */

Status

Config::SetWalConfigEnable(const std::string& value) {

    CONFIG_CHECK(CheckWalConfigEnable(value));

    return SetConfigValueInMem(CONFIG_WAL, CONFIG_WAL_ENABLE, value);

}



Status

Config::SetWalConfigRecoveryErrorIgnore(const std::string& value) {

    CONFIG_CHECK(CheckWalConfigRecoveryErrorIgnore(value));

    return SetConfigValueInMem(CONFIG_WAL, CONFIG_WAL_RECOVERY_ERROR_IGNORE_DEFAULT, value);

}



Status

Config::SetWalConfigBufferSize(const std::string& value) {

    CONFIG_CHECK(CheckWalConfigBufferSize(value));

    return SetConfigValueInMem(CONFIG_WAL, CONFIG_WAL_BUFFER_SIZE, value);

}



Status

Config::SetWalConfigWalPath(const std::string& value) {

    CONFIG_CHECK(CheckWalConfigWalPath(value));

    return SetConfigValueInMem(CONFIG_WAL, CONFIG_WAL_WAL_PATH, value);

}



#ifdef MILVUS_GPU_VERSION

Status

Config::SetEngineConfigGpuSearchThreshold(const std::string& value) {

core/src/server/Config.h

+16 −4
Original line number Diff line number Diff line
@@ -288,6 +288,8 @@ class Config { 
    CheckWalConfigRecoveryErrorIgnore(const std::string& value);

    Status

    CheckWalConfigBufferSize(const std::string& value);

    Status

    CheckWalConfigWalPath(const std::string& value);



    std::string

    GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = "");
@@ -392,13 +394,13 @@ class Config { 


    /* wal config */

    Status

    GetWalConfigEnable(bool& wal_enable);

    GetWalConfigEnable(bool& value);

    Status

    GetWalConfigRecoveryErrorIgnore(bool& recovery_error_ignore);

    GetWalConfigRecoveryErrorIgnore(bool& value);

    Status

    GetWalConfigBufferSize(int64_t& buffer_size);

    GetWalConfigBufferSize(int64_t& value);

    Status

    GetWalConfigWalPath(std::string& wal_path);

    GetWalConfigWalPath(std::string& value);



    Status

    GetServerRestartRequired(bool& required);
@@ -468,6 +470,16 @@ class Config { 
    Status

    SetEngineConfigUseAVX512(const std::string& value);



    /* wal config */

    Status

    SetWalConfigEnable(const std::string& value);

    Status

    SetWalConfigRecoveryErrorIgnore(const std::string& value);

    Status

    SetWalConfigBufferSize(const std::string& value);

    Status

    SetWalConfigWalPath(const std::string& value);



#ifdef MILVUS_GPU_VERSION

    Status

    SetEngineConfigGpuSearchThreshold(const std::string& value);

core/unittest/server/test_config.cpp

+70 −0
Original line number Diff line number Diff line
@@ -56,6 +56,7 @@ TEST_F(ConfigTest, CONFIG_TEST) { 
    milvus::server::ConfigNode& db_config = root_config.GetChild("db_config");

    milvus::server::ConfigNode& metric_config = root_config.GetChild("metric_config");

    milvus::server::ConfigNode& cache_config = root_config.GetChild("cache_config");

    milvus::server::ConfigNode& wal_config = root_config.GetChild("wal_config");

    milvus::server::ConfigNode invalid_config = root_config.GetChild("invalid_config");



    const auto& im_config_mgr = *static_cast<milvus::server::YamlConfigMgr*>(config_mgr);
@@ -323,6 +324,27 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { 
        ASSERT_TRUE(std::stoll(build_index_resources[i].substr(3)) == build_index_res_vec[i]);

    }

#endif



    /* wal config */

    bool wal_enable = false;

    ASSERT_TRUE(config.SetWalConfigEnable(std::to_string(wal_enable)).ok());

    ASSERT_TRUE(config.GetWalConfigEnable(bool_val).ok());

    ASSERT_TRUE(bool_val == wal_enable);



    bool wal_recovery_ignore = true;

    ASSERT_TRUE(config.SetWalConfigRecoveryErrorIgnore(std::to_string(wal_recovery_ignore)).ok());

    ASSERT_TRUE(config.GetWalConfigRecoveryErrorIgnore(bool_val).ok());

    ASSERT_TRUE(bool_val == wal_recovery_ignore);



    int64_t wal_buffer_size = 128;

    ASSERT_TRUE(config.SetWalConfigBufferSize(std::to_string(wal_buffer_size)).ok());

    ASSERT_TRUE(config.GetWalConfigBufferSize(int64_val).ok());

    ASSERT_TRUE(int64_val == wal_buffer_size);



    std::string wal_path = "/tmp/aaa/wal";

    ASSERT_TRUE(config.SetWalConfigWalPath(wal_path).ok());

    ASSERT_TRUE(config.GetWalConfigWalPath(str_val).ok());

    ASSERT_TRUE(str_val == wal_path);

}



std::string
@@ -625,6 +647,12 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { 
    ASSERT_FALSE(config.SetGpuResourceConfigBuildIndexResources("gpu16").ok());

    ASSERT_FALSE(config.SetGpuResourceConfigBuildIndexResources("gpu0, gpu0, gpu1").ok());

#endif



    /* wal config */

    ASSERT_FALSE(config.SetWalConfigWalPath("hello/world").ok());

    ASSERT_FALSE(config.SetWalConfigWalPath("").ok());

    ASSERT_FALSE(config.SetWalConfigBufferSize("-1").ok());

    ASSERT_FALSE(config.SetWalConfigBufferSize("a").ok());

}



TEST_F(ConfigTest, SERVER_CONFIG_TEST) {
@@ -844,6 +872,27 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_FAIL_TEST) { 
    ASSERT_FALSE(s.ok());

    fiu_disable("Config.GetGpuResourceConfigCacheThreshold.diable_gpu_resource");

#endif



    /* wal config */

    fiu_enable("check_config_wal_enable_fail", 1, NULL, 0);

    s = config.ValidateConfig();

    ASSERT_FALSE(s.ok());

    fiu_disable("check_config_wal_enable_fail");



    fiu_enable("check_config_wal_recovery_error_ignore_fail", 1, NULL, 0);

    s = config.ValidateConfig();

    ASSERT_FALSE(s.ok());

    fiu_disable("check_config_wal_recovery_error_ignore_fail");



    fiu_enable("check_config_wal_buffer_size_fail", 1, NULL, 0);

    s = config.ValidateConfig();

    ASSERT_FALSE(s.ok());

    fiu_disable("check_config_wal_buffer_size_fail");



    fiu_enable("check_wal_path_fail", 1, NULL, 0);

    s = config.ValidateConfig();

    ASSERT_FALSE(s.ok());

    fiu_disable("check_wal_path_fail");

}



TEST_F(ConfigTest, SERVER_CONFIG_RESET_DEFAULT_CONFIG_FAIL_TEST) {
@@ -977,6 +1026,27 @@ TEST_F(ConfigTest, SERVER_CONFIG_RESET_DEFAULT_CONFIG_FAIL_TEST) { 


    s = config.ResetDefaultConfig();

    ASSERT_TRUE(s.ok());



    /* wal config */

    fiu_enable("check_config_wal_enable_fail", 1, NULL, 0);

    s = config.ResetDefaultConfig();

    ASSERT_FALSE(s.ok());

    fiu_disable("check_config_wal_enable_fail");



    fiu_enable("check_config_wal_recovery_error_ignore_fail", 1, NULL, 0);

    s = config.ResetDefaultConfig();

    ASSERT_FALSE(s.ok());

    fiu_disable("check_config_wal_recovery_error_ignore_fail");



    fiu_enable("check_config_wal_buffer_size_fail", 1, NULL, 0);

    s = config.ResetDefaultConfig();

    ASSERT_FALSE(s.ok());

    fiu_disable("check_config_wal_buffer_size_fail");



    fiu_enable("check_wal_path_fail", 1, NULL, 0);

    s = config.ResetDefaultConfig();

    ASSERT_FALSE(s.ok());

    fiu_disable("check_wal_path_fail");

}



TEST_F(ConfigTest, SERVER_CONFIG_OTHER_CONFIGS_FAIL_TEST) {