Commit 1b0d4533 authored by 蔡宇东's avatar 蔡宇东
Browse files

#207 add more unittest for config set/get

parent ea505c86

core/src/server/Config.cpp

+47 −15
Original line number Diff line number Diff line
@@ -25,6 +25,7 @@ 
#include "config/YamlConfigMgr.h"

#include "server/Config.h"

#include "utils/CommonUtil.h"

#include "utils/StringHelpFunctions.h"

#include "utils/ValidationUtil.h"



namespace milvus {
@@ -341,6 +342,11 @@ Config::ResetDefaultConfig() { 
        return s;

    }



    s = SetResourceConfigSearchResources(CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT);

    if (!s.ok()) {

        return s;

    }



    s = SetResourceConfigIndexBuildDevice(CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT);

    if (!s.ok()) {

        return s;
@@ -788,6 +794,17 @@ Config::GetConfigStr(const std::string& parent_key, const std::string& child_key 
    return value;

}



std::string

Config::GetConfigSequenceStr(const std::string& parent_key, const std::string& child_key, const std::string& delim) {

    std::string value;

    if (!GetConfigValueInMem(parent_key, child_key, value).ok()) {

        std::vector<std::string> sequence = GetConfigNode(parent_key).GetSequence(child_key);

        server::StringHelpFunctions::MergeStringWithDelimeter(sequence, delim, value);

        SetConfigValueInMem(parent_key, child_key, value);

    }

    return value;

}



Status

Config::GetServerConfigAddress(std::string& value) {

    value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT);
@@ -1011,8 +1028,9 @@ Config::GetResourceConfigMode(std::string& value) { 


Status

Config::GetResourceConfigSearchResources(std::vector<std::string>& value) {

    ConfigNode resource_config = GetConfigNode(CONFIG_RESOURCE);

    value = resource_config.GetSequence(CONFIG_RESOURCE_SEARCH_RESOURCES);

    std::string str = GetConfigSequenceStr(CONFIG_RESOURCE, CONFIG_RESOURCE_SEARCH_RESOURCES,

                                           CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER);

    server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, value);

    return CheckResourceConfigSearchResources(value);

}
@@ -1155,7 +1173,7 @@ Config::SetMetricConfigEnableMonitor(const std::string& value) { 
        return s;

    }



    SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_ENABLE_MONITOR, value);

    SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value);

    return Status::OK();

}
@@ -1166,7 +1184,7 @@ Config::SetMetricConfigCollector(const std::string& value) { 
        return s;

    }



    SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_COLLECTOR, value);

    SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value);

    return Status::OK();

}
@@ -1177,7 +1195,7 @@ Config::SetMetricConfigPrometheusPort(const std::string& value) { 
        return s;

    }



    SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_PROMETHEUS_PORT, value);

    SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value);

    return Status::OK();

}
@@ -1189,7 +1207,7 @@ Config::SetCacheConfigCpuCacheCapacity(const std::string& value) { 
        return s;

    }



    SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_CACHE_CAPACITY, value);

    SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, value);

    return Status::OK();

}
@@ -1200,7 +1218,7 @@ Config::SetCacheConfigCpuCacheThreshold(const std::string& value) { 
        return s;

    }



    SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value);

    SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value);

    return Status::OK();

}
@@ -1211,7 +1229,7 @@ Config::SetCacheConfigGpuCacheCapacity(const std::string& value) { 
        return s;

    }



    SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_CACHE_CAPACITY, value);

    SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, value);

    return Status::OK();

}
@@ -1222,7 +1240,7 @@ Config::SetCacheConfigGpuCacheThreshold(const std::string& value) { 
        return s;

    }



    SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value);

    SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value);

    return Status::OK();

}
@@ -1233,7 +1251,7 @@ Config::SetCacheConfigCacheInsertData(const std::string& value) { 
        return s;

    }



    SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CACHE_INSERT_DATA, value);

    SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value);

    return Status::OK();

}
@@ -1245,7 +1263,7 @@ Config::SetEngineConfigUseBlasThreshold(const std::string& value) { 
        return s;

    }



    SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value);

    SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value);

    return Status::OK();

}
@@ -1256,7 +1274,7 @@ Config::SetEngineConfigOmpThreadNum(const std::string& value) { 
        return s;

    }



    SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_OMP_THREAD_NUM, value);

    SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value);

    return Status::OK();

}
@@ -1267,7 +1285,7 @@ Config::SetEngineConfigGpuSearchThreshold(const std::string& value) { 
        return s;

    }



    SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value);

    SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value);

    return Status::OK();

}
@@ -1279,7 +1297,21 @@ Config::SetResourceConfigMode(const std::string& value) { 
        return s;

    }



    SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_MODE, value);

    SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value);

    return Status::OK();

}



Status

Config::SetResourceConfigSearchResources(const std::string& value) {

    std::vector<std::string> res_vec;

    server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, res_vec);



    Status s = CheckResourceConfigSearchResources(res_vec);

    if (!s.ok()) {

        return s;

    }



    SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_SEARCH_RESOURCES, value);

    return Status::OK();

}
@@ -1290,7 +1322,7 @@ Config::SetResourceConfigIndexBuildDevice(const std::string& value) { 
        return s;

    }



    SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, value);

    SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, value);

    return Status::OK();

}

core/src/server/Config.h

+6 −0
Original line number Diff line number Diff line
@@ -92,6 +92,8 @@ static const char* CONFIG_RESOURCE = "resource_config"; 
static const char* CONFIG_RESOURCE_MODE = "mode";

static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple";

static const char* CONFIG_RESOURCE_SEARCH_RESOURCES = "search_resources";

static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER = ",";

static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu,gpu0";

static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE = "index_build_device";

static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "gpu0";
@@ -183,6 +185,8 @@ class Config { 


    std::string

    GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = "");

    std::string

    GetConfigSequenceStr(const std::string& parent_key, const std::string& child_key, const std::string& delim);



 public:

    /* server config */
@@ -304,6 +308,8 @@ class Config { 
    Status

    SetResourceConfigMode(const std::string& value);

    Status

    SetResourceConfigSearchResources(const std::string& value);

    Status

    SetResourceConfigIndexBuildDevice(const std::string& value);



 private:

core/src/utils/StringHelpFunctions.cpp

+29 −15
Original line number Diff line number Diff line
@@ -39,39 +39,53 @@ StringHelpFunctions::TrimStringQuote(std::string& string, const std::string& qou 
    }

}



Status

void

StringHelpFunctions::SplitStringByDelimeter(const std::string& str, const std::string& delimeter,

                                            std::vector<std::string>& result) {

    if (str.empty()) {

        return Status::OK();

        return;

    }



    size_t last = 0;

    size_t index = str.find_first_of(delimeter, last);

    while (index != std::string::npos) {

        result.emplace_back(str.substr(last, index - last));

        last = index + 1;

        index = str.find_first_of(delimeter, last);

    size_t prev = 0, pos = 0;

    while (true) {

        pos = str.find_first_of(delimeter, prev);

        if (pos == std::string::npos) {

            result.emplace_back(str.substr(prev));

            break;

        } else {

            result.emplace_back(str.substr(prev, pos - prev));

            prev = pos + 1;

        }

    }

    if (index - last > 0) {

        std::string temp = str.substr(last);

        result.emplace_back(temp);

}



    return Status::OK();

void

StringHelpFunctions::MergeStringWithDelimeter(const std::vector<std::string>& strs, const std::string& delimeter,

                                              std::string& result) {

    if (strs.empty()) {

        result = "";

        return;

    }



    result = strs[0];

    for (size_t i = 1; i < strs.size(); i++) {

        result = result + delimeter + strs[i];

    }

}



Status

StringHelpFunctions::SplitStringByQuote(const std::string& str, const std::string& delimeter, const std::string& quote,

                                        std::vector<std::string>& result) {

    if (quote.empty()) {

        return SplitStringByDelimeter(str, delimeter, result);

        SplitStringByDelimeter(str, delimeter, result);

        return Status::OK();

    }



    size_t last = 0;

    size_t index = str.find_first_of(quote, last);

    if (index == std::string::npos) {

        return SplitStringByDelimeter(str, delimeter, result);

        SplitStringByDelimeter(str, delimeter, result);

        return Status::OK();

    }



    std::string process_str = str;
@@ -116,7 +130,7 @@ StringHelpFunctions::SplitStringByQuote(const std::string& str, const std::strin 
    }



    if (!process_str.empty()) {

        return SplitStringByDelimeter(process_str, delimeter, result);

        SplitStringByDelimeter(process_str, delimeter, result);

    }



    return Status::OK();

core/src/utils/StringHelpFunctions.h

+4 −1
Original line number Diff line number Diff line
@@ -43,9 +43,12 @@ class StringHelpFunctions { 
    // ,b,                | b |

    // ,,                 |   |

    // a                    a

    static Status

    static void

    SplitStringByDelimeter(const std::string& str, const std::string& delimeter, std::vector<std::string>& result);



    static void

    MergeStringWithDelimeter(const std::vector<std::string>& strs, const std::string& delimeter, std::string& result);



    // assume the table has two columns, quote='\"', delimeter=','

    //  a,b             a | b

    //  "aa,gg,yy",b    aa,gg,yy | b

core/unittest/server/test_config.cpp

+201 −8
Original line number Diff line number Diff line
@@ -19,10 +19,11 @@ 
#include <gtest/gtest-death-test.h>



#include "config/YamlConfigMgr.h"

#include "utils/CommonUtil.h"

#include "utils/ValidationUtil.h"

#include "server/Config.h"

#include "server/utils.h"

#include "utils/CommonUtil.h"

#include "utils/ValidationUtil.h"

#include "utils/StringHelpFunctions.h"



namespace {
@@ -98,19 +99,191 @@ TEST_F(ConfigTest, CONFIG_TEST) { 
    ASSERT_TRUE(seqs.empty());

}



TEST_F(ConfigTest, SERVER_CONFIG_TEST) {

TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {

    std::string config_path(CONFIG_PATH);

    milvus::server::Config &config = milvus::server::Config::GetInstance();

    milvus::Status s = config.LoadConfigFile(config_path + VALID_CONFIG_FILE);

    milvus::Status s;

    std::string str_val;

    int32_t int32_val;

    int64_t int64_val;

    float float_val;

    bool bool_val;



    /* server config */

    std::string server_addr = "192.168.1.155";

    s = config.SetServerConfigAddress(server_addr);

    ASSERT_TRUE(s.ok());

    s = config.GetServerConfigAddress(str_val);

    ASSERT_TRUE(s.ok());

    ASSERT_TRUE(str_val == server_addr);



    s = config.ValidateConfig();

    std::string server_port = "12345";

    s = config.SetServerConfigPort(server_port);

    ASSERT_TRUE(s.ok());

    s = config.GetServerConfigPort(str_val);

    ASSERT_TRUE(s.ok());

    ASSERT_TRUE(str_val == server_port);



    config.PrintAll();

    std::string server_mode = "cluster_readonly";

    s = config.SetServerConfigDeployMode(server_mode);

    ASSERT_TRUE(s.ok());

    s = config.GetServerConfigDeployMode(str_val);

    ASSERT_TRUE(s.ok());

    ASSERT_TRUE(str_val == server_mode);



    s = config.ResetDefaultConfig();

    std::string server_time_zone = "UTC+6";

    s = config.SetServerConfigTimeZone(server_time_zone);

    ASSERT_TRUE(s.ok());

    s = config.GetServerConfigTimeZone(str_val);

    ASSERT_TRUE(s.ok());

    ASSERT_TRUE(str_val == server_time_zone);



    /* db config */

    std::string db_primary_path = "/home/zilliz";

    s = config.SetDBConfigPrimaryPath(db_primary_path);

    ASSERT_TRUE(s.ok());

    s = config.GetDBConfigPrimaryPath(str_val);

    ASSERT_TRUE(s.ok());

    ASSERT_TRUE(str_val == db_primary_path);



    std::string db_secondary_path = "/home/zilliz";

    s = config.SetDBConfigSecondaryPath(db_secondary_path);

    ASSERT_TRUE(s.ok());

    s = config.GetDBConfigSecondaryPath(str_val);

    ASSERT_TRUE(s.ok());

    ASSERT_TRUE(str_val == db_secondary_path);



    std::string db_backend_url = "mysql://root:123456@127.0.0.1:19530/milvus";

    s = config.SetDBConfigBackendUrl(db_backend_url);

    ASSERT_TRUE(s.ok());

    s = config.GetDBConfigBackendUrl(str_val);

    ASSERT_TRUE(s.ok());

    ASSERT_TRUE(str_val == db_backend_url);



    int32_t db_archive_disk_threshold = 100;

    s = config.SetDBConfigArchiveDiskThreshold(std::to_string(db_archive_disk_threshold));

    ASSERT_TRUE(s.ok());

    s = config.GetDBConfigArchiveDiskThreshold(int32_val);

    ASSERT_TRUE(s.ok());

    ASSERT_TRUE(int32_val == db_archive_disk_threshold);



    int32_t db_archive_days_threshold = 365;

    s = config.SetDBConfigArchiveDaysThreshold(std::to_string(db_archive_days_threshold));

    ASSERT_TRUE(s.ok());

    s = config.GetDBConfigArchiveDaysThreshold(int32_val);

    ASSERT_TRUE(s.ok());

    ASSERT_TRUE(int32_val == db_archive_days_threshold);



    int32_t db_insert_buffer_size = 2;

    s = config.SetDBConfigInsertBufferSize(std::to_string(db_insert_buffer_size));

    ASSERT_TRUE(s.ok());

    s = config.GetDBConfigInsertBufferSize(int32_val);

    ASSERT_TRUE(s.ok());

    ASSERT_TRUE(int32_val == db_insert_buffer_size);



    /* metric config */

    bool metric_enable_monitor = false;

    s = config.SetMetricConfigEnableMonitor(std::to_string(metric_enable_monitor));

    ASSERT_TRUE(s.ok());

    s = config.GetMetricConfigEnableMonitor(bool_val);

    ASSERT_TRUE(s.ok());

    ASSERT_TRUE(bool_val == metric_enable_monitor);



    std::string metric_collector = "prometheus";

    s = config.SetMetricConfigCollector(metric_collector);

    ASSERT_TRUE(s.ok());

    s = config.GetMetricConfigCollector(str_val);

    ASSERT_TRUE(str_val == metric_collector);



    std::string metric_prometheus_port = "2222";

    s = config.SetMetricConfigPrometheusPort(metric_prometheus_port);

    ASSERT_TRUE(s.ok());

    s = config.GetMetricConfigPrometheusPort(str_val);

    ASSERT_TRUE(s.ok());

    ASSERT_TRUE(str_val == metric_prometheus_port);



    /* cache config */

    int64_t cache_cpu_cache_capacity = 5;

    s = config.SetCacheConfigCpuCacheCapacity(std::to_string(cache_cpu_cache_capacity));

    ASSERT_TRUE(s.ok());

    s = config.GetCacheConfigCpuCacheCapacity(int64_val);

    ASSERT_TRUE(s.ok());

    ASSERT_TRUE(int64_val == cache_cpu_cache_capacity);



    float cache_cpu_cache_threshold = 0.1;

    s = config.SetCacheConfigCpuCacheThreshold(std::to_string(cache_cpu_cache_threshold));

    ASSERT_TRUE(s.ok());

    s = config.GetCacheConfigCpuCacheThreshold(float_val);

    ASSERT_TRUE(float_val == cache_cpu_cache_threshold);



    int64_t cache_gpu_cache_capacity = 1;

    s = config.SetCacheConfigGpuCacheCapacity(std::to_string(cache_gpu_cache_capacity));

    ASSERT_TRUE(s.ok());

    s = config.GetCacheConfigGpuCacheCapacity(int64_val);

    ASSERT_TRUE(s.ok());

    ASSERT_TRUE(int64_val == cache_gpu_cache_capacity);



    float cache_gpu_cache_threshold = 0.2;

    s = config.SetCacheConfigGpuCacheThreshold(std::to_string(cache_gpu_cache_threshold));

    ASSERT_TRUE(s.ok());

    s = config.GetCacheConfigGpuCacheThreshold(float_val);

    ASSERT_TRUE(float_val == cache_gpu_cache_threshold);



    bool cache_insert_data = true;

    s = config.SetCacheConfigCacheInsertData(std::to_string(cache_insert_data));

    ASSERT_TRUE(s.ok());

    s = config.GetCacheConfigCacheInsertData(bool_val);

    ASSERT_TRUE(bool_val == cache_insert_data);



    /* engine config */

    int32_t engine_use_blas_threshold = 50;

    s = config.SetEngineConfigUseBlasThreshold(std::to_string(engine_use_blas_threshold));

    ASSERT_TRUE(s.ok());

    s = config.GetEngineConfigUseBlasThreshold(int32_val);

    ASSERT_TRUE(s.ok());

    ASSERT_TRUE(int32_val == engine_use_blas_threshold);



    int32_t engine_omp_thread_num = 8;

    s = config.SetEngineConfigOmpThreadNum(std::to_string(engine_omp_thread_num));

    ASSERT_TRUE(s.ok());

    s = config.GetEngineConfigOmpThreadNum(int32_val);

    ASSERT_TRUE(s.ok());

    ASSERT_TRUE(int32_val == engine_omp_thread_num);



    int32_t engine_gpu_search_threshold = 800;

    s = config.SetEngineConfigGpuSearchThreshold(std::to_string(engine_gpu_search_threshold));

    ASSERT_TRUE(s.ok());

    s = config.GetEngineConfigGpuSearchThreshold(int32_val);

    ASSERT_TRUE(s.ok());

    ASSERT_TRUE(int32_val == engine_gpu_search_threshold);



    /* resource config */

    std::string resource_mode = "simple";

    s = config.SetResourceConfigMode(resource_mode);

    ASSERT_TRUE(s.ok());

    s = config.GetResourceConfigMode(str_val);

    ASSERT_TRUE(s.ok());

    ASSERT_TRUE(str_val == resource_mode);



    std::vector<std::string> search_resources = {"cpu", "gpu0"};

    std::vector<std::string> res_vec;

    std::string res_str;

    milvus::server::StringHelpFunctions::MergeStringWithDelimeter(search_resources,

            milvus::server::CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, res_str);

    s = config.SetResourceConfigSearchResources(res_str);

    ASSERT_TRUE(s.ok());

    s = config.GetResourceConfigSearchResources(res_vec);

    ASSERT_TRUE(s.ok());

    for (size_t i = 0; i < search_resources.size(); i++) {

        ASSERT_TRUE(search_resources[i] == res_vec[i]);

    }



    int32_t resource_index_build_device = 0;

    s = config.SetResourceConfigIndexBuildDevice("gpu" + std::to_string(resource_index_build_device));

    ASSERT_TRUE(s.ok());

    s = config.GetResourceConfigIndexBuildDevice(int32_val);

    ASSERT_TRUE(s.ok());

    ASSERT_TRUE(int32_val == resource_index_build_device);

}



TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) {
@@ -226,9 +399,29 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { 
    s = config.SetResourceConfigMode("default");

    ASSERT_FALSE(s.ok());



    s = config.SetResourceConfigSearchResources("gpu10");

    ASSERT_FALSE(s.ok());

    s = config.SetResourceConfigSearchResources("cpu");

    ASSERT_FALSE(s.ok());



    s = config.SetResourceConfigIndexBuildDevice("gup2");

    ASSERT_FALSE(s.ok());

    s = config.SetResourceConfigIndexBuildDevice("gpu16");

    ASSERT_FALSE(s.ok());

}



TEST_F(ConfigTest, SERVER_CONFIG_TEST) {

    std::string config_path(CONFIG_PATH);

    milvus::server::Config& config = milvus::server::Config::GetInstance();

    milvus::Status s = config.LoadConfigFile(config_path + VALID_CONFIG_FILE);

    ASSERT_TRUE(s.ok());



    s = config.ValidateConfig();

    ASSERT_TRUE(s.ok());



    config.PrintAll();



    s = config.ResetDefaultConfig();

    ASSERT_TRUE(s.ok());

}