Merge pull request #210 from cydrain/caiyd_test_config (113d08f8) · Commits · SUMMER2020 / students / proj-2018085

CHANGELOG.md

+12 −2

Original line number	Diff line number	Diff line
		# Changelog

		Please mark all change in change log and use the ticket from JIRA.
		# Milvus 0.5.3 (TODO)

		# Milvus 0.5.2 (TODO)
		## Bug

		## Feature

		## Improvement
		- \#207 - Add more unittest for config set/get

		## Task

		# Milvus 0.5.2 (2019-11-05)

		## Bug
		- \#194 - Search faild: message="Table file doesn't exist"
		@@ -10,7 +20,7 @@ Please mark all change in change log and use the ticket from JIRA.
		## Feature

		## Improvement
		- \#190 - Update default config:use_blas_threshold to 1100 and server version printout to 0.52
		- \#190 - Update default config:use_blas_threshold to 1100 and server version printout to 0.5.2

		## Task

core/src/server/Config.cpp

+47 −15

Original line number	Diff line number	Diff line
		@@ -25,6 +25,7 @@
		#include "config/YamlConfigMgr.h"
		#include "server/Config.h"
		#include "utils/CommonUtil.h"
		#include "utils/StringHelpFunctions.h"
		#include "utils/ValidationUtil.h"

		namespace milvus {
		@@ -341,6 +342,11 @@ Config::ResetDefaultConfig() {
		return s;
		}

		s = SetResourceConfigSearchResources(CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT);
		if (!s.ok()) {
		return s;
		}

		s = SetResourceConfigIndexBuildDevice(CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT);
		if (!s.ok()) {
		return s;
		@@ -788,6 +794,17 @@ Config::GetConfigStr(const std::string& parent_key, const std::string& child_key
		return value;
		}

		std::string
		Config::GetConfigSequenceStr(const std::string& parent_key, const std::string& child_key, const std::string& delim) {
		std::string value;
		if (!GetConfigValueInMem(parent_key, child_key, value).ok()) {
		std::vector<std::string> sequence = GetConfigNode(parent_key).GetSequence(child_key);
		server::StringHelpFunctions::MergeStringWithDelimeter(sequence, delim, value);
		SetConfigValueInMem(parent_key, child_key, value);
		}
		return value;
		}

		Status
		Config::GetServerConfigAddress(std::string& value) {
		value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT);
		@@ -1011,8 +1028,9 @@ Config::GetResourceConfigMode(std::string& value) {

		Status
		Config::GetResourceConfigSearchResources(std::vector<std::string>& value) {
		ConfigNode resource_config = GetConfigNode(CONFIG_RESOURCE);
		value = resource_config.GetSequence(CONFIG_RESOURCE_SEARCH_RESOURCES);
		std::string str = GetConfigSequenceStr(CONFIG_RESOURCE, CONFIG_RESOURCE_SEARCH_RESOURCES,
		CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER);
		server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, value);
		return CheckResourceConfigSearchResources(value);
		}

		@@ -1155,7 +1173,7 @@ Config::SetMetricConfigEnableMonitor(const std::string& value) {
		return s;
		}

		SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_ENABLE_MONITOR, value);
		SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value);
		return Status::OK();
		}

		@@ -1166,7 +1184,7 @@ Config::SetMetricConfigCollector(const std::string& value) {
		return s;
		}

		SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_COLLECTOR, value);
		SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value);
		return Status::OK();
		}

		@@ -1177,7 +1195,7 @@ Config::SetMetricConfigPrometheusPort(const std::string& value) {
		return s;
		}

		SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_PROMETHEUS_PORT, value);
		SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value);
		return Status::OK();
		}

		@@ -1189,7 +1207,7 @@ Config::SetCacheConfigCpuCacheCapacity(const std::string& value) {
		return s;
		}

		SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_CACHE_CAPACITY, value);
		SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, value);
		return Status::OK();
		}

		@@ -1200,7 +1218,7 @@ Config::SetCacheConfigCpuCacheThreshold(const std::string& value) {
		return s;
		}

		SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value);
		SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value);
		return Status::OK();
		}

		@@ -1211,7 +1229,7 @@ Config::SetCacheConfigGpuCacheCapacity(const std::string& value) {
		return s;
		}

		SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_CACHE_CAPACITY, value);
		SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, value);
		return Status::OK();
		}

		@@ -1222,7 +1240,7 @@ Config::SetCacheConfigGpuCacheThreshold(const std::string& value) {
		return s;
		}

		SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value);
		SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value);
		return Status::OK();
		}

		@@ -1233,7 +1251,7 @@ Config::SetCacheConfigCacheInsertData(const std::string& value) {
		return s;
		}

		SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CACHE_INSERT_DATA, value);
		SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value);
		return Status::OK();
		}

		@@ -1245,7 +1263,7 @@ Config::SetEngineConfigUseBlasThreshold(const std::string& value) {
		return s;
		}

		SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value);
		SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value);
		return Status::OK();
		}

		@@ -1256,7 +1274,7 @@ Config::SetEngineConfigOmpThreadNum(const std::string& value) {
		return s;
		}

		SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_OMP_THREAD_NUM, value);
		SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value);
		return Status::OK();
		}

		@@ -1267,7 +1285,7 @@ Config::SetEngineConfigGpuSearchThreshold(const std::string& value) {
		return s;
		}

		SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value);
		SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value);
		return Status::OK();
		}

		@@ -1279,7 +1297,21 @@ Config::SetResourceConfigMode(const std::string& value) {
		return s;
		}

		SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_MODE, value);
		SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value);
		return Status::OK();
		}

		Status
		Config::SetResourceConfigSearchResources(const std::string& value) {
		std::vector<std::string> res_vec;
		server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, res_vec);

		Status s = CheckResourceConfigSearchResources(res_vec);
		if (!s.ok()) {
		return s;
		}

		SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_SEARCH_RESOURCES, value);
		return Status::OK();
		}

		@@ -1290,7 +1322,7 @@ Config::SetResourceConfigIndexBuildDevice(const std::string& value) {
		return s;
		}

		SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, value);
		SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, value);
		return Status::OK();
		}

core/src/server/Config.h

+6 −0

Original line number	Diff line number	Diff line
		@@ -92,6 +92,8 @@ static const char* CONFIG_RESOURCE = "resource_config";
		static const char* CONFIG_RESOURCE_MODE = "mode";
		static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple";
		static const char* CONFIG_RESOURCE_SEARCH_RESOURCES = "search_resources";
		static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER = ",";
		static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu,gpu0";
		static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE = "index_build_device";
		static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "gpu0";

		@@ -183,6 +185,8 @@ class Config {

		std::string
		GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = "");
		std::string
		GetConfigSequenceStr(const std::string& parent_key, const std::string& child_key, const std::string& delim);

		public:
		/* server config */
		@@ -304,6 +308,8 @@ class Config {
		Status
		SetResourceConfigMode(const std::string& value);
		Status
		SetResourceConfigSearchResources(const std::string& value);
		Status
		SetResourceConfigIndexBuildDevice(const std::string& value);

		private:

core/src/utils/StringHelpFunctions.cpp

+29 −15

Original line number	Diff line number	Diff line
		@@ -39,39 +39,53 @@ StringHelpFunctions::TrimStringQuote(std::string& string, const std::string& qou
		}
		}

		Status
		void
		StringHelpFunctions::SplitStringByDelimeter(const std::string& str, const std::string& delimeter,
		std::vector<std::string>& result) {
		if (str.empty()) {
		return Status::OK();
		return;
		}

		size_t last = 0;
		size_t index = str.find_first_of(delimeter, last);
		while (index != std::string::npos) {
		result.emplace_back(str.substr(last, index - last));
		last = index + 1;
		index = str.find_first_of(delimeter, last);
		size_t prev = 0, pos = 0;
		while (true) {
		pos = str.find_first_of(delimeter, prev);
		if (pos == std::string::npos) {
		result.emplace_back(str.substr(prev));
		break;
		} else {
		result.emplace_back(str.substr(prev, pos - prev));
		prev = pos + 1;
		}
		}
		if (index - last > 0) {
		std::string temp = str.substr(last);
		result.emplace_back(temp);
		}

		return Status::OK();
		void
		StringHelpFunctions::MergeStringWithDelimeter(const std::vector<std::string>& strs, const std::string& delimeter,
		std::string& result) {
		if (strs.empty()) {
		result = "";
		return;
		}

		result = strs[0];
		for (size_t i = 1; i < strs.size(); i++) {
		result = result + delimeter + strs[i];
		}
		}

		Status
		StringHelpFunctions::SplitStringByQuote(const std::string& str, const std::string& delimeter, const std::string& quote,
		std::vector<std::string>& result) {
		if (quote.empty()) {
		return SplitStringByDelimeter(str, delimeter, result);
		SplitStringByDelimeter(str, delimeter, result);
		return Status::OK();
		}

		size_t last = 0;
		size_t index = str.find_first_of(quote, last);
		if (index == std::string::npos) {
		return SplitStringByDelimeter(str, delimeter, result);
		SplitStringByDelimeter(str, delimeter, result);
		return Status::OK();
		}

		std::string process_str = str;
		@@ -116,7 +130,7 @@ StringHelpFunctions::SplitStringByQuote(const std::string& str, const std::strin
		}

		if (!process_str.empty()) {
		return SplitStringByDelimeter(process_str, delimeter, result);
		SplitStringByDelimeter(process_str, delimeter, result);
		}

		return Status::OK();

core/src/utils/StringHelpFunctions.h

+4 −1

Original line number	Diff line number	Diff line
		@@ -43,9 +43,12 @@ class StringHelpFunctions {
		// ,b, \| b \|
		// ,, \| \|
		// a a
		static Status
		static void
		SplitStringByDelimeter(const std::string& str, const std::string& delimeter, std::vector<std::string>& result);

		static void
		MergeStringWithDelimeter(const std::vector<std::string>& strs, const std::string& delimeter, std::string& result);

		// assume the table has two columns, quote='\"', delimeter=','
		// a,b a \| b
		// "aa,gg,yy",b aa,gg,yy \| b