merge json to master to get docker image (#1500) (a08b51c2) · Commits · SUMMER2020 / students / proj-2018085

CHANGELOG.md

+4 −3

Original line number	Diff line number	Diff line
		@@ -11,7 +11,7 @@ Please mark all change in change log and use the issue from GitHub
		- \#805 IVFTest.gpu_seal_test unittest failed
		- \#831 Judge branch error in CommonUtil.cpp
		- \#977 Server crash when create tables concurrently
		- \#990 check gpu resources setting when assign repeated value
		- \#990 Check gpu resources setting when assign repeated value
		- \#995 table count set to 0 if no tables found
		- \#1010 improve error message when offset or page_size is equal 0
		- \#1022 check if partition name is legal
		@@ -19,8 +19,8 @@ Please mark all change in change log and use the issue from GitHub
		- \#1029 check if table exists when try to delete partition
		- \#1066 optimize http insert and search speed
		- \#1067 Add binary vectors support in http server
		- \#1075 improve error message when page size or offset is illegal
		- \#1082 check page_size or offset value to avoid float
		- \#1075 Improve error message when page size or offset is illegal
		- \#1082 Check page_size or offset value to avoid float
		- \#1115 http server support load table into memory
		- \#1152 Error log output continuously after server start
		- \#1211 Server down caused by searching with index_type: HNSW
		@@ -86,6 +86,7 @@ Please mark all change in change log and use the issue from GitHub
		- \#1320 Remove debug logging from faiss
		- \#1426 Support to configure whether to enabled autoflush and the autoflush interval
		- \#1444 Improve delete
		- \#1448 General proto api for NNS libraries
		- \#1480 Add return code for AVX512 selection
		- \#1524 Update config "preload_table" description

ci/jenkins/step/singleDevTest.groovy

+1 −0

Original line number	Diff line number	Diff line
		@@ -2,6 +2,7 @@ timeout(time: 60, unit: 'MINUTES') {
		dir ("tests/milvus_python_test") {
		// sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'
		sh 'python3 -m pip install -r requirements.txt'
		sh 'python3 -m pip install git+https://github.com/BossZou/pymilvus.git@nns'
		sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local"
		}

core/src/db/DB.h

+6 −6

Original line number	Diff line number	Diff line
		@@ -112,18 +112,18 @@ class DB {

		virtual Status
		QueryByID(const std::shared_ptr<server::Context>& context, const std::string& table_id,
		const std::vector<std::string>& partition_tags, uint64_t k, uint64_t nprobe, IDNumber vector_id,
		ResultIds& result_ids, ResultDistances& result_distances) = 0;
		const std::vector<std::string>& partition_tags, uint64_t k, const milvus::json& extra_params,
		IDNumber vector_id, ResultIds& result_ids, ResultDistances& result_distances) = 0;

		virtual Status
		Query(const std::shared_ptr<server::Context>& context, const std::string& table_id,
		const std::vector<std::string>& partition_tags, uint64_t k, uint64_t nprobe, const VectorsData& vectors,
		ResultIds& result_ids, ResultDistances& result_distances) = 0;
		const std::vector<std::string>& partition_tags, uint64_t k, const milvus::json& extra_params,
		const VectorsData& vectors, ResultIds& result_ids, ResultDistances& result_distances) = 0;

		virtual Status
		QueryByFileID(const std::shared_ptr<server::Context>& context, const std::string& table_id,
		const std::vector<std::string>& file_ids, uint64_t k, uint64_t nprobe, const VectorsData& vectors,
		ResultIds& result_ids, ResultDistances& result_distances) = 0;
		const std::vector<std::string>& file_ids, uint64_t k, const milvus::json& extra_params,
		const VectorsData& vectors, ResultIds& result_ids, ResultDistances& result_distances) = 0;

		virtual Status
		Size(uint64_t& result) = 0;

core/src/db/DBImpl.cpp

+18 −15

Original line number	Diff line number	Diff line
		@@ -371,8 +371,10 @@ DBImpl::PreloadTable(const std::string& table_id) {
		} else {
		engine_type = (EngineType)file.engine_type_;
		}
		ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, file.location_, engine_type,
		(MetricType)file.metric_type_, file.nlist_);

		auto json = milvus::json::parse(file.index_params_);
		ExecutionEnginePtr engine =
		EngineFactory::Build(file.dimension_, file.location_, engine_type, (MetricType)file.metric_type_, json);
		fiu_do_on("DBImpl.PreloadTable.null_engine", engine = nullptr);
		if (engine == nullptr) {
		ENGINE_LOG_ERROR << "Invalid engine type";
		@@ -382,7 +384,7 @@ DBImpl::PreloadTable(const std::string& table_id) {
		size += engine->PhysicalSize();
		fiu_do_on("DBImpl.PreloadTable.exceed_cache", size = available_size + 1);
		if (size > available_size) {
		ENGINE_LOG_DEBUG << "Pre-load canceled since cache almost full";
		ENGINE_LOG_DEBUG << "Pre-load cancelled since cache is almost full";
		return Status(SERVER_CACHE_FULL, "Cache is full");
		} else {
		try {
		@@ -1110,8 +1112,8 @@ DBImpl::DropIndex(const std::string& table_id) {

		Status
		DBImpl::QueryByID(const std::shared_ptr<server::Context>& context, const std::string& table_id,
		const std::vector<std::string>& partition_tags, uint64_t k, uint64_t nprobe, IDNumber vector_id,
		ResultIds& result_ids, ResultDistances& result_distances) {
		const std::vector<std::string>& partition_tags, uint64_t k, const milvus::json& extra_params,
		IDNumber vector_id, ResultIds& result_ids, ResultDistances& result_distances) {
		if (!initialized_.load(std::memory_order_acquire)) {
		return SHUTDOWN_ERROR;
		}
		@@ -1119,14 +1121,15 @@ DBImpl::QueryByID(const std::shared_ptr<server::Context>& context, const std::st
		VectorsData vectors_data = VectorsData();
		vectors_data.id_array_.emplace_back(vector_id);
		vectors_data.vector_count_ = 1;
		Status result = Query(context, table_id, partition_tags, k, nprobe, vectors_data, result_ids, result_distances);
		Status result =
		Query(context, table_id, partition_tags, k, extra_params, vectors_data, result_ids, result_distances);
		return result;
		}

		Status
		DBImpl::Query(const std::shared_ptr<server::Context>& context, const std::string& table_id,
		const std::vector<std::string>& partition_tags, uint64_t k, uint64_t nprobe, const VectorsData& vectors,
		ResultIds& result_ids, ResultDistances& result_distances) {
		const std::vector<std::string>& partition_tags, uint64_t k, const milvus::json& extra_params,
		const VectorsData& vectors, ResultIds& result_ids, ResultDistances& result_distances) {
		auto query_ctx = context->Child("Query");

		if (!initialized_.load(std::memory_order_acquire)) {
		@@ -1169,7 +1172,7 @@ DBImpl::Query(const std::shared_ptr<server::Context>& context, const std::string
		}

		cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info before query
		status = QueryAsync(query_ctx, table_id, files_array, k, nprobe, vectors, result_ids, result_distances);
		status = QueryAsync(query_ctx, table_id, files_array, k, extra_params, vectors, result_ids, result_distances);
		cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info after query

		query_ctx->GetTraceContext()->GetSpan()->Finish();
		@@ -1179,8 +1182,8 @@ DBImpl::Query(const std::shared_ptr<server::Context>& context, const std::string

		Status
		DBImpl::QueryByFileID(const std::shared_ptr<server::Context>& context, const std::string& table_id,
		const std::vector<std::string>& file_ids, uint64_t k, uint64_t nprobe, const VectorsData& vectors,
		ResultIds& result_ids, ResultDistances& result_distances) {
		const std::vector<std::string>& file_ids, uint64_t k, const milvus::json& extra_params,
		const VectorsData& vectors, ResultIds& result_ids, ResultDistances& result_distances) {
		auto query_ctx = context->Child("Query by file id");

		if (!initialized_.load(std::memory_order_acquire)) {
		@@ -1208,7 +1211,7 @@ DBImpl::QueryByFileID(const std::shared_ptr<server::Context>& context, const std
		}

		cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info before query
		status = QueryAsync(query_ctx, table_id, files_array, k, nprobe, vectors, result_ids, result_distances);
		status = QueryAsync(query_ctx, table_id, files_array, k, extra_params, vectors, result_ids, result_distances);
		cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info after query

		query_ctx->GetTraceContext()->GetSpan()->Finish();
		@@ -1230,8 +1233,8 @@ DBImpl::Size(uint64_t& result) {
		///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		Status
		DBImpl::QueryAsync(const std::shared_ptr<server::Context>& context, const std::string& table_id,
		const meta::TableFilesSchema& files, uint64_t k, uint64_t nprobe, const VectorsData& vectors,
		ResultIds& result_ids, ResultDistances& result_distances) {
		const meta::TableFilesSchema& files, uint64_t k, const milvus::json& extra_params,
		const VectorsData& vectors, ResultIds& result_ids, ResultDistances& result_distances) {
		auto query_async_ctx = context->Child("Query Async");

		server::CollectQueryMetrics metrics(vectors.vector_count_);
		@@ -1242,7 +1245,7 @@ DBImpl::QueryAsync(const std::shared_ptr<server::Context>& context, const std::s
		auto status = OngoingFileChecker::GetInstance().MarkOngoingFiles(files);

		ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size();
		scheduler::SearchJobPtr job = std::make_shared<scheduler::SearchJob>(query_async_ctx, k, nprobe, vectors);
		scheduler::SearchJobPtr job = std::make_shared<scheduler::SearchJob>(query_async_ctx, k, extra_params, vectors);
		for (auto& file : files) {
		scheduler::TableFileSchemaPtr file_ptr = std::make_shared<meta::TableFileSchema>(file);
		job->AddIndexFile(file_ptr);

core/src/db/DBImpl.h

+8 −8

Original line number	Diff line number	Diff line
		@@ -131,18 +131,18 @@ class DBImpl : public DB, public server::CacheConfigHandler {

		Status
		QueryByID(const std::shared_ptr<server::Context>& context, const std::string& table_id,
		const std::vector<std::string>& partition_tags, uint64_t k, uint64_t nprobe, IDNumber vector_id,
		ResultIds& result_ids, ResultDistances& result_distances) override;
		const std::vector<std::string>& partition_tags, uint64_t k, const milvus::json& extra_params,
		IDNumber vector_id, ResultIds& result_ids, ResultDistances& result_distances) override;

		Status
		Query(const std::shared_ptr<server::Context>& context, const std::string& table_id,
		const std::vector<std::string>& partition_tags, uint64_t k, uint64_t nprobe, const VectorsData& vectors,
		ResultIds& result_ids, ResultDistances& result_distances) override;
		const std::vector<std::string>& partition_tags, uint64_t k, const milvus::json& extra_params,
		const VectorsData& vectors, ResultIds& result_ids, ResultDistances& result_distances) override;

		Status
		QueryByFileID(const std::shared_ptr<server::Context>& context, const std::string& table_id,
		const std::vector<std::string>& file_ids, uint64_t k, uint64_t nprobe, const VectorsData& vectors,
		ResultIds& result_ids, ResultDistances& result_distances) override;
		const std::vector<std::string>& file_ids, uint64_t k, const milvus::json& extra_params,
		const VectorsData& vectors, ResultIds& result_ids, ResultDistances& result_distances) override;

		Status
		Size(uint64_t& result) override;
		@@ -154,8 +154,8 @@ class DBImpl : public DB, public server::CacheConfigHandler {
		private:
		Status
		QueryAsync(const std::shared_ptr<server::Context>& context, const std::string& table_id,
		const meta::TableFilesSchema& files, uint64_t k, uint64_t nprobe, const VectorsData& vectors,
		ResultIds& result_ids, ResultDistances& result_distances);
		const meta::TableFilesSchema& files, uint64_t k, const milvus::json& extra_params,
		const VectorsData& vectors, ResultIds& result_ids, ResultDistances& result_distances);

		Status
		GetVectorByIdHelper(const std::string& table_id, IDNumber vector_id, VectorsData& vector,