merge 0.6.0 (07ac2e32) · Commits · SUMMER2020 / students / proj-2018085

CHANGELOG.md

+5 −0

Original line number	Diff line number	Diff line
		@@ -9,6 +9,7 @@ Please mark all change in change log and use the ticket from JIRA.
		- \#248 - Reside src/external in thirdparty
		- \#316 - Some files not merged after vectors added
		- \#327 - Search does not use GPU when index type is FLAT
		- \#331 - Add exception handle when search fail
		- \#340 - Test cases run failed on 0.6.0
		- \#353 - Rename config.h.in to version.h.in
		- \#374 - sdk_simple return empty result
		@@ -35,12 +36,15 @@ Please mark all change in change log and use the ticket from JIRA.
		- \#543 - client raise exception in shards when search results is empty
		- \#545 - Avoid dead circle of build index thread when error occurs
		- \#547 - NSG build failed using GPU-edition if set gpu_enable false
		- \#548 - NSG search accuracy is too low
		- \#552 - Server down during building index_type: IVF_PQ using GPU-edition
		- \#561 - Milvus server should report exception/error message or terminate on mysql metadata backend error
		- \#579 - Build index hang in GPU version when gpu_resources disabled
		- \#596 - Frequently insert operation cost too much disk space
		- \#599 - Build index log is incorrect
		- \#602 - Optimizer specify wrong gpu_id
		- \#606 - No log generated during building index with CPU
		- \#631 - FAISS isn't compiled with O3 option

		## Feature
		- \#12 - Pure CPU version for Milvus
		@@ -75,6 +79,7 @@ Please mark all change in change log and use the ticket from JIRA.
		- \#470 - Small raw files should not be build index
		- \#584 - Intergrate internal FAISS
		- \#611 - Remove MILVUS_CPU_VERSION
		- \#634 - FAISS GPU version is compiled with O0

		## Task

core/src/db/DBImpl.cpp

+0 −4

Original line number	Diff line number	Diff line
		@@ -1044,11 +1044,7 @@ DBImpl::BuildTableIndexRecursively(const std::string& table_id, const TableIndex
		if (!failed_files.empty()) {
		std::string msg = "Failed to build index for " + std::to_string(failed_files.size()) +
		((failed_files.size() == 1) ? " file" : " files");
		#ifdef MILVUS_GPU_VERSION
		msg += ", file size is too large or gpu memory is not enough.";
		#else
		msg += ", please double check index parameters.";
		#endif
		return Status(DB_ERROR, msg);
		}

core/src/db/engine/ExecutionEngineImpl.cpp

+17 −9

Original line number	Diff line number	Diff line
		@@ -86,6 +86,11 @@ ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index, const std::string& l

		VecIndexPtr
		ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
		#ifdef MILVUS_GPU_VERSION
		server::Config& config = server::Config::GetInstance();
		bool gpu_resource_enable = true;
		config.GetGpuResourceConfigEnable(gpu_resource_enable);
		#endif
		std::shared_ptr<VecIndex> index;
		switch (type) {
		case EngineType::FAISS_IDMAP: {
		@@ -94,18 +99,20 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
		}
		case EngineType::FAISS_IVFFLAT: {
		#ifdef MILVUS_GPU_VERSION
		if (gpu_resource_enable)
		index = GetVecIndexFactory(IndexType::FAISS_IVFFLAT_MIX);
		#else
		index = GetVecIndexFactory(IndexType::FAISS_IVFFLAT_CPU);
		else
		#endif
		index = GetVecIndexFactory(IndexType::FAISS_IVFFLAT_CPU);
		break;
		}
		case EngineType::FAISS_IVFSQ8: {
		#ifdef MILVUS_GPU_VERSION
		if (gpu_resource_enable)
		index = GetVecIndexFactory(IndexType::FAISS_IVFSQ8_MIX);
		#else
		index = GetVecIndexFactory(IndexType::FAISS_IVFSQ8_CPU);
		else
		#endif
		index = GetVecIndexFactory(IndexType::FAISS_IVFSQ8_CPU);
		break;
		}
		case EngineType::NSG_MIX: {
		@@ -120,10 +127,11 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
		#endif
		case EngineType::FAISS_PQ: {
		#ifdef MILVUS_GPU_VERSION
		if (gpu_resource_enable)
		index = GetVecIndexFactory(IndexType::FAISS_IVFPQ_MIX);
		#else
		index = GetVecIndexFactory(IndexType::FAISS_IVFPQ_CPU);
		else
		#endif
		index = GetVecIndexFactory(IndexType::FAISS_IVFPQ_CPU);
		break;
		}
		case EngineType::SPTAG_KDT: {

core/src/index/cmake/ThirdPartyPackagesCore.cmake

+1 −1

Original line number	Diff line number	Diff line
		@@ -708,7 +708,7 @@ macro(build_faiss)
		set(FAISS_CONFIGURE_ARGS
		"--prefix=${FAISS_PREFIX}"
		"CFLAGS=${EP_C_FLAGS}"
		"CXXFLAGS=${EP_CXX_FLAGS} -mavx2 -mf16c"
		"CXXFLAGS=${EP_CXX_FLAGS} -mavx2 -mf16c -O3"
		--without-python)

		if (FAISS_WITH_MKL)

core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.cpp

+34 −0

Original line number	Diff line number	Diff line
		@@ -126,4 +126,38 @@ GPUIDMAP::search_impl(int64_t n, const float* data, int64_t k, float* distances,
		index_->search(n, (float*)data, k, distances, labels);
		}

		void
		GPUIDMAP::GenGraph(float* data, const int64_t& k, Graph& graph, const Config& config) {
		int64_t K = k + 1;
		auto ntotal = Count();

		size_t dim = config->d;
		auto batch_size = 1000;
		auto tail_batch_size = ntotal % batch_size;
		auto batch_search_count = ntotal / batch_size;
		auto total_search_count = tail_batch_size == 0 ? batch_search_count : batch_search_count + 1;

		std::vector<float> res_dis(K * batch_size);
		graph.resize(ntotal);
		Graph res_vec(total_search_count);
		for (int i = 0; i < total_search_count; ++i) {
		auto b_size = (i == (total_search_count - 1)) && tail_batch_size != 0 ? tail_batch_size : batch_size;

		auto& res = res_vec[i];
		res.resize(K * b_size);

		auto xq = data + batch_size * dim * i;
		search_impl(b_size, (float*)xq, K, res_dis.data(), res.data(), config);

		for (int j = 0; j < b_size; ++j) {
		auto& node = graph[batch_size * i + j];
		node.resize(k);
		auto start_pos = j * K + 1;
		for (int m = 0, cursor = start_pos; m < k && cursor < start_pos + k; ++m, ++cursor) {
		node[m] = res[cursor];
		}
		}
		}
		}

		} // namespace knowhere