MS-613 Integrate new faiss (a42dd429) · Commits · SUMMER2020 / students / proj-2018085

cpp/src/core/cmake/ThirdPartyPackagesCore.cmake

+3 −2

Original line number	Diff line number	Diff line
		@@ -235,8 +235,9 @@ else()
		message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}")
		endif()
		# set(FAISS_MD5 "a589663865a8558205533c8ac414278c")
		# set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c
		set(FAISS_MD5 "21deb1c708490ca40ecb899122c01403") # commit-id 643e48f479637fd947e7b93fa4ca72b38ecc9a39
		# set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c branch-0.2.0
		# set(FAISS_MD5 "21deb1c708490ca40ecb899122c01403") # commit-id 643e48f479637fd947e7b93fa4ca72b38ecc9a39 branch-0.2.0
		set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1

		if(DEFINED ENV{KNOWHERE_ARROW_URL})
		set(ARROW_SOURCE_URL "$ENV{KNOWHERE_ARROW_URL}")

cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp

+13 −7

Original line number	Diff line number	Diff line
		@@ -147,27 +147,33 @@ void GPUIVF::search_impl(int64_t n,
		std::lock_guard<std::mutex> lk(mutex_);

		// TODO(linxj): gpu index support GenParams
		if (auto device_index = std::static_pointer_cast<faiss::gpu::GpuIndexIVF>(index_)) {
		if (auto device_index = std::dynamic_pointer_cast<faiss::gpu::GpuIndexIVF>(index_)) {
		auto search_cfg = std::dynamic_pointer_cast<IVFCfg>(cfg);
		device_index->setNumProbes(search_cfg->nprobe);

		{
		// TODO(linxj): allocate mem
		// TODO(linxj): allocate gpu mem
		ResScope rs(res_, gpu_id_);
		device_index->search(n, (float *) data, k, distances, labels);
		}
		} else {
		KNOWHERE_THROW_MSG("Not a GpuIndexIVF type.");
		}
		}

		VectorIndexPtr GPUIVF::CopyGpuToCpu(const Config &config) {
		std::lock_guard<std::mutex> lk(mutex_);

		if ( auto device_idx = std::dynamic_pointer_cast<faiss::gpu::GpuIndexIVF>(index_)) {
		faiss::Index *device_index = index_.get();
		faiss::Index *host_index = faiss::gpu::index_gpu_to_cpu(device_index);

		std::shared_ptr<faiss::Index> new_index;
		new_index.reset(host_index);
		return std::make_shared<IVF>(new_index);
		} else {
		return std::make_shared<IVF>(index_);
		}
		}

		VectorIndexPtr GPUIVF::Clone() {

cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.cpp

+1 −1

Original line number	Diff line number	Diff line
		@@ -141,7 +141,7 @@ void IVF::set_index_model(IndexModelPtr model) {
		}

		std::shared_ptr<faiss::IVFSearchParameters> IVF::GenParams(const Config &config) {
		auto params = std::make_shared<faiss::IVFPQSearchParameters>();
		auto params = std::make_shared<faiss::IVFSearchParameters>();

		auto search_cfg = std::dynamic_pointer_cast<IVFCfg>(config);
		params->nprobe = search_cfg->nprobe;

cpp/src/core/test/test_ivf.cpp

+24 −9

Original line number	Diff line number	Diff line
		@@ -134,6 +134,15 @@ class IVFTest
		FaissGpuResourceMgr::GetInstance().Free();
		}

		VectorIndexPtr ChooseTodo() {
		std::vector<std::string> gpu_idx{"GPUIVFSQ"};
		auto finder = std::find(gpu_idx.cbegin(), gpu_idx.cend(), index_type);
		if (finder != gpu_idx.cend()) {
		return CopyCpuToGpu(index_, device_id, Config());
		}
		return index_;
		}

		protected:
		std::string index_type;
		Config conf;
		@@ -193,7 +202,9 @@ TEST_P(IVFTest, ivf_basic) {
		index_->Add(base_dataset, conf);
		EXPECT_EQ(index_->Count(), nb);
		EXPECT_EQ(index_->Dimension(), dim);
		auto result = index_->Search(query_dataset, conf);

		auto new_idx = ChooseTodo();
		auto result = new_idx->Search(query_dataset, conf);
		AssertAnns(result, nq, conf->k);
		//PrintResult(result, nq, k);
		}
		@@ -250,7 +261,8 @@ TEST_P(IVFTest, ivf_serialize) {

		index_->set_index_model(model);
		index_->Add(base_dataset, conf);
		auto result = index_->Search(query_dataset, conf);
		auto new_idx = ChooseTodo();
		auto result = new_idx->Search(query_dataset, conf);
		AssertAnns(result, nq, conf->k);
		}

		@@ -274,7 +286,8 @@ TEST_P(IVFTest, ivf_serialize) {
		index_->Load(binaryset);
		EXPECT_EQ(index_->Count(), nb);
		EXPECT_EQ(index_->Dimension(), dim);
		auto result = index_->Search(query_dataset, conf);
		auto new_idx = ChooseTodo();
		auto result = new_idx->Search(query_dataset, conf);
		AssertAnns(result, nq, conf->k);
		}
		}
		@@ -290,7 +303,8 @@ TEST_P(IVFTest, clone_test) {
		index_->Add(base_dataset, conf);
		EXPECT_EQ(index_->Count(), nb);
		EXPECT_EQ(index_->Dimension(), dim);
		auto result = index_->Search(query_dataset, conf);
		auto new_idx = ChooseTodo();
		auto result = new_idx->Search(query_dataset, conf);
		AssertAnns(result, nq, conf->k);
		//PrintResult(result, nq, k);

		@@ -382,7 +396,8 @@ TEST_P(IVFTest, seal_test) {
		index_->Add(base_dataset, conf);
		EXPECT_EQ(index_->Count(), nb);
		EXPECT_EQ(index_->Dimension(), dim);
		auto result = index_->Search(query_dataset, conf);
		auto new_idx = ChooseTodo();
		auto result = new_idx->Search(query_dataset, conf);
		AssertAnns(result, nq, conf->k);

		auto cpu_idx = CopyGpuToCpu(index_, Config());
		@@ -504,8 +519,8 @@ TEST_F(GPURESTEST, gpuivfsq) {
		auto model = index_->Train(base_dataset, conf);
		index_->set_index_model(model);
		index_->Add(base_dataset, conf);
		auto result = index_->Search(query_dataset, conf);
		AssertAnns(result, nq, k);
		// auto result = index_->Search(query_dataset, conf);
		// AssertAnns(result, nq, k);

		auto cpu_idx = CopyGpuToCpu(index_, Config());
		cpu_idx->Seal();
		@@ -578,8 +593,8 @@ TEST_F(GPURESTEST, copyandsearch) {
		auto model = index_->Train(base_dataset, conf);
		index_->set_index_model(model);
		index_->Add(base_dataset, conf);
		auto result = index_->Search(query_dataset, conf);
		AssertAnns(result, nq, k);
		// auto result = index_->Search(query_dataset, conf);
		// AssertAnns(result, nq, k);

		auto cpu_idx = CopyGpuToCpu(index_, Config());
		cpu_idx->Seal();

cpp/src/core/thirdparty/versions.txt

+1 −1

Original line number	Diff line number	Diff line
		@@ -3,4 +3,4 @@ BOOST_VERSION=1.70.0
		GTEST_VERSION=1.8.1
		LAPACK_VERSION=v3.8.0
		OPENBLAS_VERSION=v0.3.6
		FAISS_VERSION=branch-0.2.0
		No newline at end of file
		FAISS_VERSION=branch-0.2.1
		No newline at end of file