Merge pull request #1573 from yhmo/binary (b20e16ab) · Commits · SUMMER2020 / students / proj-2018085

CHANGELOG.md

+7 −6

Original line number	Diff line number	Diff line
		@@ -12,16 +12,16 @@ Please mark all change in change log and use the issue from GitHub
		- \#831 Judge branch error in CommonUtil.cpp
		- \#977 Server crash when create tables concurrently
		- \#990 Check gpu resources setting when assign repeated value
		- \#995 table count set to 0 if no tables found
		- \#995 Table count set to 0 if no tables found
		- \#1010 Improve error message when offset or page_size is equal 0
		- \#1022 check if partition name is legal
		- \#1028 check if table exists when show partitions
		- \#1029 check if table exists when try to delete partition
		- \#1066 optimize http insert and search speed
		- \#1022 Check if partition name is legal
		- \#1028 Check if table exists when show partitions
		- \#1029 Check if table exists when try to delete partition
		- \#1066 Optimize http insert and search speed
		- \#1067 Add binary vectors support in http server
		- \#1075 Improve error message when page size or offset is illegal
		- \#1082 Check page_size or offset value to avoid float
		- \#1115 http server support load table into memory
		- \#1115 Http server support load table into memory
		- \#1152 Error log output continuously after server start
		- \#1211 Server down caused by searching with index_type: HNSW
		- \#1240 Update license declaration
		@@ -47,6 +47,7 @@ Please mark all change in change log and use the issue from GitHub
		- \#1556 Index file not created after table and index created
		- \#1560 Search crashed with Super-high dimensional binary vector
		- \#1564 Too low recall for glove-200-angular, ivf_pq index
		- \#1571 Meta engine type become IDMAP after dropping index for BINARY table
		- \#1574 Set all existing bitset in cache when applying deletes
		- \#1577 Row count incorrect if delete vectors then create index

core/src/db/DBImpl.cpp

+5 −5

Original line number	Diff line number	Diff line
		@@ -366,8 +366,8 @@ DBImpl::PreloadTable(const std::string& table_id) {
		if (file.file_type_ == meta::TableFileSchema::FILE_TYPE::RAW \|\|
		file.file_type_ == meta::TableFileSchema::FILE_TYPE::TO_INDEX \|\|
		file.file_type_ == meta::TableFileSchema::FILE_TYPE::BACKUP) {
		engine_type = server::ValidationUtil::IsBinaryMetricType(file.metric_type_) ? EngineType::FAISS_BIN_IDMAP
		: EngineType::FAISS_IDMAP;
		engine_type =
		utils::IsBinaryMetricType(file.metric_type_) ? EngineType::FAISS_BIN_IDMAP : EngineType::FAISS_IDMAP;
		} else {
		engine_type = (EngineType)file.engine_type_;
		}
		@@ -731,7 +731,7 @@ DBImpl::Compact(const std::string& table_id) {
		break;
		}
		} else {
		ENGINE_LOG_ERROR << "Segment " << file.segment_id_ << " has no deleted data. No need to compact";
		ENGINE_LOG_DEBUG << "Segment " << file.segment_id_ << " has no deleted data. No need to compact";
		}
		}

		@@ -739,7 +739,7 @@ DBImpl::Compact(const std::string& table_id) {
		ENGINE_LOG_DEBUG << "Finished compacting table: " << table_id;
		}

		ENGINE_LOG_ERROR << "Updating meta after compaction...";
		ENGINE_LOG_DEBUG << "Updating meta after compaction...";

		/*
		// Drop index again, in case some files were in the index building process during compacting
		@@ -1019,7 +1019,7 @@ DBImpl::GetVectorByIdHelper(const std::string& table_id, IDNumber vector_id, Vec
		auto deleted = std::find(deleted_docs.begin(), deleted_docs.end(), offset);
		if (deleted == deleted_docs.end()) {
		// Load raw vector
		bool is_binary = server::ValidationUtil::IsBinaryMetricType(file.metric_type_);
		bool is_binary = utils::IsBinaryMetricType(file.metric_type_);
		size_t single_vector_bytes = is_binary ? file.dimension_ / 8 : file.dimension_ * sizeof(float);
		std::vector<uint8_t> raw_vector;
		status = segment_reader.LoadVectors(offset * single_vector_bytes, single_vector_bytes, raw_vector);

core/src/db/Utils.cpp

+13 −0

Original line number	Diff line number	Diff line
		@@ -220,6 +220,19 @@ IsRawIndexType(int32_t type) {
		return (type == (int32_t)EngineType::FAISS_IDMAP) \|\| (type == (int32_t)EngineType::FAISS_BIN_IDMAP);
		}

		bool
		IsBinaryIndexType(int32_t index_type) {
		return (index_type == (int32_t)engine::EngineType::FAISS_BIN_IDMAP) \|\|
		(index_type == (int32_t)engine::EngineType::FAISS_BIN_IVFFLAT);
		}

		bool
		IsBinaryMetricType(int32_t metric_type) {
		return (metric_type == (int32_t)engine::MetricType::HAMMING) \|\|
		(metric_type == (int32_t)engine::MetricType::JACCARD) \|\|
		(metric_type == (int32_t)engine::MetricType::TANIMOTO);
		}

		meta::DateT
		GetDate(const std::time_t& t, int day_delta) {
		struct tm ltm;

core/src/db/Utils.h

+6 −0

Original line number	Diff line number	Diff line
		@@ -48,6 +48,12 @@ IsSameIndex(const TableIndex& index1, const TableIndex& index2);
		bool
		IsRawIndexType(int32_t type);

		static bool
		IsBinaryIndexType(int32_t index_type);

		bool
		IsBinaryMetricType(int32_t metric_type);

		meta::DateT
		GetDate(const std::time_t& t, int day_delta = 0);
		meta::DateT

core/src/db/engine/ExecutionEngineImpl.cpp

+2 −3

Original line number	Diff line number	Diff line
		@@ -100,9 +100,8 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, const std::string&
		index_type_(index_type),
		metric_type_(metric_type),
		index_params_(index_params) {
		EngineType tmp_index_type = server::ValidationUtil::IsBinaryMetricType((int32_t)metric_type)
		? EngineType::FAISS_BIN_IDMAP
		: EngineType::FAISS_IDMAP;
		EngineType tmp_index_type =
		utils::IsBinaryMetricType((int32_t)metric_type) ? EngineType::FAISS_BIN_IDMAP : EngineType::FAISS_IDMAP;
		index_ = CreatetVecIndex(tmp_index_type);
		if (!index_) {
		throw Exception(DB_ERROR, "Unsupported index type");