Unverified Commit ff0aad81 authored by Jin Hai's avatar Jin Hai Committed by GitHub
Browse files

Merge branch 'master' into fix_1517

parents 66a14e16 b20e16ab

CHANGELOG.md

+7 −2
Original line number Diff line number Diff line
@@ -12,16 +12,20 @@ Please mark all change in change log and use the issue from GitHub 
-   \#831 Judge branch error in CommonUtil.cpp

-   \#977 Server crash when create tables concurrently

-   \#990 Check gpu resources setting when assign repeated value

-   \#995 table count set to 0 if no tables found

-   \#995 Table count set to 0 if no tables found

-   \#1010 Improve error message when offset or page_size is equal 0

-   \#1022 Check if partition name is valid

-   \#1028 check if table exists when show partitions

-   \#1029 check if table exists when try to delete partition

-   \#1066 optimize http insert and search speed

-   \#1022 Check if partition name is legal

-   \#1028 Check if table exists when show partitions

-   \#1029 Check if table exists when try to delete partition

-   \#1066 Optimize http insert and search speed

-   \#1067 Add binary vectors support in http server

-   \#1075 Improve error message when page size or offset is illegal

-   \#1082 Check page_size or offset value to avoid float

-   \#1115 http server support load table into memory

-   \#1115 Http server support load table into memory

-   \#1152 Error log output continuously after server start

-   \#1211 Server down caused by searching with index_type: HNSW

-   \#1240 Update license declaration
@@ -48,6 +52,7 @@ Please mark all change in change log and use the issue from GitHub 
-   \#1556 Index file not created after table and index created

-   \#1560 Search crashed with Super-high dimensional binary vector

-   \#1564 Too low recall for glove-200-angular, ivf_pq index

-   \#1571 Meta engine type become IDMAP after dropping index for BINARY table

-   \#1574 Set all existing bitset in cache when applying deletes

-   \#1577 Row count incorrect if delete vectors then create index

core/src/db/DBImpl.cpp

+5 −5
Original line number Diff line number Diff line
@@ -366,8 +366,8 @@ DBImpl::PreloadTable(const std::string& table_id) { 
        if (file.file_type_ == meta::TableFileSchema::FILE_TYPE::RAW ||

            file.file_type_ == meta::TableFileSchema::FILE_TYPE::TO_INDEX ||

            file.file_type_ == meta::TableFileSchema::FILE_TYPE::BACKUP) {

            engine_type = server::ValidationUtil::IsBinaryMetricType(file.metric_type_) ? EngineType::FAISS_BIN_IDMAP

                                                                                        : EngineType::FAISS_IDMAP;

            engine_type =

                utils::IsBinaryMetricType(file.metric_type_) ? EngineType::FAISS_BIN_IDMAP : EngineType::FAISS_IDMAP;

        } else {

            engine_type = (EngineType)file.engine_type_;

        }
@@ -731,7 +731,7 @@ DBImpl::Compact(const std::string& table_id) { 
                break;

            }

        } else {

            ENGINE_LOG_ERROR << "Segment " << file.segment_id_ << " has no deleted data. No need to compact";

            ENGINE_LOG_DEBUG << "Segment " << file.segment_id_ << " has no deleted data. No need to compact";

        }

    }
@@ -739,7 +739,7 @@ DBImpl::Compact(const std::string& table_id) { 
        ENGINE_LOG_DEBUG << "Finished compacting table: " << table_id;

    }



    ENGINE_LOG_ERROR << "Updating meta after compaction...";

    ENGINE_LOG_DEBUG << "Updating meta after compaction...";



    /*

    // Drop index again, in case some files were in the index building process during compacting
@@ -1019,7 +1019,7 @@ DBImpl::GetVectorByIdHelper(const std::string& table_id, IDNumber vector_id, Vec 
                auto deleted = std::find(deleted_docs.begin(), deleted_docs.end(), offset);

                if (deleted == deleted_docs.end()) {

                    // Load raw vector

                    bool is_binary = server::ValidationUtil::IsBinaryMetricType(file.metric_type_);

                    bool is_binary = utils::IsBinaryMetricType(file.metric_type_);

                    size_t single_vector_bytes = is_binary ? file.dimension_ / 8 : file.dimension_ * sizeof(float);

                    std::vector<uint8_t> raw_vector;

                    status = segment_reader.LoadVectors(offset * single_vector_bytes, single_vector_bytes, raw_vector);

core/src/db/Utils.cpp

+13 −0
Original line number Diff line number Diff line
@@ -220,6 +220,19 @@ IsRawIndexType(int32_t type) { 
    return (type == (int32_t)EngineType::FAISS_IDMAP) || (type == (int32_t)EngineType::FAISS_BIN_IDMAP);

}



bool

IsBinaryIndexType(int32_t index_type) {

    return (index_type == (int32_t)engine::EngineType::FAISS_BIN_IDMAP) ||

           (index_type == (int32_t)engine::EngineType::FAISS_BIN_IVFFLAT);

}



bool

IsBinaryMetricType(int32_t metric_type) {

    return (metric_type == (int32_t)engine::MetricType::HAMMING) ||

           (metric_type == (int32_t)engine::MetricType::JACCARD) ||

           (metric_type == (int32_t)engine::MetricType::TANIMOTO);

}



meta::DateT

GetDate(const std::time_t& t, int day_delta) {

    struct tm ltm;

core/src/db/Utils.h

+6 −0
Original line number Diff line number Diff line
@@ -48,6 +48,12 @@ IsSameIndex(const TableIndex& index1, const TableIndex& index2); 
bool

IsRawIndexType(int32_t type);



static bool

IsBinaryIndexType(int32_t index_type);



bool

IsBinaryMetricType(int32_t metric_type);



meta::DateT

GetDate(const std::time_t& t, int day_delta = 0);

meta::DateT

core/src/db/engine/ExecutionEngineImpl.cpp

+2 −3
Original line number Diff line number Diff line
@@ -100,9 +100,8 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, const std::string& 
      index_type_(index_type),

      metric_type_(metric_type),

      index_params_(index_params) {

    EngineType tmp_index_type = server::ValidationUtil::IsBinaryMetricType((int32_t)metric_type)

                                    ? EngineType::FAISS_BIN_IDMAP

                                    : EngineType::FAISS_IDMAP;

    EngineType tmp_index_type =

        utils::IsBinaryMetricType((int32_t)metric_type) ? EngineType::FAISS_BIN_IDMAP : EngineType::FAISS_IDMAP;

    index_ = CreatetVecIndex(tmp_index_type);

    if (!index_) {

        throw Exception(DB_ERROR, "Unsupported index type");