Commit 4068c2ac authored by groot's avatar groot
Browse files

merge master branch 



Signed-off-by: default avatargroot <yihua.mo@zilliz.com>
parents 0ec50358 7cd727aa

CHANGELOG.md

+2 −0
Original line number Diff line number Diff line
@@ -43,9 +43,11 @@ Please mark all change in change log and use the issue from GitHub 
-   \#1530 Set table file with correct engine type in meta

-   \#1532 Search with ivf_flat failed with open-dataset: sift-256-hamming

-   \#1535 Degradation searching performance with metric_type: binary_idmap

-   \#1549 Fix server/wal config setting bug

-   \#1556 Index file not created after table and index created

-   \#1560 Search crashed with Super-high dimensional binary vector

-   \#1571 Meta engine type become IDMAP after dropping index for BINARY table

-   \#1574 Set all existing bitset in cache when applying deletes



## Feature

-   \#216 Add CLI to get server info

core/src/db/insert/MemTable.cpp

+24 −14
Original line number Diff line number Diff line
@@ -236,11 +236,27 @@ MemTable::ApplyDeletes() { 
        utils::GetParentPath(table_file.location_, segment_dir);

        segment::SegmentReader segment_reader(segment_dir);



        auto& segment_id = table_file.segment_id_;

        meta::TableFilesSchema segment_files;

        status = meta_->GetTableFilesBySegmentId(segment_id, segment_files);

        if (!status.ok()) {

            break;

        }



        // Get all index that contains blacklist in cache

        std::vector<VecIndexPtr> indexes;

        std::vector<faiss::ConcurrentBitsetPtr> blacklists;

        for (auto& file : segment_files) {

            auto index =

            std::static_pointer_cast<VecIndex>(cache::CpuCacheMgr::GetInstance()->GetIndex(table_file.location_));

                std::static_pointer_cast<VecIndex>(cache::CpuCacheMgr::GetInstance()->GetIndex(file.location_));

            faiss::ConcurrentBitsetPtr blacklist = nullptr;

            if (index != nullptr) {

            status = index->GetBlacklist(blacklist);

                index->GetBlacklist(blacklist);

                if (blacklist != nullptr) {

                    indexes.emplace_back(index);

                    blacklists.emplace_back(blacklist);

                }

            }

        }



        std::vector<segment::doc_id_t> uids;
@@ -293,7 +309,7 @@ MemTable::ApplyDeletes() { 
                    id_bloom_filter_ptr->Remove(uids[i]);

                }



                if (blacklist != nullptr) {

                for (auto& blacklist : blacklists) {

                    if (!blacklist->test(i)) {

                        blacklist->set(i);

                    }
@@ -308,8 +324,8 @@ MemTable::ApplyDeletes() { 
                         << find_diff.count() << " s in total";

        ENGINE_LOG_DEBUG << "Setting deleted docs and bloom filter took " << set_diff.count() << " s in total";



        if (index != nullptr) {

            index->SetBlacklist(blacklist);

        for (auto i = 0; i < indexes.size(); ++i) {

            indexes[i]->SetBlacklist(blacklists[i]);

        }



        start = std::chrono::high_resolution_clock::now();
@@ -339,12 +355,6 @@ MemTable::ApplyDeletes() { 
                         << " s";



        // Update table file row count

        auto& segment_id = table_file.segment_id_;

        meta::TableFilesSchema segment_files;

        status = meta_->GetTableFilesBySegmentId(segment_id, segment_files);

        if (!status.ok()) {

            break;

        }

        for (auto& file : segment_files) {

            if (file.file_type_ == meta::TableFileSchema::RAW || file.file_type_ == meta::TableFileSchema::TO_INDEX ||

                file.file_type_ == meta::TableFileSchema::INDEX || file.file_type_ == meta::TableFileSchema::BACKUP) {

core/src/server/Config.cpp

+6 −11
Original line number Diff line number Diff line
@@ -1529,8 +1529,7 @@ Status 
Config::GetStorageConfigS3Enable(bool& value) {

    std::string str = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_S3_ENABLE, CONFIG_STORAGE_S3_ENABLE_DEFAULT);

    CONFIG_CHECK(CheckStorageConfigS3Enable(str));

    std::transform(str.begin(), str.end(), str.begin(), ::tolower);

    value = (str == "true" || str == "on" || str == "yes" || str == "1");

    CONFIG_CHECK(StringHelpFunctions::ConvertToBoolean(str, value));

    return Status::OK();

}
@@ -1569,8 +1568,7 @@ Status 
Config::GetMetricConfigEnableMonitor(bool& value) {

    std::string str = GetConfigStr(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, CONFIG_METRIC_ENABLE_MONITOR_DEFAULT);

    CONFIG_CHECK(CheckMetricConfigEnableMonitor(str));

    std::transform(str.begin(), str.end(), str.begin(), ::tolower);

    value = (str == "true" || str == "on" || str == "yes" || str == "1");

    CONFIG_CHECK(StringHelpFunctions::ConvertToBoolean(str, value));

    return Status::OK();

}
@@ -1671,8 +1669,7 @@ Status 
Config::GetGpuResourceConfigEnable(bool& value) {

    std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE, CONFIG_GPU_RESOURCE_ENABLE_DEFAULT);

    CONFIG_CHECK(CheckGpuResourceConfigEnable(str));

    std::transform(str.begin(), str.end(), str.begin(), ::tolower);

    value = (str == "true" || str == "on" || str == "yes" || str == "1");

    CONFIG_CHECK(StringHelpFunctions::ConvertToBoolean(str, value));

    return Status::OK();

}
@@ -1774,8 +1771,7 @@ Status 
Config::GetWalConfigEnable(bool& wal_enable) {

    std::string str = GetConfigStr(CONFIG_WAL, CONFIG_WAL_ENABLE, CONFIG_WAL_ENABLE_DEFAULT);

    CONFIG_CHECK(CheckWalConfigEnable(str));

    std::transform(str.begin(), str.end(), str.begin(), ::tolower);

    wal_enable = (str == "true" || str == "on" || str == "yes" || str == "1");

    CONFIG_CHECK(StringHelpFunctions::ConvertToBoolean(str, wal_enable));

    return Status::OK();

}
@@ -1784,8 +1780,7 @@ Config::GetWalConfigRecoveryErrorIgnore(bool& recovery_error_ignore) { 
    std::string str =

        GetConfigStr(CONFIG_WAL, CONFIG_WAL_RECOVERY_ERROR_IGNORE, CONFIG_WAL_RECOVERY_ERROR_IGNORE_DEFAULT);

    CONFIG_CHECK(CheckWalConfigRecoveryErrorIgnore(str));

    std::transform(str.begin(), str.end(), str.begin(), ::tolower);

    recovery_error_ignore = (str == "true" || str == "on" || str == "yes" || str == "1");

    CONFIG_CHECK(StringHelpFunctions::ConvertToBoolean(str, recovery_error_ignore));

    return Status::OK();

}
@@ -2013,7 +2008,7 @@ Config::SetWalConfigEnable(const std::string& value) { 
Status

Config::SetWalConfigRecoveryErrorIgnore(const std::string& value) {

    CONFIG_CHECK(CheckWalConfigRecoveryErrorIgnore(value));

    return SetConfigValueInMem(CONFIG_WAL, CONFIG_WAL_RECOVERY_ERROR_IGNORE_DEFAULT, value);

    return SetConfigValueInMem(CONFIG_WAL, CONFIG_WAL_RECOVERY_ERROR_IGNORE, value);

}



Status

core/src/server/web_impl/controller/WebController.hpp

+4 −4
Original line number Diff line number Diff line
@@ -336,7 +336,7 @@ class WebController : public oatpp::web::server::api::ApiController { 


    ADD_CORS(CreateIndex)



    ENDPOINT("POST", "/tables/{collection_name}/indexes", CreateIndex,

    ENDPOINT("POST", "/collections/{collection_name}/indexes", CreateIndex,

             PATH(String, collection_name), BODY_STRING(String, body)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/tables/" + collection_name->std_str() + "/indexes\'");

        tr.RecordSection("Received request.");
@@ -674,15 +674,15 @@ class WebController : public oatpp::web::server::api::ApiController { 


    ADD_CORS(SystemOp)



    ENDPOINT("PUT", "/system/{Op}", SystemOp, PATH(String, Op), BODY_STRING(String, body_str)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "PUT \'/system/" + Op->std_str() + "\'");

    ENDPOINT("PUT", "/system/{op}", SystemOp, PATH(String, op), BODY_STRING(String, body_str)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "PUT \'/system/" + op->std_str() + "\'");

        tr.RecordSection("Received request.");



        WebRequestHandler handler = WebRequestHandler();

        handler.RegisterRequestHandler(::milvus::server::RequestHandler());



        String response_str;

        auto status_dto = handler.SystemOp(Op, body_str, response_str);

        auto status_dto = handler.SystemOp(op, body_str, response_str);



        std::shared_ptr<OutgoingResponse> response;

        switch (status_dto->code->getValue()) {

core/src/server/web_impl/handler/WebRequestHandler.cpp

+2 −0
Original line number Diff line number Diff line
@@ -1046,7 +1046,9 @@ WebRequestHandler::CreateIndex(const OString& table_name, const OString& body) { 
        auto status = request_handler_.CreateIndex(context_ptr_, table_name->std_str(), index, request_json["params"]);

        ASSIGN_RETURN_STATUS_DTO(status);

    } catch (nlohmann::detail::parse_error& e) {

        RETURN_STATUS_DTO(BODY_PARSE_FAIL, e.what())

    } catch (nlohmann::detail::type_error& e) {

        RETURN_STATUS_DTO(BODY_PARSE_FAIL, e.what())

    }



    ASSIGN_RETURN_STATUS_DTO(Status::OK())