Unverified Commit 354f29a0 authored by BossZou's avatar BossZou Committed by GitHub
Browse files

Refactor log on search and insert request (#1887) 



* fix stash conflict

Signed-off-by: default avatarYhz <yinghao.zou@zilliz.com>

* Log config information and device information (fix #1893)

Signed-off-by: default avatarYhz <yinghao.zou@zilliz.com>

* Add index annoy into http module (fix #1882)

Signed-off-by: default avatarYhz <yinghao.zou@zilliz.com>

* Refactor log on search and insert request (fix #1886)

Signed-off-by: default avatarYhz <yinghao.zou@zilliz.com>

* use vsnprintf to avoid oom

Signed-off-by: default avatarYhz <yinghao.zou@zilliz.com>

* fix PR Quality problem & remove surplus comments

Signed-off-by: default avatarYhz <yinghao.zou@zilliz.com>

* code format

Signed-off-by: default avatarYhz <yinghao.zou@zilliz.com>

* replace strlen with strnlen & remove some surplus comments

Signed-off-by: default avatarYhz <yinghao.zou@zilliz.com>

* format

Signed-off-by: default avatarYhz <yinghao.zou@zilliz.com>
parent 66e38d20

CHANGELOG.md

+3 −0
Original line number Diff line number Diff line
@@ -17,11 +17,14 @@ Please mark all change in change log and use the issue from GitHub 
-   \#1661 HNSW support deleted vectors searching

-   \#1825 Add annoy index type in C++ sdk

-   \#1849 NSG support deleted vectors searching

-   \#1893 Log config information and device information



## Improvement

-   \#1627 Move read/write index APIs into codec

-   \#1784 Add Substructure and Superstructure in http module

-   \#1858 Disable S3 build

-   \#1882 Add index annoy into http module

-   \#1886 Refactor log on search and insert request



## Task

core/src/config/Config.cpp

+2 −2
Original line number Diff line number Diff line
@@ -314,9 +314,9 @@ Config::ResetDefaultConfig() { 
}



void

Config::GetConfigJsonStr(std::string& result) {

Config::GetConfigJsonStr(std::string& result, int64_t indent) {

    nlohmann::json config_json(config_map_);

    result = config_json.dump();

    result = config_json.dump(indent);

}



Status

core/src/config/Config.h

+1 −1
Original line number Diff line number Diff line
@@ -161,7 +161,7 @@ class Config { 
    Status

    ResetDefaultConfig();

    void

    GetConfigJsonStr(std::string& result);

    GetConfigJsonStr(std::string& result, int64_t indent = -1);

    Status

    ProcessConfigCli(std::string& result, const std::string& cmd);

core/src/db/DBImpl.cpp

+5 −1
Original line number Diff line number Diff line
@@ -508,6 +508,7 @@ DBImpl::InsertVectors(const std::string& collection_id, const std::string& parti 
        SafeIDGenerator& id_generator = SafeIDGenerator::GetInstance();

        Status status = id_generator.GetNextIDNumbers(vectors.vector_count_, vectors.id_array_);

        if (!status.ok()) {

            ENGINE_LOG_ERROR << LogOut("[%s][%ld] Get next id number fail: %s", "insert", 0, status.message().c_str());

            return status;

        }

    }
@@ -517,6 +518,7 @@ DBImpl::InsertVectors(const std::string& collection_id, const std::string& parti 
        std::string target_collection_name;

        status = GetPartitionByTag(collection_id, partition_tag, target_collection_name);

        if (!status.ok()) {

            ENGINE_LOG_ERROR << LogOut("[%s][%ld] Get partition fail: %s", "insert", 0, status.message().c_str());

            return status;

        }
@@ -1215,7 +1217,7 @@ DBImpl::QueryAsync(const std::shared_ptr<server::Context>& context, const meta:: 
    // step 1: construct search job

    auto status = OngoingFileChecker::GetInstance().MarkOngoingFiles(files);



    ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size();

    ENGINE_LOG_DEBUG << LogOut("Engine query begin, index file count: %ld", files.size());

    scheduler::SearchJobPtr job = std::make_shared<scheduler::SearchJob>(tracer.Context(), k, extra_params, vectors);

    for (auto& file : files) {

        scheduler::SegmentSchemaPtr file_ptr = std::make_shared<meta::SegmentSchema>(file);
@@ -1856,6 +1858,7 @@ DBImpl::ExecWalRecord(const wal::MXLogRecord& record) { 
            std::string target_collection_name;

            status = GetPartitionByTag(record.collection_id, record.partition_tag, target_collection_name);

            if (!status.ok()) {

                WAL_LOG_ERROR << LogOut("[%s][%ld] ", "insert", 0) << "Get partition fail: " << status.message();

                return status;

            }
@@ -1875,6 +1878,7 @@ DBImpl::ExecWalRecord(const wal::MXLogRecord& record) { 
            std::string target_collection_name;

            status = GetPartitionByTag(record.collection_id, record.partition_tag, target_collection_name);

            if (!status.ok()) {

                WAL_LOG_ERROR << LogOut("[%s][%ld] ", "insert", 0) << "Get partition fail: " << status.message();

                return status;

            }

core/src/db/engine/ExecutionEngineImpl.cpp

+15 −9
Original line number Diff line number Diff line
@@ -763,10 +763,10 @@ ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, const milvu 
        }

    }

#endif

    TimeRecorder rc("ExecutionEngineImpl::Search float");

    TimeRecorder rc(LogOut("[%s][%ld] ExecutionEngineImpl::Search float", "search", 0));



    if (index_ == nullptr) {

        ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search";

        ENGINE_LOG_ERROR << LogOut("[%s][%ld] ExecutionEngineImpl: index is null, failed to search", "search", 0);

        return Status(DB_ERROR, "index is null");

    }
@@ -774,6 +774,7 @@ ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, const milvu 
    conf[knowhere::meta::TOPK] = k;

    auto adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(index_->index_type());

    if (!adapter->CheckSearch(conf, index_->index_type(), index_->index_mode())) {

        ENGINE_LOG_ERROR << LogOut("[%s][%ld] Illegal search params", "search", 0);

        throw Exception(DB_ERROR, "Illegal search params");

    }
@@ -786,7 +787,8 @@ ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, const milvu 
    auto result = index_->Query(dataset, conf);

    rc.RecordSection("query done");



    ENGINE_LOG_DEBUG << "get uids " << index_->GetUids().size() << " from index " << location_;

    ENGINE_LOG_DEBUG << LogOut("[%s][%ld] get %ld uids from index %s", "search", 0, index_->GetUids().size(),

                               location_.c_str());

    MapAndCopyResult(result, index_->GetUids(), n, k, distances, labels);

    rc.RecordSection("map uids " + std::to_string(n * k));
@@ -800,10 +802,10 @@ ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, const milvu 
Status

ExecutionEngineImpl::Search(int64_t n, const uint8_t* data, int64_t k, const milvus::json& extra_params,

                            float* distances, int64_t* labels, bool hybrid) {

    TimeRecorder rc("ExecutionEngineImpl::Search uint8");

    TimeRecorder rc(LogOut("[%s][%ld] ExecutionEngineImpl::Search uint8", "search", 0));



    if (index_ == nullptr) {

        ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search";

        ENGINE_LOG_ERROR << LogOut("[%s][%ld] ExecutionEngineImpl: index is null, failed to search", "search", 0);

        return Status(DB_ERROR, "index is null");

    }
@@ -811,6 +813,7 @@ ExecutionEngineImpl::Search(int64_t n, const uint8_t* data, int64_t k, const mil 
    conf[knowhere::meta::TOPK] = k;

    auto adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(index_->index_type());

    if (!adapter->CheckSearch(conf, index_->index_type(), index_->index_mode())) {

        ENGINE_LOG_ERROR << LogOut("[%s][%ld] Illegal search params", "search", 0);

        throw Exception(DB_ERROR, "Illegal search params");

    }
@@ -823,7 +826,8 @@ ExecutionEngineImpl::Search(int64_t n, const uint8_t* data, int64_t k, const mil 
    auto result = index_->Query(dataset, conf);

    rc.RecordSection("query done");



    ENGINE_LOG_DEBUG << "get uids " << index_->GetUids().size() << " from index " << location_;

    ENGINE_LOG_DEBUG << LogOut("[%s][%ld] get %ld uids from index %s", "search", 0, index_->GetUids().size(),

                               location_.c_str());

    MapAndCopyResult(result, index_->GetUids(), n, k, distances, labels);

    rc.RecordSection("map uids " + std::to_string(n * k));
@@ -837,10 +841,10 @@ ExecutionEngineImpl::Search(int64_t n, const uint8_t* data, int64_t k, const mil 
Status

ExecutionEngineImpl::Search(int64_t n, const std::vector<int64_t>& ids, int64_t k, const milvus::json& extra_params,

                            float* distances, int64_t* labels, bool hybrid) {

    TimeRecorder rc("ExecutionEngineImpl::Search vector of ids");

    TimeRecorder rc(LogOut("[%s][%ld] ExecutionEngineImpl::Search vector of ids", "search", 0));



    if (index_ == nullptr) {

        ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search";

        ENGINE_LOG_ERROR << LogOut("[%s][%ld] ExecutionEngineImpl: index is null, failed to search", "search", 0);

        return Status(DB_ERROR, "index is null");

    }
@@ -848,6 +852,7 @@ ExecutionEngineImpl::Search(int64_t n, const std::vector<int64_t>& ids, int64_t 
    conf[knowhere::meta::TOPK] = k;

    auto adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(index_->index_type());

    if (!adapter->CheckSearch(conf, index_->index_type(), index_->index_mode())) {

        ENGINE_LOG_ERROR << LogOut("[%s][%ld] Illegal search params", "search", 0);

        throw Exception(DB_ERROR, "Illegal search params");

    }
@@ -901,7 +906,8 @@ ExecutionEngineImpl::Search(int64_t n, const std::vector<int64_t>& ids, int64_t 
        auto result = index_->QueryById(dataset, conf);

        rc.RecordSection("query by id done");



        ENGINE_LOG_DEBUG << "get uids " << index_->GetUids().size() << " from index " << location_;

        ENGINE_LOG_DEBUG << LogOut("[%s][%ld] get %ld uids from index %s", "search", 0, index_->GetUids().size(),

                                   location_.c_str());

        MapAndCopyResult(result, uids, offsets.size(), k, distances, labels);

        rc.RecordSection("map uids " + std::to_string(offsets.size() * k));

    }