Unverified Commit 0fd9bf8e authored by 蔡宇东's avatar 蔡宇东 Committed by GitHub
Browse files

#1529 fix server crash when cache_insert_data enabled (#1541) 



* #1529 fix server crash when cache_insert_data enabled

Signed-off-by: default avataryudong.cai <yudong.cai@zilliz.com>

* #1529 fix bug

Signed-off-by: default avataryudong.cai <yudong.cai@zilliz.com>

* #1529 code opt

Signed-off-by: default avataryudong.cai <yudong.cai@zilliz.com>

* #1529 set uids for CopyGpuToCpu()

Signed-off-by: default avataryudong.cai <yudong.cai@zilliz.com>
parent a08b51c2

CHANGELOG.md

+1 −0
Original line number Diff line number Diff line
@@ -39,6 +39,7 @@ Please mark all change in change log and use the issue from GitHub 
-   \#1518 Table count did not match after deleting vectors and compact

-   \#1521 Make cache_insert_data take effect in-service

-   \#1525 Add setter API for config preload_table

-   \#1529 Fix server crash when cache_insert_data enabled

-   \#1530 Set table file with correct engine type in meta

-   \#1535 Degradation searching performance with metric_type: binary_idmap

core/src/db/engine/ExecutionEngineImpl.cpp

+12 −0
Original line number Diff line number Diff line
@@ -431,6 +431,7 @@ ExecutionEngineImpl::Load(bool to_cache) { 


            auto vectors_uids = vectors->GetUids();

            index_->SetUids(vectors_uids);

            ENGINE_LOG_DEBUG << "set uids " << index_->GetUids().size() << " for index " << location_;



            auto vectors_data = vectors->GetData();
@@ -510,6 +511,7 @@ ExecutionEngineImpl::Load(bool to_cache) { 
                    std::vector<segment::doc_id_t> uids;

                    segment_reader_ptr->LoadUids(uids);

                    index_->SetUids(uids);

                    ENGINE_LOG_DEBUG << "set uids " << index_->GetUids().size() << " for index " << location_;



                    ENGINE_LOG_DEBUG << "Finished loading index file from segment " << segment_dir;

                }
@@ -734,12 +736,19 @@ ExecutionEngineImpl::BuildIndex(const std::string& location, EngineType engine_t 
        throw Exception(DB_ERROR, "Illegal index params");

    }

    ENGINE_LOG_DEBUG << "Index config: " << conf.dump();



    auto status = Status::OK();

    std::vector<segment::doc_id_t> uids;

    if (from_index) {

        status = to_index->BuildAll(Count(), from_index->GetRawVectors(), from_index->GetRawIds(), conf);

        uids = from_index->GetUids();

    } else if (bin_from_index) {

        status = to_index->BuildAll(Count(), bin_from_index->GetRawVectors(), bin_from_index->GetRawIds(), conf);

        uids = bin_from_index->GetUids();

    }

    to_index->SetUids(uids);

    ENGINE_LOG_DEBUG << "set uids " << to_index->GetUids().size() << " for " << location;



    if (!status.ok()) {

        throw Exception(DB_ERROR, status.message());

    }
@@ -838,6 +847,7 @@ ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, const milvu 
    rc.RecordSection("search done");



    // map offsets to ids

    ENGINE_LOG_DEBUG << "get uids " << index_->GetUids().size() << " from index " << location_;

    MapUids(index_->GetUids(), labels, n * k);



    rc.RecordSection("map uids " + std::to_string(n * k));
@@ -879,6 +889,7 @@ ExecutionEngineImpl::Search(int64_t n, const uint8_t* data, int64_t k, const mil 
    rc.RecordSection("search done");



    // map offsets to ids

    ENGINE_LOG_DEBUG << "get uids " << index_->GetUids().size() << " from index " << location_;

    MapUids(index_->GetUids(), labels, n * k);



    rc.RecordSection("map uids " + std::to_string(n * k));
@@ -962,6 +973,7 @@ ExecutionEngineImpl::Search(int64_t n, const std::vector<int64_t>& ids, int64_t 
        rc.RecordSection("search done");



        // map offsets to ids

        ENGINE_LOG_DEBUG << "get uids " << index_->GetUids().size() << " from index " << location_;

        MapUids(uids, labels, offsets.size() * k);



        rc.RecordSection("map uids " + std::to_string(offsets.size() * k));

core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp

+4 −1
Original line number Diff line number Diff line
@@ -24,7 +24,10 @@ namespace cloner { 
VectorIndexPtr

CopyGpuToCpu(const VectorIndexPtr& index, const Config& config) {

    if (auto device_index = std::dynamic_pointer_cast<GPUIndex>(index)) {

        return device_index->CopyGpuToCpu(config);

        VectorIndexPtr result = device_index->CopyGpuToCpu(config);

        auto uids = index->GetUids();

        result->SetUids(uids);

        return result;

    } else {

        KNOWHERE_THROW_MSG("index type is not gpuindex");

    }