Unverified Commit 6f58f755 authored by Jin Hai's avatar Jin Hai Committed by GitHub
Browse files

Merge pull request #608 from fishpenguin/0.6.0-yk-refactor-scheduler 

NSG build failed using GPU-edition if set gpu_enable false
parents 299e245d 10d50d2f

CHANGELOG.md

+1 −0
Original line number Diff line number Diff line
@@ -34,6 +34,7 @@ Please mark all change in change log and use the ticket from JIRA. 
- \#533 - NSG build failed with MetricType Inner Product

- \#543 - client raise exception in shards when search results is empty

- \#545 - Avoid dead circle of build index thread when error occurs

- \#547 - NSG build failed using GPU-edition if set gpu_enable false

- \#552 - Server down during building index_type: IVF_PQ using GPU-edition

- \#561 - Milvus server should report exception/error message or terminate on mysql metadata backend error

- \#599 - Build index log is incorrect

core/src/db/engine/ExecutionEngineImpl.cpp

+3 −0
Original line number Diff line number Diff line
@@ -611,6 +611,9 @@ ExecutionEngineImpl::Init() { 
    server::Config& config = server::Config::GetInstance();

    std::vector<int64_t> gpu_ids;

    Status s = config.GetGpuResourceConfigBuildIndexResources(gpu_ids);

    if (!s.ok()) {

        gpu_num_ = knowhere::INVALID_VALUE;

    }

    for (auto id : gpu_ids) {

        if (gpu_num_ == id) {

            return Status::OK();

core/src/grpc/README.md

deleted100644 → 0
+0 −6
Original line number Diff line number Diff line 
We manually change two APIs in "milvus.pb.h":

    add_vector_data()

    add_row_id_array()

    add_ids()

    add_distances()

If proto files need be generated again, remember to re-change above APIs.
 
 No newline at end of file

core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp

+15 −4
Original line number Diff line number Diff line
@@ -116,17 +116,28 @@ NSG::Train(const DatasetPtr& dataset, const Config& config) { 
    }



    // TODO(linxj): dev IndexFactory, support more IndexType

    Graph knng;

#ifdef MILVUS_GPU_VERSION

    if (build_cfg->gpu_id == knowhere::INVALID_VALUE) {

        auto preprocess_index = std::make_shared<IVF>();

        auto model = preprocess_index->Train(dataset, config);

        preprocess_index->set_index_model(model);

        preprocess_index->AddWithoutIds(dataset, config);

        preprocess_index->GenGraph(build_cfg->knng, knng, dataset, config);

    } else {

        auto preprocess_index = std::make_shared<GPUIVF>(build_cfg->gpu_id);

        auto model = preprocess_index->Train(dataset, config);

        preprocess_index->set_index_model(model);

        preprocess_index->AddWithoutIds(dataset, config);

        preprocess_index->GenGraph(build_cfg->knng, knng, dataset, config);

    }

#else

    auto preprocess_index = std::make_shared<IVF>();

#endif

    auto model = preprocess_index->Train(dataset, config);

    preprocess_index->set_index_model(model);

    preprocess_index->AddWithoutIds(dataset, config);



    Graph knng;

    preprocess_index->GenGraph(build_cfg->knng, knng, dataset, config);

#endif



    algo::BuildParams b_params;

    b_params.candidate_pool_size = build_cfg->candidate_pool_size;

core/src/scheduler/SchedInst.h

+4 −0
Original line number Diff line number Diff line
@@ -25,6 +25,7 @@ 
#include "optimizer/BuildIndexPass.h"

#include "optimizer/FaissFlatPass.h"

#include "optimizer/FaissIVFFlatPass.h"

#include "optimizer/FaissIVFPQPass.h"

#include "optimizer/FaissIVFSQ8HPass.h"

#include "optimizer/FaissIVFSQ8Pass.h"

#include "optimizer/FallbackPass.h"
@@ -129,7 +130,10 @@ class OptimizerInst { 
                    pass_list.push_back(std::make_shared<FaissFlatPass>());

                    pass_list.push_back(std::make_shared<FaissIVFFlatPass>());

                    pass_list.push_back(std::make_shared<FaissIVFSQ8Pass>());

#ifdef CUSTOMIZATION

                    pass_list.push_back(std::make_shared<FaissIVFSQ8HPass>());

#endif

                    pass_list.push_back(std::make_shared<FaissIVFPQPass>());

                }

#endif

                pass_list.push_back(std::make_shared<FallbackPass>());