Commit 968c4f64 authored by groot's avatar groot
Browse files

Merge remote-tracking branch 'source/0.5.1' into 0.5.1 


Former-commit-id: c5a0d3d7097cbd02f48da67598c67d69fe60b184
parents 601ddc08 d4fe7607

CHANGELOG.md

+3 −0
Original line number Diff line number Diff line
@@ -14,6 +14,9 @@ Please mark all change in change log and use the ticket from JIRA. 
- \#82 - Move easyloggingpp into "external" directory

- \#92 - Speed up CMake build process

- \#96 - Remove .a file in milvus/lib for docker-version

- \#118 - Using shared_ptr instead of weak_ptr to avoid performance loss

- \#122 - Add unique id for Job



## Feature

- \#115 - Using new structure for tasktable

core/src/db/DBImpl.cpp

+3 −3
Original line number Diff line number Diff line
@@ -136,7 +136,7 @@ DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& dates) { 


        // scheduler will determine when to delete table files

        auto nres = scheduler::ResMgrInst::GetInstance()->GetNumOfComputeResource();

        scheduler::DeleteJobPtr job = std::make_shared<scheduler::DeleteJob>(0, table_id, meta_ptr_, nres);

        scheduler::DeleteJobPtr job = std::make_shared<scheduler::DeleteJob>(table_id, meta_ptr_, nres);

        scheduler::JobMgrInst::GetInstance()->Put(job);

        job->WaitAndDelete();

    } else {
@@ -439,7 +439,7 @@ DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSchema& fi 


    // step 1: get files to search

    ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size();

    scheduler::SearchJobPtr job = std::make_shared<scheduler::SearchJob>(0, k, nq, nprobe, vectors);

    scheduler::SearchJobPtr job = std::make_shared<scheduler::SearchJob>(k, nq, nprobe, vectors);

    for (auto& file : files) {

        scheduler::TableFileSchemaPtr file_ptr = std::make_shared<meta::TableFileSchema>(file);

        job->AddIndexFile(file_ptr);
@@ -754,7 +754,7 @@ DBImpl::BackgroundBuildIndex() { 
    Status status;



    if (!to_index_files.empty()) {

        scheduler::BuildIndexJobPtr job = std::make_shared<scheduler::BuildIndexJob>(0, meta_ptr_, options_);

        scheduler::BuildIndexJobPtr job = std::make_shared<scheduler::BuildIndexJob>(meta_ptr_, options_);



        // step 2: put build index task to scheduler

        for (auto& file : to_index_files) {

core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp

+30 −20
Original line number Diff line number Diff line
@@ -426,26 +426,6 @@ test_ivfsq8h(const std::string& ann_test_name, int32_t index_add_loops, const st 
        cpu_ivf_index->to_readonly();

    }



    faiss::gpu::GpuClonerOptions option;

    option.allInGpu = true;



    faiss::IndexComposition index_composition;

    index_composition.index = cpu_index;

    index_composition.quantizer = nullptr;

    index_composition.mode = 1;



    double copy_time = elapsed();

    auto index = faiss::gpu::index_cpu_to_gpu(&res, 0, &index_composition, &option);

    delete index;



    if (pure_gpu_mode) {

        index_composition.mode = 2;  // 0: all data, 1: copy quantizer, 2: copy data

        index = faiss::gpu::index_cpu_to_gpu(&res, 0, &index_composition, &option);

    }



    copy_time = elapsed() - copy_time;

    printf("[%.3f s] Copy quantizer completed, cost %f s\n", elapsed() - t0, copy_time);



    size_t nq;

    float* xq;

    {
@@ -472,6 +452,36 @@ test_ivfsq8h(const std::string& ann_test_name, int32_t index_add_loops, const st 
        delete[] gt_int;

    }



    faiss::gpu::GpuClonerOptions option;

    option.allInGpu = true;



    faiss::IndexComposition index_composition;

    index_composition.index = cpu_index;

    index_composition.quantizer = nullptr;



    faiss::Index* index;

    double copy_time;



    if (!pure_gpu_mode) {

        index_composition.mode = 1;  // 0: all data, 1: copy quantizer, 2: copy data

        index = faiss::gpu::index_cpu_to_gpu(&res, 0, &index_composition, &option);

        delete index;



        copy_time = elapsed();

        index = faiss::gpu::index_cpu_to_gpu(&res, 0, &index_composition, &option);

        delete index;

    } else {

        index_composition.mode = 2;

        index = faiss::gpu::index_cpu_to_gpu(&res, 0, &index_composition, &option);

        delete index;



        copy_time = elapsed();

        index = faiss::gpu::index_cpu_to_gpu(&res, 0, &index_composition, &option);

    }



    copy_time = elapsed() - copy_time;

    printf("[%.3f s] Copy quantizer completed, cost %f s\n", elapsed() - t0, copy_time);



    const size_t NQ = 1000, K = 1000;

    if (!pure_gpu_mode) {

        for (auto nprobe : nprobes) {

core/src/scheduler/Algorithm.cpp

+1 −1
Original line number Diff line number Diff line
@@ -54,7 +54,7 @@ ShortestPath(const ResourcePtr& src, const ResourcePtr& dest, const ResourceMgrP 
        auto cur_neighbours = cur_node->GetNeighbours();



        for (auto& neighbour : cur_neighbours) {

            auto neighbour_res = std::static_pointer_cast<Resource>(neighbour.neighbour_node.lock());

            auto neighbour_res = std::static_pointer_cast<Resource>(neighbour.neighbour_node);

            dis_matrix[name_id_map.at(res->name())][name_id_map.at(neighbour_res->name())] =

                neighbour.connection.transport_cost();

        }

core/src/scheduler/ResourceMgr.h

+0 −1
Original line number Diff line number Diff line
@@ -75,7 +75,6 @@ class ResourceMgr : public interface::dumpable { 
        return gpu_resources_;

    }



    // TODO(wxyu): why return shared pointer

    inline std::vector<ResourcePtr>

    GetAllResources() {

        return resources_;