Commit 876d7da1 authored by xiaojun.lin's avatar xiaojun.lin
Browse files

Merge branch '0.5.0' into knowhere_ut 

# Conflicts:
#	core/src/index/unittest/test_nsg/test_nsg.cpp


Former-commit-id: 79aae5431956389a5d3aa22b6d71b333966a11b1
parents a764aa57 7c3e5bfe

CHANGELOG.md

+2 −0
Original line number Diff line number Diff line
@@ -26,6 +26,8 @@ Please mark all change in change log and use the ticket from JIRA. 
- MS-653 - When config check fail, Milvus close without message

- MS-654 - Describe index timeout when building index

- MS-658 - Fix SQ8 Hybrid can't search

- \#20 - C++ sdk example get grpc error 

- \#23 - Add unittest to improve code coverage



## Improvement

- MS-552 - Add and change the easylogging library

core/src/scheduler/optimizer/LargeSQ8HPass.cpp

+39 −39
Original line number Diff line number Diff line
@@ -28,45 +28,45 @@ namespace scheduler { 


bool

LargeSQ8HPass::Run(const TaskPtr& task) {

    if (task->Type() != TaskType::SearchTask) {

        return false;

    }



    auto search_task = std::static_pointer_cast<XSearchTask>(task);

    if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8H) {

        return false;

    }



    auto search_job = std::static_pointer_cast<SearchJob>(search_task->job_.lock());



    // TODO: future, Index::IVFSQ8H, if nq < threshold set cpu, else set gpu

    if (search_job->nq() < 100) {

        return false;

    }



    std::vector<uint64_t> gpus = scheduler::get_gpu_pool();

    std::vector<int64_t> all_free_mem;

    for (auto& gpu : gpus) {

        auto cache = cache::GpuCacheMgr::GetInstance(gpu);

        auto free_mem = cache->CacheCapacity() - cache->CacheUsage();

        all_free_mem.push_back(free_mem);

    }



    auto max_e = std::max_element(all_free_mem.begin(), all_free_mem.end());

    auto best_index = std::distance(all_free_mem.begin(), max_e);

    auto best_device_id = gpus[best_index];



    ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id);

    if (not res_ptr) {

        SERVER_LOG_ERROR << "GpuResource " << best_device_id << " invalid.";

        // TODO: throw critical error and exit

        return false;

    }



    auto label = std::make_shared<SpecResLabel>(std::weak_ptr<Resource>(res_ptr));

    task->label() = label;



    return true;

    //    if (task->Type() != TaskType::SearchTask) {

    //        return false;

    //    }

    //

    //    auto search_task = std::static_pointer_cast<XSearchTask>(task);

    //    if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8H) {

    //        return false;

    //    }

    //

    //    auto search_job = std::static_pointer_cast<SearchJob>(search_task->job_.lock());

    //

    //    // TODO: future, Index::IVFSQ8H, if nq < threshold set cpu, else set gpu

    //    if (search_job->nq() < 100) {

    //        return false;

    //    }

    //

    //    std::vector<uint64_t> gpus = scheduler::get_gpu_pool();

    //    std::vector<int64_t> all_free_mem;

    //    for (auto& gpu : gpus) {

    //        auto cache = cache::GpuCacheMgr::GetInstance(gpu);

    //        auto free_mem = cache->CacheCapacity() - cache->CacheUsage();

    //        all_free_mem.push_back(free_mem);

    //    }

    //

    //    auto max_e = std::max_element(all_free_mem.begin(), all_free_mem.end());

    //    auto best_index = std::distance(all_free_mem.begin(), max_e);

    //    auto best_device_id = gpus[best_index];

    //

    //    ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id);

    //    if (not res_ptr) {

    //        SERVER_LOG_ERROR << "GpuResource " << best_device_id << " invalid.";

    //        // TODO: throw critical error and exit

    //        return false;

    //    }

    //

    //    auto label = std::make_shared<SpecResLabel>(std::weak_ptr<Resource>(res_ptr));

    //    task->label() = label;

    //

    //    return true;

}



}  // namespace scheduler

core/src/scheduler/optimizer/Optimizer.cpp

+3 −3
Original line number Diff line number Diff line
@@ -22,9 +22,9 @@ namespace scheduler { 


void

Optimizer::Init() {

    for (auto& pass : pass_list_) {

        pass->Init();

    }

    //    for (auto& pass : pass_list_) {

    //        pass->Init();

    //    }

}



bool

core/src/scheduler/task/BuildIndexTask.cpp

+3 −5
Original line number Diff line number Diff line
@@ -55,9 +55,6 @@ XBuildIndexTask::Load(milvus::scheduler::LoadType type, uint8_t device_id) { 
            } else if (type == LoadType::CPU2GPU) {

                stat = to_index_engine_->CopyToIndexFileToGpu(device_id);

                type_str = "CPU2GPU";

            } else if (type == LoadType::GPU2CPU) {

                stat = to_index_engine_->CopyToCpu();

                type_str = "GPU2CPU";

            } else {

                error_msg = "Wrong load type";

                stat = Status(SERVER_UNEXPECTED_ERROR, error_msg);
@@ -199,8 +196,9 @@ XBuildIndexTask::Execute() { 
            ENGINE_LOG_DEBUG << "New index file " << table_file.file_id_ << " of size " << index->PhysicalSize()

                             << " bytes"

                             << " from file " << origin_file.file_id_;



            //            index->Cache();

            if (build_index_job->options().insert_cache_immediately_) {

                index->Cache();

            }

        } else {

            // failed to update meta, mark the new file as to_delete, don't delete old file

            origin_file.file_type_ = engine::meta::TableFileSchema::TO_INDEX;

core/unittest/db/test_db.cpp

+18 −0
Original line number Diff line number Diff line
@@ -308,6 +308,12 @@ TEST_F(DBTest, SEARCH_TEST) { 
        ASSERT_TRUE(stat.ok());

    }



    {

        milvus::engine::QueryResults large_nq_results;

        stat = db_->Query(TABLE_NAME, k, 200, 10, xq.data(), large_nq_results);

        ASSERT_TRUE(stat.ok());

    }



    {//search by specify index file

        milvus::engine::meta::DatesT dates;

        std::vector<std::string> file_ids = {"1", "2", "3", "4", "5", "6"};
@@ -315,6 +321,8 @@ TEST_F(DBTest, SEARCH_TEST) { 
        stat = db_->Query(TABLE_NAME, file_ids, k, nq, 10, xq.data(), dates, results);

        ASSERT_TRUE(stat.ok());

    }





#endif

}
@@ -412,6 +420,16 @@ TEST_F(DBTest, INDEX_TEST) { 
    stat = db_->CreateIndex(table_info.table_id_, index);

    ASSERT_TRUE(stat.ok());



    index.engine_type_ = (int) milvus::engine::EngineType::FAISS_IVFFLAT;

    stat = db_->CreateIndex(table_info.table_id_, index);

    ASSERT_TRUE(stat.ok());



#ifdef CUSTOMIZATION

    index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8H;

    stat = db_->CreateIndex(table_info.table_id_, index);

    ASSERT_TRUE(stat.ok());

#endif



    milvus::engine::TableIndex index_out;

    stat = db_->DescribeIndex(table_info.table_id_, index_out);

    ASSERT_TRUE(stat.ok());