Commit 4e3d74c4 authored by jinhai's avatar jinhai
Browse files

Merge branch 'branch-0.5.0-yk' into 'branch-0.5.0' 

MS-636 Add optimizer in scheduler for FAISS_IVFSQ8H

See merge request megasearch/milvus!690

Former-commit-id: d1d100fa90ae75925a43a9172973815861a5a92a
parents 7410dd92 6d5874f5

cpp/CHANGELOG.md

+1 −0
Original line number Diff line number Diff line
@@ -37,6 +37,7 @@ Please mark all change in change log and use the ticket from JIRA. 
## New Feature

- MS-627 - Integrate new index: IVFSQHybrid

- MS-631 - IVFSQ8H Index support

- MS-636 - Add optimizer in scheduler for FAISS_IVFSQ8H



## Task

- MS-554 - Change license to Apache 2.0

cpp/src/db/DBImpl.cpp

+13 −12
Original line number Diff line number Diff line
@@ -900,6 +900,7 @@ DBImpl::BackgroundBuildIndex() { 
    meta_ptr_->FilesToIndex(to_index_files);

    Status status;



    if (!to_index_files.empty()) {

        scheduler::BuildIndexJobPtr job = std::make_shared<scheduler::BuildIndexJob>(0, meta_ptr_, options_);



        // step 2: put build index task to scheduler
@@ -913,7 +914,7 @@ DBImpl::BackgroundBuildIndex() { 
            Status status = job->GetStatus();

            ENGINE_LOG_ERROR << "Building index failed: " << status.ToString();

        }



    }

    //    for (auto &file : to_index_files) {

    //        status = BuildIndex(file);

    //        if (!status.ok()) {

cpp/src/scheduler/Algorithm.cpp

+0 −2
Original line number Diff line number Diff line
@@ -29,8 +29,6 @@ constexpr uint64_t MAXINT = std::numeric_limits<uint32_t>::max(); 
uint64_t

ShortestPath(const ResourcePtr& src, const ResourcePtr& dest, const ResourceMgrPtr& res_mgr,

             std::vector<std::string>& path) {

    std::vector<std::vector<std::string>> paths;



    uint64_t num_of_resources = res_mgr->GetAllResources().size();

    std::unordered_map<uint64_t, std::string> id_name_map;

    std::unordered_map<std::string, uint64_t> name_id_map;

cpp/src/scheduler/JobMgr.cpp

+5 −2
Original line number Diff line number Diff line
@@ -16,7 +16,9 @@ 
// under the License.



#include "scheduler/JobMgr.h"

#include "SchedInst.h"

#include "TaskCreator.h"

#include "optimizer/Optimizer.h"

#include "task/Task.h"



#include <src/scheduler/optimizer/Optimizer.h>
@@ -67,8 +69,9 @@ JobMgr::worker_function() { 
        }



        auto tasks = build_task(job);



        // TODO: optimizer all task

        for (auto& task : tasks) {

            OptimizerInst::GetInstance()->Run(task);

        }



        // disk resources NEVER be empty.

        if (auto disk = res_mgr_->GetDiskResources()[0].lock()) {

cpp/src/scheduler/SchedInst.cpp

+3 −0
Original line number Diff line number Diff line
@@ -38,6 +38,9 @@ std::mutex SchedInst::mutex_; 
scheduler::JobMgrPtr JobMgrInst::instance = nullptr;

std::mutex JobMgrInst::mutex_;



OptimizerPtr OptimizerInst::instance = nullptr;

std::mutex OptimizerInst::mutex_;



void

load_simple_config() {

    server::Config& config = server::Config::GetInstance();