Commit 05b99b8e authored by 余昆's avatar 余昆
Browse files

#164 - Add CPU version for building index 


Former-commit-id: 120ca273dbbbbd32f680f85730c931ef64115f19
parent 490269e8

CHANGELOG.md

+1 −0
Original line number Diff line number Diff line
@@ -14,6 +14,7 @@ Please mark all change in change log and use the ticket from JIRA. 
- \#115 - Using new structure for tasktable

- \#139 - New config option use_gpu_threshold

- \#146 - Add only GPU and only CPU version for IVF_SQ8 and IVF_FLAT

- \#164 - Add CPU version for building index



## Improvement

- \#64 - Improvement dump function in scheduler

core/src/scheduler/JobMgr.cpp

+18 −12
Original line number Diff line number Diff line
@@ -104,10 +104,7 @@ JobMgr::build_task(const JobPtr& job) { 


void

JobMgr::calculate_path(const TaskPtr& task) {

    if (task->type_ != TaskType::SearchTask) {

        return;

    }



    if (task->type_ == TaskType::SearchTask) {

        if (task->label()->Type() != TaskLabelType::SPECIFIED_RESOURCE) {

            return;

        }
@@ -118,6 +115,15 @@ JobMgr::calculate_path(const TaskPtr& task) { 
        auto dest = spec_label->resource();

        ShortestPath(src.lock(), dest.lock(), res_mgr_, path);

        task->path() = Path(path, path.size() - 1);

        

    } else if (task->type_ == TaskType::BuildIndexTask) {

        auto spec_label = std::static_pointer_cast<SpecResLabel>(task->label());

        auto src = res_mgr_->GetDiskResources()[0];

        auto dest = spec_label->resource();

        std::vector<std::string> path;

        ShortestPath(src.lock(), dest.lock(), res_mgr_, path);

        task->path() = Path(path, path.size() - 1);

    }

}



}  // namespace scheduler

core/src/scheduler/SchedInst.cpp

+1 −1
Original line number Diff line number Diff line
@@ -75,7 +75,7 @@ load_simple_config() { 
        }

    }



    if (not find_build_gpu_id) {

    if (not find_build_gpu_id && build_gpu_id != server::CPU_DEVICE_ID) {

        ResMgrInst::GetInstance()->Add(

            ResourceFactory::Create(std::to_string(build_gpu_id), "GPU", build_gpu_id, true, true));

        ResMgrInst::GetInstance()->Connect("cpu", std::to_string(build_gpu_id), pcie);

core/src/scheduler/SchedInst.h

+0 −1
Original line number Diff line number Diff line
@@ -106,7 +106,6 @@ class OptimizerInst { 
                        has_cpu = true;

                    }

                }



                std::vector<PassPtr> pass_list;

                pass_list.push_back(std::make_shared<LargeSQ8HPass>());

                pass_list.push_back(std::make_shared<HybridPass>());

core/src/scheduler/TaskCreator.cpp

+10 −2
Original line number Diff line number Diff line
@@ -70,8 +70,16 @@ TaskCreator::Create(const DeleteJobPtr& job) { 
std::vector<TaskPtr>

TaskCreator::Create(const BuildIndexJobPtr& job) {

    std::vector<TaskPtr> tasks;

    // TODO(yukun): remove "disk" hardcode here

    ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource("disk");

    

    server::Config& config = server::Config::GetInstance();

    int32_t build_index_id;

    Status stat = config.GetResourceConfigIndexBuildDevice(build_index_id);

    ResourcePtr res_ptr;

    if (build_index_id == server::CPU_DEVICE_ID) {

        res_ptr = ResMgrInst::GetInstance()->GetResource("cpu");

    } else {

        res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, build_index_id);

    }



    for (auto& to_index_file : job->to_index_files()) {

        auto label = std::make_shared<SpecResLabel>(std::weak_ptr<Resource>(res_ptr));