Commit 3bf12c0d authored by Jin Hai's avatar Jin Hai Committed by GitHub
Browse files

Merge pull request #165 from fishpenguin/branch-0.5.1-yk 

 #164 - Add CPU version for building index 

Former-commit-id: 0787555623a1e34dc7c9d52a95e674100e445e95
parents 490269e8 84a75305

CHANGELOG.md

+1 −0
Original line number Diff line number Diff line
@@ -14,6 +14,7 @@ Please mark all change in change log and use the ticket from JIRA. 
- \#115 - Using new structure for tasktable

- \#139 - New config option use_gpu_threshold

- \#146 - Add only GPU and only CPU version for IVF_SQ8 and IVF_FLAT

- \#164 - Add CPU version for building index



## Improvement

- \#64 - Improvement dump function in scheduler

core/src/scheduler/JobMgr.cpp

+17 −12
Original line number Diff line number Diff line
@@ -104,10 +104,7 @@ JobMgr::build_task(const JobPtr& job) { 


void

JobMgr::calculate_path(const TaskPtr& task) {

    if (task->type_ != TaskType::SearchTask) {

        return;

    }



    if (task->type_ == TaskType::SearchTask) {

        if (task->label()->Type() != TaskLabelType::SPECIFIED_RESOURCE) {

            return;

        }
@@ -118,6 +115,14 @@ JobMgr::calculate_path(const TaskPtr& task) { 
        auto dest = spec_label->resource();

        ShortestPath(src.lock(), dest.lock(), res_mgr_, path);

        task->path() = Path(path, path.size() - 1);

    } else if (task->type_ == TaskType::BuildIndexTask) {

        auto spec_label = std::static_pointer_cast<SpecResLabel>(task->label());

        auto src = res_mgr_->GetDiskResources()[0];

        auto dest = spec_label->resource();

        std::vector<std::string> path;

        ShortestPath(src.lock(), dest.lock(), res_mgr_, path);

        task->path() = Path(path, path.size() - 1);

    }

}



}  // namespace scheduler

core/src/scheduler/SchedInst.cpp

+6 −6
Original line number Diff line number Diff line
@@ -55,8 +55,8 @@ load_simple_config() { 
    // get resources

    auto gpu_ids = get_gpu_pool();



    int32_t build_gpu_id;

    config.GetResourceConfigIndexBuildDevice(build_gpu_id);

    int32_t index_build_device_id;

    config.GetResourceConfigIndexBuildDevice(index_build_device_id);



    // create and connect

    ResMgrInst::GetInstance()->Add(ResourceFactory::Create("disk", "DISK", 0, true, false));
@@ -70,15 +70,15 @@ load_simple_config() { 
    for (auto& gpu_id : gpu_ids) {

        ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true));

        ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), pcie);

        if (build_gpu_id == gpu_id) {

        if (index_build_device_id == gpu_id) {

            find_build_gpu_id = true;

        }

    }



    if (not find_build_gpu_id) {

    if (not find_build_gpu_id && index_build_device_id != server::CPU_DEVICE_ID) {

        ResMgrInst::GetInstance()->Add(

            ResourceFactory::Create(std::to_string(build_gpu_id), "GPU", build_gpu_id, true, true));

        ResMgrInst::GetInstance()->Connect("cpu", std::to_string(build_gpu_id), pcie);

            ResourceFactory::Create(std::to_string(index_build_device_id), "GPU", index_build_device_id, true, true));

        ResMgrInst::GetInstance()->Connect("cpu", std::to_string(index_build_device_id), pcie);

    }

}

core/src/scheduler/SchedInst.h

+0 −1
Original line number Diff line number Diff line
@@ -106,7 +106,6 @@ class OptimizerInst { 
                        has_cpu = true;

                    }

                }



                std::vector<PassPtr> pass_list;

                pass_list.push_back(std::make_shared<LargeSQ8HPass>());

                pass_list.push_back(std::make_shared<HybridPass>());

core/src/scheduler/TaskCreator.cpp

+9 −2
Original line number Diff line number Diff line
@@ -70,8 +70,15 @@ TaskCreator::Create(const DeleteJobPtr& job) { 
std::vector<TaskPtr>

TaskCreator::Create(const BuildIndexJobPtr& job) {

    std::vector<TaskPtr> tasks;

    // TODO(yukun): remove "disk" hardcode here

    ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource("disk");

    server::Config& config = server::Config::GetInstance();

    int32_t build_index_id;

    Status stat = config.GetResourceConfigIndexBuildDevice(build_index_id);

    ResourcePtr res_ptr;

    if (build_index_id == server::CPU_DEVICE_ID) {

        res_ptr = ResMgrInst::GetInstance()->GetResource("cpu");

    } else {

        res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, build_index_id);

    }



    for (auto& to_index_file : job->to_index_files()) {

        auto label = std::make_shared<SpecResLabel>(std::weak_ptr<Resource>(res_ptr));