Unverified Commit 8c48d95f authored by Jin Hai's avatar Jin Hai Committed by GitHub
Browse files

Merge pull request #418 from fishpenguin/0.6.0-yk-refactor-scheduler 

Refactor optimizer
parents 796750f6 a176eb94

core/src/scheduler/JobMgr.cpp

+4 −4
Original line number Diff line number Diff line
@@ -85,7 +85,7 @@ JobMgr::worker_function() { 
        }



        for (auto& task : tasks) {

            calculate_path(task);

            calculate_path(res_mgr_, task);

        }



        // disk resources NEVER be empty.
@@ -103,7 +103,7 @@ JobMgr::build_task(const JobPtr& job) { 
}



void

JobMgr::calculate_path(const TaskPtr& task) {

JobMgr::calculate_path(const ResourceMgrPtr& res_mgr, const TaskPtr& task) {

    if (task->type_ != TaskType::SearchTask && task->type_ != TaskType::BuildIndexTask) {

        return;

    }
@@ -114,9 +114,9 @@ JobMgr::calculate_path(const TaskPtr& task) { 


    std::vector<std::string> path;

    auto spec_label = std::static_pointer_cast<SpecResLabel>(task->label());

    auto src = res_mgr_->GetDiskResources()[0];

    auto src = res_mgr->GetDiskResources()[0];

    auto dest = spec_label->resource();

    ShortestPath(src.lock(), dest.lock(), res_mgr_, path);

    ShortestPath(src.lock(), dest.lock(), res_mgr, path);

    task->path() = Path(path, path.size() - 1);

}

core/src/scheduler/JobMgr.h

+3 −2
Original line number Diff line number Diff line
@@ -59,8 +59,9 @@ class JobMgr : public interface::dumpable { 
    static std::vector<TaskPtr>

    build_task(const JobPtr& job);



    void

    calculate_path(const TaskPtr& task);

 public:

    static void

    calculate_path(const ResourceMgrPtr& res_mgr, const TaskPtr& task);



 private:

    bool running_ = false;

core/src/scheduler/SchedInst.h

+10 −12
Original line number Diff line number Diff line
@@ -23,10 +23,11 @@ 
#include "Scheduler.h"

#include "Utils.h"

#include "optimizer/BuildIndexPass.h"

#include "optimizer/FaissFlatPass.h"

#include "optimizer/FaissIVFFlatPass.h"

#include "optimizer/FaissIVFSQ8HPass.h"

#include "optimizer/FaissIVFSQ8Pass.h"

#include "optimizer/FallbackPass.h"

#include "optimizer/HybridPass.h"

#include "optimizer/LargeSQ8HPass.h"

#include "optimizer/OnlyCPUPass.h"

#include "optimizer/Optimizer.h"

#include "server/Config.h"
@@ -100,15 +101,12 @@ class OptimizerInst { 
            std::lock_guard<std::mutex> lock(mutex_);

            if (instance == nullptr) {

                std::vector<PassPtr> pass_list;

                pass_list.push_back(std::make_shared<LargeSQ8HPass>());

                pass_list.push_back(std::make_shared<HybridPass>());

#ifdef MILVUS_CPU_VERSION

                pass_list.push_back(std::make_shared<OnlyCPUPass>());

#else

                server::Config& config = server::Config::GetInstance();

                std::vector<int32_t> build_resources;

                config.GetGpuResourceConfigBuildIndexResources(build_resources);

                pass_list.push_back(std::make_shared<BuildIndexPass>(build_resources));

#ifdef MILVUS_GPU_VERSION

                pass_list.push_back(std::make_shared<BuildIndexPass>());

                pass_list.push_back(std::make_shared<FaissFlatPass>());

                pass_list.push_back(std::make_shared<FaissIVFFlatPass>());

                pass_list.push_back(std::make_shared<FaissIVFSQ8Pass>());

                pass_list.push_back(std::make_shared<FaissIVFSQ8HPass>());

#endif

                pass_list.push_back(std::make_shared<FallbackPass>());

                instance = std::make_shared<Optimizer>(pass_list);

core/src/scheduler/Scheduler.cpp

+0 −4
Original line number Diff line number Diff line
@@ -108,10 +108,6 @@ Scheduler::OnLoadCompleted(const EventPtr& event) { 


    auto task_table_type = load_completed_event->task_table_item_->task->label()->Type();

    switch (task_table_type) {

        case TaskLabelType::DEFAULT: {

            Action::DefaultLabelTaskScheduler(res_mgr_, resource, load_completed_event);

            break;

        }

        case TaskLabelType::SPECIFIED_RESOURCE: {

            Action::SpecifiedResourceLabelTaskScheduler(res_mgr_, resource, load_completed_event);

            break;

core/src/scheduler/TaskCreator.cpp

+2 −7
Original line number Diff line number Diff line
@@ -18,7 +18,6 @@ 
#include "scheduler/TaskCreator.h"

#include "SchedInst.h"

#include "tasklabel/BroadcastLabel.h"

#include "tasklabel/DefaultLabel.h"

#include "tasklabel/SpecResLabel.h"



namespace milvus {
@@ -47,8 +46,7 @@ std::vector<TaskPtr> 
TaskCreator::Create(const SearchJobPtr& job) {

    std::vector<TaskPtr> tasks;

    for (auto& index_file : job->index_files()) {

        auto label = std::make_shared<DefaultLabel>();

        auto task = std::make_shared<XSearchTask>(index_file.second, label);

        auto task = std::make_shared<XSearchTask>(index_file.second, nullptr);

        task->job_ = job;

        tasks.emplace_back(task);

    }
@@ -70,11 +68,8 @@ TaskCreator::Create(const DeleteJobPtr& job) { 
std::vector<TaskPtr>

TaskCreator::Create(const BuildIndexJobPtr& job) {

    std::vector<TaskPtr> tasks;

    ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource("cpu");



    for (auto& to_index_file : job->to_index_files()) {

        auto label = std::make_shared<SpecResLabel>(std::weak_ptr<Resource>(res_ptr));

        auto task = std::make_shared<XBuildIndexTask>(to_index_file.second, label);

        auto task = std::make_shared<XBuildIndexTask>(to_index_file.second, nullptr);

        task->job_ = job;

        tasks.emplace_back(task);

    }