Commit d4fe7607 authored by Jin Hai's avatar Jin Hai Committed by GitHub
Browse files

Merge pull request #119 from scsven/dev 

Using shared_ptr instead of weak_ptr to avoid performance loss

Former-commit-id: 3edd181f61f7a70ea8a062302079ed50e5fdba2c
parents 44941b3e 56e734df

CHANGELOG.md

+3 −0
Original line number Diff line number Diff line
@@ -13,6 +13,9 @@ Please mark all change in change log and use the ticket from JIRA. 
- \#82 - Move easyloggingpp into "external" directory

- \#92 - Speed up CMake build process

- \#96 - Remove .a file in milvus/lib for docker-version

- \#118 - Using shared_ptr instead of weak_ptr to avoid performance loss

- \#122 - Add unique id for Job



## Feature

- \#115 - Using new structure for tasktable

core/src/db/DBImpl.cpp

+3 −3
Original line number Diff line number Diff line
@@ -136,7 +136,7 @@ DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& dates) { 


        // scheduler will determine when to delete table files

        auto nres = scheduler::ResMgrInst::GetInstance()->GetNumOfComputeResource();

        scheduler::DeleteJobPtr job = std::make_shared<scheduler::DeleteJob>(0, table_id, meta_ptr_, nres);

        scheduler::DeleteJobPtr job = std::make_shared<scheduler::DeleteJob>(table_id, meta_ptr_, nres);

        scheduler::JobMgrInst::GetInstance()->Put(job);

        job->WaitAndDelete();

    } else {
@@ -439,7 +439,7 @@ DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSchema& fi 


    // step 1: get files to search

    ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size();

    scheduler::SearchJobPtr job = std::make_shared<scheduler::SearchJob>(0, k, nq, nprobe, vectors);

    scheduler::SearchJobPtr job = std::make_shared<scheduler::SearchJob>(k, nq, nprobe, vectors);

    for (auto& file : files) {

        scheduler::TableFileSchemaPtr file_ptr = std::make_shared<meta::TableFileSchema>(file);

        job->AddIndexFile(file_ptr);
@@ -754,7 +754,7 @@ DBImpl::BackgroundBuildIndex() { 
    Status status;



    if (!to_index_files.empty()) {

        scheduler::BuildIndexJobPtr job = std::make_shared<scheduler::BuildIndexJob>(0, meta_ptr_, options_);

        scheduler::BuildIndexJobPtr job = std::make_shared<scheduler::BuildIndexJob>(meta_ptr_, options_);



        // step 2: put build index task to scheduler

        for (auto& file : to_index_files) {

core/src/scheduler/Algorithm.cpp

+1 −1
Original line number Diff line number Diff line
@@ -54,7 +54,7 @@ ShortestPath(const ResourcePtr& src, const ResourcePtr& dest, const ResourceMgrP 
        auto cur_neighbours = cur_node->GetNeighbours();



        for (auto& neighbour : cur_neighbours) {

            auto neighbour_res = std::static_pointer_cast<Resource>(neighbour.neighbour_node.lock());

            auto neighbour_res = std::static_pointer_cast<Resource>(neighbour.neighbour_node);

            dis_matrix[name_id_map.at(res->name())][name_id_map.at(neighbour_res->name())] =

                neighbour.connection.transport_cost();

        }

core/src/scheduler/ResourceMgr.h

+0 −1
Original line number Diff line number Diff line
@@ -75,7 +75,6 @@ class ResourceMgr : public interface::dumpable { 
        return gpu_resources_;

    }



    // TODO(wxyu): why return shared pointer

    inline std::vector<ResourcePtr>

    GetAllResources() {

        return resources_;

core/src/scheduler/SchedInst.cpp

+0 −70
Original line number Diff line number Diff line
@@ -82,79 +82,9 @@ load_simple_config() { 
    }

}



void

load_advance_config() {

    //    try {

    //        server::ConfigNode &config = server::Config::GetInstance().GetConfig(server::CONFIG_RESOURCE);

    //

    //        if (config.GetChildren().empty()) throw "resource_config null exception";

    //

    //        auto resources = config.GetChild(server::CONFIG_RESOURCES).GetChildren();

    //

    //        if (resources.empty()) throw "Children of resource_config null exception";

    //

    //        for (auto &resource : resources) {

    //            auto &resname = resource.first;

    //            auto &resconf = resource.second;

    //            auto type = resconf.GetValue(server::CONFIG_RESOURCE_TYPE);

    ////        auto memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_MEMORY);

    //            auto device_id = resconf.GetInt64Value(server::CONFIG_RESOURCE_DEVICE_ID);

    ////            auto enable_loader = resconf.GetBoolValue(server::CONFIG_RESOURCE_ENABLE_LOADER);

    //            auto enable_loader = true;

    //            auto enable_executor = resconf.GetBoolValue(server::CONFIG_RESOURCE_ENABLE_EXECUTOR);

    //            auto pinned_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_PIN_MEMORY);

    //            auto temp_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_TEMP_MEMORY);

    //            auto resource_num = resconf.GetInt64Value(server::CONFIG_RESOURCE_NUM);

    //

    //            auto res = ResMgrInst::GetInstance()->Add(ResourceFactory::Create(resname,

    //                                                                              type,

    //                                                                              device_id,

    //                                                                              enable_loader,

    //                                                                              enable_executor));

    //

    //            if (res.lock()->type() == ResourceType::GPU) {

    //                auto pinned_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_PIN_MEMORY, 300);

    //                auto temp_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_TEMP_MEMORY, 300);

    //                auto resource_num = resconf.GetInt64Value(server::CONFIG_RESOURCE_NUM, 2);

    //                pinned_memory = 1024 * 1024 * pinned_memory;

    //                temp_memory = 1024 * 1024 * temp_memory;

    //                knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(device_id,

    //                                                                        pinned_memory,

    //                                                                        temp_memory,

    //                                                                        resource_num);

    //            }

    //        }

    //

    //        knowhere::FaissGpuResourceMgr::GetInstance().InitResource();

    //

    //        auto connections = config.GetChild(server::CONFIG_RESOURCE_CONNECTIONS).GetChildren();

    //        if (connections.empty()) throw "connections config null exception";

    //        for (auto &conn : connections) {

    //            auto &connect_name = conn.first;

    //            auto &connect_conf = conn.second;

    //            auto connect_speed = connect_conf.GetInt64Value(server::CONFIG_SPEED_CONNECTIONS);

    //            auto connect_endpoint = connect_conf.GetValue(server::CONFIG_ENDPOINT_CONNECTIONS);

    //

    //            std::string delimiter = "===";

    //            std::string left = connect_endpoint.substr(0, connect_endpoint.find(delimiter));

    //            std::string right = connect_endpoint.substr(connect_endpoint.find(delimiter) + 3,

    //                                                        connect_endpoint.length());

    //

    //            auto connection = Connection(connect_name, connect_speed);

    //            ResMgrInst::GetInstance()->Connect(left, right, connection);

    //        }

    //    } catch (const char *msg) {

    //        SERVER_LOG_ERROR << msg;

    //        // TODO(wxyu): throw exception instead

    //        exit(-1);

    ////        throw std::exception();

    //    }

}



void

StartSchedulerService() {

    load_simple_config();

    //    load_advance_config();

    ResMgrInst::GetInstance()->Start();

    SchedInst::GetInstance()->Start();

    JobMgrInst::GetInstance()->Start();