Commit a602302a authored by JinHai-CN's avatar JinHai-CN
Browse files

Merge remote-tracking branch 'upstream/0.5.0' into 0.5.0 


Former-commit-id: f90856809e386ecdce5bb88210d895e141e53d82
parents 7e4badb8 8eaed609

CHANGELOG.md

+4 −0
Original line number Diff line number Diff line
@@ -27,9 +27,12 @@ Please mark all change in change log and use the ticket from JIRA. 
- MS-654 - Describe index timeout when building index

- MS-658 - Fix SQ8 Hybrid can't search

- \#9 Change default gpu_cache_capacity to 4

- MS-665 - IVF_SQ8H search crash when no GPU resource in search_resources

- \#20 - C++ sdk example get grpc error 

- \#23 - Add unittest to improve code coverage

- \#31 - make clang-format failed after run build.sh -l

- \#39 - Create SQ8H index hang if using github server version

- \#30 - Some troubleshoot messages in Milvus do not provide enough information



## Improvement

- MS-552 - Add and change the easylogging library
@@ -51,6 +54,7 @@ Please mark all change in change log and use the ticket from JIRA. 
- MS-626 - Refactor DataObj to support cache any type data

- MS-648 - Improve unittest

- MS-655 - Upgrade SPTAG

- \#42 - Put union of index_build_device and search resources to gpu_pool



## New Feature

- MS-614 - Preload table at startup

core/conf/server_config.template

+2 −3
Original line number Diff line number Diff line
@@ -16,7 +16,6 @@ db_config: 


  insert_buffer_size: 4             # GB, maximum insert buffer size allowed

                                    # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory

  build_index_gpu: 0                # gpu id used for building index



  preload_table:                    # preload data at startup, '*' means load all tables, empty value means no preload

                                    # you can specify preload tables like this: table1,table2,table3
@@ -39,6 +38,6 @@ engine_config: 
                                    # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times



resource_config:

  resource_pool:

    - cpu

  search_resources:                 # define the GPUs used for search computation, valid value: gpux

    - gpu0

  index_build_device: gpu0          # GPU used for building index
 
 No newline at end of file

core/src/db/engine/ExecutionEngineImpl.cpp

+4 −4
Original line number Diff line number Diff line
@@ -65,7 +65,7 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, const std::string& 
    : location_(location), dim_(dimension), index_type_(index_type), metric_type_(metric_type), nlist_(nlist) {

    index_ = CreatetVecIndex(EngineType::FAISS_IDMAP);

    if (!index_) {

        throw Exception(DB_ERROR, "Could not create VecIndex");

        throw Exception(DB_ERROR, "Unsupported index type");

    }



    TempMetaConf temp_conf;
@@ -111,7 +111,7 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) { 
            break;

        }

        default: {

            ENGINE_LOG_ERROR << "Invalid engine type";

            ENGINE_LOG_ERROR << "Unsupported index type";

            return nullptr;

        }

    }
@@ -373,7 +373,7 @@ ExecutionEngineImpl::BuildIndex(const std::string& location, EngineType engine_t 


    auto to_index = CreatetVecIndex(engine_type);

    if (!to_index) {

        throw Exception(DB_ERROR, "Could not create VecIndex");

        throw Exception(DB_ERROR, "Unsupported index type");

    }



    TempMetaConf temp_conf;
@@ -503,7 +503,7 @@ ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { 
Status

ExecutionEngineImpl::Init() {

    server::Config& config = server::Config::GetInstance();

    Status s = config.GetDBConfigBuildIndexGPU(gpu_num_);

    Status s = config.GetResourceConfigIndexBuildDevice(gpu_num_);

    if (!s.ok()) {

        return s;

    }

core/src/scheduler/SchedInst.cpp

+18 −9
Original line number Diff line number Diff line
@@ -50,29 +50,38 @@ load_simple_config() { 
    std::string mode;

    config.GetResourceConfigMode(mode);

    std::vector<std::string> pool;

    config.GetResourceConfigPool(pool);

    config.GetResourceConfigSearchResources(pool);



    // get resources

    bool use_cpu_to_compute = false;

    for (auto& resource : pool) {

        if (resource == "cpu") {

            use_cpu_to_compute = true;

            break;

        }

    }

    auto gpu_ids = get_gpu_pool();



    int32_t build_gpu_id;

    config.GetResourceConfigIndexBuildDevice(build_gpu_id);



    // create and connect

    ResMgrInst::GetInstance()->Add(ResourceFactory::Create("disk", "DISK", 0, true, false));



    auto io = Connection("io", 500);

    ResMgrInst::GetInstance()->Add(ResourceFactory::Create("cpu", "CPU", 0, true, use_cpu_to_compute));

    ResMgrInst::GetInstance()->Add(ResourceFactory::Create("cpu", "CPU", 0, true, true));

    ResMgrInst::GetInstance()->Connect("disk", "cpu", io);



    auto pcie = Connection("pcie", 12000);

    bool find_build_gpu_id = false;

    for (auto& gpu_id : gpu_ids) {

        ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true));

        ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), pcie);

        if (build_gpu_id == gpu_id) {

            find_build_gpu_id = true;

        }

    }



    if (not find_build_gpu_id) {

        ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(build_gpu_id),

                                                               "GPU",

                                                               build_gpu_id,

                                                               true,

                                                               true));

        ResMgrInst::GetInstance()->Connect("cpu", std::to_string(build_gpu_id), pcie);

    }

}

core/src/scheduler/Utils.cpp

+1 −1
Original line number Diff line number Diff line
@@ -48,7 +48,7 @@ get_gpu_pool() { 


    server::Config& config = server::Config::GetInstance();

    std::vector<std::string> pool;

    Status s = config.GetResourceConfigPool(pool);

    Status s = config.GetResourceConfigSearchResources(pool);

    if (!s.ok()) {

        SERVER_LOG_ERROR << s.message();

    }