Merge remote-tracking branch 'source/branch-0.5.0' into branch-0.5.0

- MS-562 - Add JobMgr and TaskCreator in Scheduler

- MS-566 - Refactor cmake

- MS-555 - Remove old scheduler

- MS-574 - Milvus configuration refactor

- MS-578 - Make sure milvus5.0 don't crack 0.3.1 data

- MS-585 - Update namespace in scheduler

server_config:

  address: 0.0.0.0              # milvus server ip address (IPv4)

  port: 19530                 # the port milvus listen to, default: 19530, range: 1025 ~ 65534

  mode: single                # milvus deployment type: single, cluster, read_only

  time_zone: UTC+8            # Use the UTC-x or UTC+x to specify a time zone. eg. UTC+8 for China Standard Time

  port: 19530                   # port range: 1025 ~ 65534

  mode: single                  # deployment type: single, cluster, read_only

  time_zone: UTC+8

db_config:

  db_path: @MILVUS_DB_PATH@             # milvus data storage path

  db_slave_path:                        # secondry data storage path, split by semicolon

  path: @MILVUS_DB_PATH@        # milvus database path

  slave_path:                   # secondary database path, split by semicolon

  # URI format: dialect://username:password@host:port/database

  # All parts except dialect are optional, but you MUST include the delimiters

  # Currently dialect supports mysql or sqlite

  db_backend_url: sqlite://:@:/

  backend_url: sqlite://:@:/

  archive_disk_threshold: 0        # triger archive action if storage size exceed this value, 0 means no limit, unit: GB

  archive_days_threshold: 0        # files older than x days will be archived, 0 means no limit, unit: day

  insert_buffer_size: 4            # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB.

                                   # the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB

  build_index_gpu: 0               # which gpu is used to build index, default: 0, range: 0 ~ gpu number - 1

  archive_disk_threshold: 0     # GB, file will be archived when disk usage exceed, 0 for no limit

  archive_days_threshold: 0     # DAYS, older files will be archived, 0 for no limit

  buffer_size: 4                # GB, maximum insert buffer size allowed

  build_index_gpu: 0            # gpu id used for building index

metric_config:

  is_startup: off                       # if monitoring start: on, off

  collector: prometheus                 # metrics collector: prometheus

  prometheus_config:                    # following are prometheus configure

    port: 8080                          # the port prometheus use to fetch metrics

    push_gateway_ip_address: 127.0.0.1  # push method configure: push gateway ip address

    push_gateway_port: 9091             # push method configure: push gateway port

  auto_bootup: off              # whether enable monitoring when bootup

  collector: prometheus         # prometheus

  prometheus_config:

    port: 8080                  # port prometheus used to fetch metrics

cache_config:

  cpu_cache_capacity: 16            # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory

  cpu_cache_free_percent: 0.85      # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0

  insert_cache_immediately: false   # insert data will be load into cache immediately for hot query

  cpu_mem_capacity: 16          # GB, CPU memory size used for cache

  cpu_mem_threshold: 0.85       # percent of data kept when cache cleanup triggered

  cache_insert_data: false      # whether load data into cache when insert

engine_config:

  use_blas_threshold: 20

  blas_threshold: 20

resource_config:

  mode: simple

  resources:

#    - cpu

  pool:

    - cpu

    - gpu0

#include "CpuCacheMgr.h"

#include "server/ServerConfig.h"

#include "server/Config.h"

#include "utils/Log.h"

namespace zilliz {

}

CpuCacheMgr::CpuCacheMgr() {

    server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);

    int64_t cap = config.GetInt64Value(server::CONFIG_CPU_CACHE_CAPACITY, 16);

    cap *= unit;

    server::Config& config = server::Config::GetInstance();

    Status s;

    int32_t cpu_mem_cap;

    s = config.GetCacheConfigCpuMemCapacity(cpu_mem_cap);

    if (!s.ok()) {

        SERVER_LOG_ERROR << s.message();

    }

    int64_t cap = cpu_mem_cap * unit;

    cache_ = std::make_shared<Cache<DataObjPtr>>(cap, 1UL<<32);

    double free_percent = config.GetDoubleValue(server::CACHE_FREE_PERCENT, 0.85);

    if(free_percent > 0.0 && free_percent <= 1.0) {

        cache_->set_freemem_percent(free_percent);

    float cpu_mem_threshold;

    s = config.GetCacheConfigCpuMemThreshold(cpu_mem_threshold);

    if (!s.ok()) {

        SERVER_LOG_ERROR << s.message();

    }

    if (cpu_mem_threshold > 0.0 && cpu_mem_threshold <= 1.0) {

        cache_->set_freemem_percent(cpu_mem_threshold);

    } else {

        SERVER_LOG_ERROR << "Invalid cache_free_percent: " << free_percent <<

         ", defaultly set to " << cache_->freemem_percent();

        SERVER_LOG_ERROR << "Invalid cpu_mem_threshold: " << cpu_mem_threshold

                         << ", by default set to " << cache_->freemem_percent();

    }

}

#include <sstream>

#include "utils/Log.h"

#include "GpuCacheMgr.h"

#include "server/ServerConfig.h"

#include "server/Config.h"

namespace zilliz {

namespace milvus {

}

GpuCacheMgr::GpuCacheMgr() {

    server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);

    server::Config& config = server::Config::GetInstance();

    Status s;

    int64_t cap = config.GetInt64Value(server::CONFIG_GPU_CACHE_CAPACITY, 0);

    cap *= G_BYTE;

    int32_t gpu_mem_cap;

    s = config.GetCacheConfigGpuMemCapacity(gpu_mem_cap);

    if (!s.ok()) {

        SERVER_LOG_ERROR << s.message();

    }

    int32_t cap = gpu_mem_cap * G_BYTE;

    cache_ = std::make_shared<Cache<DataObjPtr>>(cap, 1UL<<32);

    double free_percent = config.GetDoubleValue(server::GPU_CACHE_FREE_PERCENT, 0.85);

    if (free_percent > 0.0 && free_percent <= 1.0) {

        cache_->set_freemem_percent(free_percent);

    float gpu_mem_threshold;

    s = config.GetCacheConfigGpuMemThreshold(gpu_mem_threshold);

    if (!s.ok()) {

        SERVER_LOG_ERROR << s.message();

    }

    if (gpu_mem_threshold > 0.0 && gpu_mem_threshold <= 1.0) {

        cache_->set_freemem_percent(gpu_mem_threshold);

    } else {

        SERVER_LOG_ERROR << "Invalid gpu_cache_free_percent: " << free_percent <<

                         ", defaultly set to " << cache_->freemem_percent();

        SERVER_LOG_ERROR << "Invalid gpu_mem_threshold: " << gpu_mem_threshold

                         << ", by default set to " << cache_->freemem_percent();

    }

}

#include "src/wrapper/vec_index.h"

#include "src/wrapper/vec_impl.h"

#include "knowhere/common/Exception.h"

#include "server/Config.h"

#include <stdexcept>

// TODO(linxj): remove.

Status ExecutionEngineImpl::Init() {

    using namespace zilliz::milvus::server;

    ServerConfig &config = ServerConfig::GetInstance();

    ConfigNode server_config = config.GetConfig(CONFIG_DB);

    gpu_num_ = server_config.GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, 0);

    server::Config &config = server::Config::GetInstance();

    Status s = config.GetDBConfigBuildIndexGPU(gpu_num_);

    if (!s.ok()) return s;

    return Status::OK();

}