Commit d28d5188 authored by peng.xu's avatar peng.xu
Browse files

Merge branch 'branch-0.5.0' into 'branch-0.5.0' 

MS-574 Milvus configuration refactor

See merge request megasearch/milvus!621

Former-commit-id: 7bbcf0817e4e2aa2ddea5260cc284b71545ff752
parents 199c7089 ea447964

cpp/conf/server_config.template

+19 −21
Original line number Diff line number Diff line 
# All the following configurations are default values.



server_config:

  address: 0.0.0.0                  # milvus server ip address (IPv4)

  port: 19530                       # port range: 1025 ~ 65534

  mode: single                  # deployment type: single, cluster, read_only

  deploy_mode: single               # deployment type: single, cluster_readonly, cluster_writable

  time_zone: UTC+8



db_config:

  path: @MILVUS_DB_PATH@        # milvus database path

  slave_path:                   # secondary database path, split by semicolon

  primary_path: @MILVUS_DB_PATH@    # path used to store data and meta

  secondary_path:                   # path used to store data only, split by semicolon



  # URI format: dialect://username:password@host:port/database

  # All parts except dialect are optional, but you MUST include the delimiters

  # Currently dialect supports mysql or sqlite

  backend_url: sqlite://:@:/

  backend_url: sqlite://:@:/        # URI format: dialect://username:password@host:port/database

                                    # Keep 'dialect://:@:/', and replace other texts with real values.

                                    # Replace 'dialect' with 'mysql' or 'sqlite'



  archive_disk_threshold: 0     # GB, file will be archived when disk usage exceed, 0 for no limit

  archive_days_threshold: 0     # DAYS, older files will be archived, 0 for no limit

  buffer_size: 4                # GB, maximum insert buffer size allowed

  insert_buffer_size: 4             # GB, maximum insert buffer size allowed

  build_index_gpu: 0                # gpu id used for building index



metric_config:

  auto_bootup: off              # whether enable monitoring when bootup

  enable_monitor: false             # enable monitoring or not

  collector: prometheus             # prometheus

  prometheus_config:

    port: 8080                      # port prometheus used to fetch metrics



cache_config:

  cpu_mem_capacity: 16          # GB, CPU memory size used for cache

  cpu_mem_threshold: 0.85       # percent of data kept when cache cleanup triggered

  cache_insert_data: false      # whether load data into cache when insert

  cpu_mem_capacity: 16              # GB, CPU memory used for cache

  cpu_mem_threshold: 0.85           # percentage of data kept when cache cleanup triggered

  cache_insert_data: false          # whether load inserted data into cache



engine_config:

  blas_threshold: 20



resource_config:

  mode: simple

  pool:

  resource_pool:

    - cpu

    - gpu0

cpp/src/db/DBImpl.cpp

+3 −3
Original line number Diff line number Diff line
@@ -77,7 +77,7 @@ Status DBImpl::Start() { 
    shutting_down_.store(false, std::memory_order_release);



    //for distribute version, some nodes are read only

    if (options_.mode_ != DBOptions::MODE::READ_ONLY) {

    if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) {

        ENGINE_LOG_TRACE << "StartTimerTasks";

        bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this);

    }
@@ -98,7 +98,7 @@ Status DBImpl::Stop() { 
    //wait compaction/buildindex finish

    bg_timer_thread_.join();



    if (options_.mode_ != DBOptions::MODE::READ_ONLY) {

    if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) {

        meta_ptr_->CleanUp();

    }
@@ -704,7 +704,7 @@ void DBImpl::BackgroundCompaction(std::set<std::string> table_ids) { 
    meta_ptr_->Archive();



    int ttl = 5*meta::M_SEC;//default: file will be deleted after 5 minutes

    if (options_.mode_ == DBOptions::MODE::CLUSTER) {

    if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) {

        ttl = meta::D_SEC;

    }

    meta_ptr_->CleanUpFilesWithTTL(ttl);

cpp/src/db/Options.h

+3 −3
Original line number Diff line number Diff line
@@ -60,9 +60,9 @@ struct DBMetaOptions { 


struct DBOptions {

    typedef enum {

        SINGLE,

        CLUSTER,

        READ_ONLY

        SINGLE = 0,

        CLUSTER_READONLY,

        CLUSTER_WRITABLE

    } MODE;



    uint16_t  merge_trigger_number_ = 2;

cpp/src/db/meta/MySQLMetaImpl.cpp

+2 −2
Original line number Diff line number Diff line
@@ -284,7 +284,7 @@ Status MySQLMetaImpl::Initialize() { 
    //step 5: create meta tables

    try {



        if (mode_ != DBOptions::MODE::READ_ONLY) {

        if (mode_ != DBOptions::MODE::CLUSTER_READONLY) {

            CleanUp();

        }
@@ -768,7 +768,7 @@ Status MySQLMetaImpl::DeleteTable(const std::string &table_id) { 


        } //Scoped Connection



        if (mode_ == DBOptions::MODE::CLUSTER) {

        if (mode_ == DBOptions::MODE::CLUSTER_WRITABLE) {

            DeleteTableFiles(table_id);

        }

cpp/src/metrics/PrometheusMetrics.cpp

+1 −1
Original line number Diff line number Diff line
@@ -31,7 +31,7 @@ ErrorCode 
PrometheusMetrics::Init() {

    try {

        Config &config = Config::GetInstance();

        Status s = config.GetMetricConfigAutoBootup(startup_);

        Status s = config.GetMetricConfigEnableMonitor(startup_);

        if (!s.ok()) return s.code();

        if (!startup_) return SERVER_SUCCESS;