Unverified Commit eba33cc4 authored by 蔡宇东's avatar 蔡宇东 Committed by GitHub
Browse files

Caiyd 090 rename config (#2340) 



* rename config items

Signed-off-by: default avataryudong.cai <yudong.cai@zilliz.com>

* update config unittest

Signed-off-by: default avataryudong.cai <yudong.cai@zilliz.com>

* fix clang-format

Signed-off-by: default avataryudong.cai <yudong.cai@zilliz.com>

* update config unittest

Signed-off-by: default avataryudong.cai <yudong.cai@zilliz.com>

* update log rotate default value

Signed-off-by: default avataryudong.cai <yudong.cai@zilliz.com>

* update parameter usage in LogUtil.cpp (#3)

Signed-off-by: default avatarwxyu <xy.wang@zilliz.com>

* retry ci

Signed-off-by: default avataryudong.cai <yudong.cai@zilliz.com>

* fix ut

Signed-off-by: default avataryudong.cai <yudong.cai@zilliz.com>

Co-authored-by: default avatarJinHai-CN <hai.jin@zilliz.com>
Co-authored-by: default avatarWang XiangYu <scsven@qq.com>
parent 860cc97a

CHANGELOG.md

+0 −1
Original line number Diff line number Diff line
@@ -70,7 +70,6 @@ Please mark all change in change log and use the issue from GitHub 


## Task





# Milvus 0.8.0 (2020-04-15)



## Bug

core/conf/demo/server_config.yaml

+8 −10
Original line number Diff line number Diff line
@@ -68,8 +68,8 @@ db_config: 
# secondary_path       | A semicolon-separated list of secondary directories used   | Path       |                 |

#                      | to save vector data and index data.                        |            |                 |

#----------------------+------------------------------------------------------------+------------+-----------------+

# file_cleanup_timeout | time gap between soft-delete and hard-delete               | Integer    | 10 (s)          |

#                      | range [0, 3600]                                            |            |                 |

# file_cleanup_timeout | The time gap between marking a file as 'deleted' and       | Integer    | 10 (s)          |

#                      | physically deleting this file from disk, range [0, 3600]   |            |                 |

#----------------------+------------------------------------------------------------+------------+-----------------+

storage_config:

  primary_path: /var/lib/milvus
@@ -206,14 +206,12 @@ wal_config: 
#----------------------+------------------------------------------------------------+------------+-----------------+

# fatal.enable         | Whether to enable fatal level logging in Milvus.           | Boolean    | true            |

#----------------------+------------------------------------------------------------+------------+-----------------+

# path                 | Location of logs files.                                    | String     |                 |

# path                 | Absolute path to the folder holding the log files.         | String     |                 |

#----------------------+------------------------------------------------------------+------------+-----------------+

# max_log_file_size    | Max size of a single log file. After exceeding this value, | Integer    | 256 (MB)        |

#                      | rename this file to xxx.log.n, means the nth file.         |            |                 |

# max_log_file_size    | The maximum size of each log file, size range [512, 4096]  | Integer    | 1024 (MB)       |

#----------------------+------------------------------------------------------------+------------+-----------------+

# delete_exceeds       | Milvus will keep up to ${delete_exceeds} log files per     | Integer    | 10              |

#                      | level. For example, after xxx.log.11 file is generated,    |            |                 |

#                      | the xxx.log.1 file will be deleted.                        |            |                 |

# log_rotate_num       | The maximum number of log files that Milvus keeps for each | Integer    | 0               |

#                      | logging level, num range [0, 1024], 0 means unlimited.     |            |                 |

#----------------------+------------------------------------------------------------+------------+-----------------+

logs:

  trace.enable: true
@@ -223,6 +221,6 @@ logs: 
  error.enable: true

  fatal.enable: true

  path: /var/lib/milvus/logs

  max_log_file_size: 256

  delete_exceeds: 10

  max_log_file_size: 1024

  log_rotate_num: 0

core/conf/server_config.template

+8 −10
Original line number Diff line number Diff line
@@ -68,8 +68,8 @@ db_config: 
# secondary_path       | A semicolon-separated list of secondary directories used   | Path       |                 |

#                      | to save vector data and index data.                        |            |                 |

#----------------------+------------------------------------------------------------+------------+-----------------+

# file_cleanup_timeout | time gap between soft-delete and hard-delete               | Integer    | 10 (s)          |

#                      | range [0, 3600]                                            |            |                 |

# file_cleanup_timeout | The time gap between marking a file as 'deleted' and       | Integer    | 10 (s)          |

#                      | physically deleting this file from disk, range [0, 3600]   |            |                 |

#----------------------+------------------------------------------------------------+------------+-----------------+

storage_config:

  primary_path: @MILVUS_DB_PATH@
@@ -206,14 +206,12 @@ wal_config: 
#----------------------+------------------------------------------------------------+------------+-----------------+

# fatal.enable         | Whether to enable fatal level logging in Milvus.           | Boolean    | true            |

#----------------------+------------------------------------------------------------+------------+-----------------+

# path                 | Location of logs files.                                    | String     |                 |

# path                 | Absolute path to the folder holding the log files.         | String     |                 |

#----------------------+------------------------------------------------------------+------------+-----------------+

# max_log_file_size    | Max size of a single log file. After exceeding this value, | Integer    | 256 (MB)        |

#                      | rename this file to xxx.log.n, means the nth file.         |            |                 |

# max_log_file_size    | The maximum size of each log file, size range [512, 4096]  | Integer    | 1024 (MB)       |

#----------------------+------------------------------------------------------------+------------+-----------------+

# delete_exceeds       | Milvus will keep up to ${delete_exceeds} log files per     | Integer    | 10              |

#                      | level. For example, after xxx.log.11 file is generated,    |            |                 |

#                      | the xxx.log.1 file will be deleted.                        |            |                 |

# log_rotate_num       | The maximum number of log files that Milvus keeps for each | Integer    | 0               |

#                      | logging level, num range [0, 1024], 0 means unlimited.     |            |                 |

#----------------------+------------------------------------------------------------+------------+-----------------+

logs:

  trace.enable: true
@@ -223,6 +221,6 @@ logs: 
  error.enable: true

  fatal.enable: true

  path: @MILVUS_DB_PATH@/logs

  max_log_file_size: 256

  delete_exceeds: 10

  max_log_file_size: 1024

  log_rotate_num: 0

core/src/config/Config.cpp

+70 −48
Original line number Diff line number Diff line
@@ -76,6 +76,8 @@ const char* CONFIG_STORAGE_SECONDARY_PATH = "secondary_path"; 
const char* CONFIG_STORAGE_SECONDARY_PATH_DEFAULT = "";

const char* CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT = "file_cleanup_timeout";

const char* CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_DEFAULT = "10";

const int64_t CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MIN = 0;

const int64_t CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MAX = 3600;

// const char* CONFIG_STORAGE_S3_ENABLE = "s3_enable";

// const char* CONFIG_STORAGE_S3_ENABLE_DEFAULT = "false";

// const char* CONFIG_STORAGE_S3_ADDRESS = "s3_address";
@@ -150,8 +152,8 @@ const char* CONFIG_WAL_RECOVERY_ERROR_IGNORE = "recovery_error_ignore"; 
const char* CONFIG_WAL_RECOVERY_ERROR_IGNORE_DEFAULT = "true";

const char* CONFIG_WAL_BUFFER_SIZE = "buffer_size";

const char* CONFIG_WAL_BUFFER_SIZE_DEFAULT = "256";

const int64_t CONFIG_WAL_BUFFER_SIZE_MAX = 4096;

const int64_t CONFIG_WAL_BUFFER_SIZE_MIN = 64;

const int64_t CONFIG_WAL_BUFFER_SIZE_MAX = 4096;

const char* CONFIG_WAL_WAL_PATH = "wal_path";

const char* CONFIG_WAL_WAL_PATH_DEFAULT = "/tmp/milvus/wal";
@@ -172,13 +174,13 @@ const char* CONFIG_LOGS_FATAL_ENABLE_DEFAULT = "true"; 
const char* CONFIG_LOGS_PATH = "path";

const char* CONFIG_LOGS_PATH_DEFAULT = "/tmp/milvus/logs";

const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE = "max_log_file_size";

const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE_DEFAULT = "256";

const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX = 512;

const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN = 64;

const char* CONFIG_LOGS_DELETE_EXCEEDS = "delete_exceeds";

const char* CONFIG_LOGS_DELETE_EXCEEDS_DEFAULT = "10";

const int64_t CONFIG_LOGS_DELETE_EXCEEDS_MAX = 4096;

const int64_t CONFIG_LOGS_DELETE_EXCEEDS_MIN = 1;

const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE_DEFAULT = "1024";

const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN = 512;

const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX = 4096;

const char* CONFIG_LOGS_LOG_ROTATE_NUM = "log_rotate_num";

const char* CONFIG_LOGS_LOG_ROTATE_NUM_DEFAULT = "0";

const int64_t CONFIG_LOGS_LOG_ROTATE_NUM_MIN = 0;

const int64_t CONFIG_LOGS_LOG_ROTATE_NUM_MAX = 1024;



constexpr int64_t GB = 1UL << 30;

constexpr int32_t PORT_NUMBER_MIN = 1024;
@@ -416,8 +418,8 @@ Config::ValidateConfig() { 
    int64_t logs_max_log_file_size;

    STATUS_CHECK(GetLogsMaxLogFileSize(logs_max_log_file_size));



    int64_t delete_exceeds;

    STATUS_CHECK(GetLogsDeleteExceeds(delete_exceeds));

    int64_t logs_log_rotate_num;

    STATUS_CHECK(GetLogsLogRotateNum(logs_log_rotate_num));



    return Status::OK();

}
@@ -493,7 +495,7 @@ Config::ResetDefaultConfig() { 
    STATUS_CHECK(SetLogsFatalEnable(CONFIG_LOGS_FATAL_ENABLE_DEFAULT));

    STATUS_CHECK(SetLogsPath(CONFIG_LOGS_PATH_DEFAULT));

    STATUS_CHECK(SetLogsMaxLogFileSize(CONFIG_LOGS_MAX_LOG_FILE_SIZE_DEFAULT));

    STATUS_CHECK(SetLogsDeleteExceeds(CONFIG_LOGS_DELETE_EXCEEDS_DEFAULT));

    STATUS_CHECK(SetLogsLogRotateNum(CONFIG_LOGS_LOG_ROTATE_NUM_DEFAULT));



    return Status::OK();

}
@@ -635,6 +637,28 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key 
        } else {

            status = Status(SERVER_UNEXPECTED_ERROR, invalid_node_str);

        }

    } else if (parent_key == CONFIG_LOGS) {

        if (child_key == CONFIG_LOGS_TRACE_ENABLE) {

            status = SetLogsTraceEnable(value);

        } else if (child_key == CONFIG_LOGS_DEBUG_ENABLE) {

            status = SetLogsDebugEnable(value);

        } else if (child_key == CONFIG_LOGS_INFO_ENABLE) {

            status = SetLogsInfoEnable(value);

        } else if (child_key == CONFIG_LOGS_WARNING_ENABLE) {

            status = SetLogsWarningEnable(value);

        } else if (child_key == CONFIG_LOGS_ERROR_ENABLE) {

            status = SetLogsErrorEnable(value);

        } else if (child_key == CONFIG_LOGS_FATAL_ENABLE) {

            status = SetLogsFatalEnable(value);

        } else if (child_key == CONFIG_LOGS_PATH) {

            status = SetLogsPath(value);

        } else if (child_key == CONFIG_LOGS_MAX_LOG_FILE_SIZE) {

            status = SetLogsMaxLogFileSize(value);

        } else if (child_key == CONFIG_LOGS_LOG_ROTATE_NUM) {

            status = SetLogsLogRotateNum(value);

        } else {

            status = Status(SERVER_UNEXPECTED_ERROR, invalid_node_str);

        }

    }



    if (status.ok()) {
@@ -1086,23 +1110,18 @@ Config::CheckStorageConfigSecondaryPath(const std::string& value) { 


Status

Config::CheckStorageConfigFileCleanupTimeout(const std::string& value) {

    auto status = Status::OK();



    if (value.empty()) {

        return status;

    }



    if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {

        std::string msg = "Invalid file cleanup timeout: " + value +

        std::string msg = "Invalid file_cleanup_timeout: " + value +

                          ". Possible reason: storage_config.file_cleanup_timeout is not a positive integer.";

        return Status(SERVER_INVALID_ARGUMENT, msg);

    } else {

        const int64_t min = 0, max = 3600;

        int64_t file_cleanup_timeout = std::stoll(value);

        if (file_cleanup_timeout < min || file_cleanup_timeout > max) {

            std::string msg = "Invalid file cleanup timeout: " + value +

        if (file_cleanup_timeout < CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MIN ||

            file_cleanup_timeout > CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MAX) {

            std::string msg = "Invalid file_cleanup_timeout: " + value +

                              ". Possible reason: storage_config.file_cleanup_timeout is not in range [" +

                              std::to_string(min) + ", " + std::to_string(max) + "].";

                              std::to_string(CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MIN) + ", " +

                              std::to_string(CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MIN) + "].";

            return Status(SERVER_INVALID_ARGUMENT, msg);

        }

    }
@@ -1673,20 +1692,39 @@ Config::CheckLogsMaxLogFileSize(const std::string& value) { 
        std::string msg = "Invalid max_log_file_size: " + value +

                          ". Possible reason: logs.max_log_file_size is not a positive integer.";

        return Status(SERVER_INVALID_ARGUMENT, msg);

    } else {

        int64_t max_log_file_size = std::stoll(value);

        if (max_log_file_size < CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN ||

            max_log_file_size > CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX) {

            std::string msg = "Invalid max_log_file_size: " + value +

                              ". Possible reason: logs.max_log_file_size is not in range [" +

                              std::to_string(CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN) + ", " +

                              std::to_string(CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX) + "].";

            return Status(SERVER_INVALID_ARGUMENT, msg);

        }

    }

    return Status::OK();

}



Status

Config::CheckLogsDeleteExceeds(const std::string& value) {

Config::CheckLogsLogRotateNum(const std::string& value) {

    auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok();

    fiu_do_on("check_logs_delete_exceeds_fail", exist_error = true);

    fiu_do_on("check_logs_log_rotate_num_fail", exist_error = true);



    if (exist_error) {

        std::string msg = "Invalid max_log_file_size: " + value +

                          ". Possible reason: logs.max_log_file_size is not a positive integer.";

        std::string msg =

            "Invalid log_rotate_num: " + value + ". Possible reason: logs.log_rotate_num is not a positive integer.";

        return Status(SERVER_INVALID_ARGUMENT, msg);

    } else {

        int64_t log_rotate_num = std::stoll(value);

        if (log_rotate_num < CONFIG_LOGS_LOG_ROTATE_NUM_MIN || log_rotate_num > CONFIG_LOGS_LOG_ROTATE_NUM_MAX) {

            std::string msg = "Invalid log_rotate_num: " + value +

                              ". Possible reason: logs.log_rotate_num is not in range [" +

                              std::to_string(CONFIG_LOGS_LOG_ROTATE_NUM_MIN) + ", " +

                              std::to_string(CONFIG_LOGS_LOG_ROTATE_NUM_MAX) + "].";

            return Status(SERVER_INVALID_ARGUMENT, msg);

        }

    }

    return Status::OK();

}
@@ -2221,30 +2259,14 @@ Config::GetLogsMaxLogFileSize(int64_t& value) { 
    std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_MAX_LOG_FILE_SIZE, CONFIG_LOGS_MAX_LOG_FILE_SIZE_DEFAULT);

    STATUS_CHECK(CheckLogsMaxLogFileSize(str));

    value = std::stoll(str);

    if (value == 0) {

        // OFF

    } else if (value > CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX) {

        value = CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX;

    } else if (value < CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN) {

        value = CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN;

    }



    return Status::OK();

}



Status

Config::GetLogsDeleteExceeds(int64_t& value) {

    std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_DELETE_EXCEEDS, CONFIG_LOGS_DELETE_EXCEEDS_DEFAULT);

    STATUS_CHECK(CheckLogsDeleteExceeds(str));

Config::GetLogsLogRotateNum(int64_t& value) {

    std::string str = GetConfigStr(CONFIG_LOGS, CONFIG_LOGS_LOG_ROTATE_NUM, CONFIG_LOGS_LOG_ROTATE_NUM_DEFAULT);

    STATUS_CHECK(CheckLogsLogRotateNum(str));

    value = std::stoll(str);

    if (value == 0) {

        // OFF

    } else if (value > CONFIG_LOGS_DELETE_EXCEEDS_MAX) {

        value = CONFIG_LOGS_DELETE_EXCEEDS_MAX;

    } else if (value < CONFIG_LOGS_DELETE_EXCEEDS_MIN) {

        value = CONFIG_LOGS_DELETE_EXCEEDS_MIN;

    }



    return Status::OK();

}
@@ -2578,9 +2600,9 @@ Config::SetLogsMaxLogFileSize(const std::string& value) { 
}



Status

Config::SetLogsDeleteExceeds(const std::string& value) {

    STATUS_CHECK(CheckLogsDeleteExceeds(value));

    return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_DELETE_EXCEEDS, value);

Config::SetLogsLogRotateNum(const std::string& value) {

    STATUS_CHECK(CheckLogsLogRotateNum(value));

    return SetConfigValueInMem(CONFIG_LOGS, CONFIG_LOGS_LOG_ROTATE_NUM, value);

}



}  // namespace server

core/src/config/Config.h

+11 −9
Original line number Diff line number Diff line
@@ -63,6 +63,8 @@ extern const char* CONFIG_STORAGE_PRIMARY_PATH_DEFAULT; 
extern const char* CONFIG_STORAGE_SECONDARY_PATH;

extern const char* CONFIG_STORAGE_SECONDARY_PATH_DEFAULT;

extern const char* CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT;

extern const int64_t CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MIN;

extern const int64_t CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MAX;

// extern const char* CONFIG_STORAGE_S3_ENABLE;

// extern const char* CONFIG_STORAGE_S3_ENABLE_DEFAULT;

// extern const char* CONFIG_STORAGE_S3_ADDRESS;
@@ -133,8 +135,8 @@ extern const char* CONFIG_WAL_RECOVERY_ERROR_IGNORE; 
extern const char* CONFIG_WAL_RECOVERY_ERROR_IGNORE_DEFAULT;

extern const char* CONFIG_WAL_BUFFER_SIZE;

extern const char* CONFIG_WAL_BUFFER_SIZE_DEFAULT;

extern const int64_t CONFIG_WAL_BUFFER_SIZE_MAX;

extern const int64_t CONFIG_WAL_BUFFER_SIZE_MIN;

extern const int64_t CONFIG_WAL_BUFFER_SIZE_MAX;

extern const char* CONFIG_WAL_WAL_PATH;

extern const char* CONFIG_WAL_WAL_PATH_DEFAULT;
@@ -155,12 +157,12 @@ extern const char* CONFIG_LOGS_FATAL_ENABLE_DEFAULT; 
extern const char* CONFIG_LOGS_PATH;

extern const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE;

extern const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE_DEFAULT;

extern const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX;

extern const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN;

extern const char* CONFIG_LOGS_DELETE_EXCEEDS;

extern const char* CONFIG_LOGS_DELETE_EXCEEDS_DEFAULT;

extern const int64_t CONFIG_LOGS_DELETE_EXCEEDS_MAX;

extern const int64_t CONFIG_LOGS_DELETE_EXCEEDS_MIN;

extern const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX;

extern const char* CONFIG_LOGS_LOG_ROTATE_NUM;

extern const char* CONFIG_LOGS_LOG_ROTATE_NUM_DEFAULT;

extern const int64_t CONFIG_LOGS_LOG_ROTATE_NUM_MIN;

extern const int64_t CONFIG_LOGS_LOG_ROTATE_NUM_MAX;



class Config {

 private:
@@ -334,7 +336,7 @@ class Config { 
    Status

    CheckLogsMaxLogFileSize(const std::string& value);

    Status

    CheckLogsDeleteExceeds(const std::string& value);

    CheckLogsLogRotateNum(const std::string& value);



    std::string

    GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = "");
@@ -469,7 +471,7 @@ class Config { 
    Status

    GetLogsMaxLogFileSize(int64_t& value);

    Status

    GetLogsDeleteExceeds(int64_t& value);

    GetLogsLogRotateNum(int64_t& value);



    Status

    GetServerRestartRequired(bool& required);
@@ -595,7 +597,7 @@ class Config { 
    Status

    SetLogsMaxLogFileSize(const std::string& value);

    Status

    SetLogsDeleteExceeds(const std::string& value);

    SetLogsLogRotateNum(const std::string& value);



 private:

    bool restart_required_ = false;