Unverified Commit 0fcdc31c authored by BossZou's avatar BossZou Committed by GitHub
Browse files

Update resources name in HTTP module (#1545) 



* set preload_table correctly (fix #1524)

Signed-off-by: default avatarYhz <yinghao.zou@zilliz.com>

* update res name in http module (fix #1544)

Signed-off-by: default avatarYhz <yinghao.zou@zilliz.com>

* add bool convertor function

Signed-off-by: default avatarYhz <yinghao.zou@zilliz.com>

* fix test fail

Signed-off-by: default avatarYhz <yinghao.zou@zilliz.com>

* optimize update config method & add boolean conversion comments

Signed-off-by: default avatarYhz <yinghao.zou@zilliz.com>

* remove comments

Signed-off-by: default avatarYhz <yinghao.zou@zilliz.com>
parent 0fd9bf8e

CHANGELOG.md

+1 −0
Original line number Diff line number Diff line
@@ -90,6 +90,7 @@ Please mark all change in change log and use the issue from GitHub 
-   \#1448 General proto api for NNS libraries 

-   \#1480 Add return code for AVX512 selection

-   \#1524 Update config "preload_table" description

-   \#1544 Update resources name in HTTP module



## Task

-   \#1327 Exclude third-party code from codebeat

core/src/config/handler/ConfigHandler.h

+0 −1
Original line number Diff line number Diff line
@@ -14,7 +14,6 @@ 
#include <string>



#include "server/Config.h"

#include "utils/Log.h"



namespace milvus {

namespace server {

core/src/server/Config.cpp

+10 −5
Original line number Diff line number Diff line
@@ -549,9 +549,14 @@ Config::UpdateFileConfigFromMem(const std::string& parent_key, const std::string 
    // convert value string to standard string stored in yaml file

    std::string value_str;

    if (child_key == CONFIG_CACHE_CACHE_INSERT_DATA || child_key == CONFIG_STORAGE_S3_ENABLE ||

        child_key == CONFIG_METRIC_ENABLE_MONITOR || child_key == CONFIG_GPU_RESOURCE_ENABLE) {

        value_str =

            (value == "True" || value == "true" || value == "On" || value == "on" || value == "1") ? "true" : "false";

        child_key == CONFIG_METRIC_ENABLE_MONITOR || child_key == CONFIG_GPU_RESOURCE_ENABLE ||

        child_key == CONFIG_WAL_ENABLE || child_key == CONFIG_WAL_RECOVERY_ERROR_IGNORE) {

        bool ok = false;

        status = StringHelpFunctions::ConvertToBoolean(value, ok);

        if (!status.ok()) {

            return status;

        }

        value_str = ok ? "true" : "false";

    } else if (child_key == CONFIG_GPU_RESOURCE_SEARCH_RESOURCES ||

               child_key == CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES) {

        std::vector<std::string> vec;
@@ -593,7 +598,6 @@ Config::UpdateFileConfigFromMem(const std::string& parent_key, const std::string 
    }



    // values of gpu resources are sequences, need to remove old here

    std::regex reg("\\S*");

    if (child_key == CONFIG_GPU_RESOURCE_SEARCH_RESOURCES || child_key == CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES) {

        while (getline(conf_fin, line)) {

            if (line.find("- gpu") != std::string::npos)
@@ -1853,7 +1857,8 @@ Config::SetDBConfigBackendUrl(const std::string& value) { 
Status

Config::SetDBConfigPreloadTable(const std::string& value) {

    CONFIG_CHECK(CheckDBConfigPreloadTable(value));

    return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRELOAD_TABLE, value);

    std::string cor_value = value == "*" ? "\'*\'" : value;

    return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRELOAD_TABLE, cor_value);

}



Status

core/src/server/web_impl/controller/WebController.hpp

+55 −60
Original line number Diff line number Diff line
@@ -208,14 +208,14 @@ class WebController : public oatpp::web::server::api::ApiController { 


    ADD_CORS(TablesOptions)



    ENDPOINT("OPTIONS", "/tables", TablesOptions) {

    ENDPOINT("OPTIONS", "/collections", TablesOptions) {

        return createResponse(Status::CODE_204, "No Content");

    }



    ADD_CORS(CreateTable)



    ENDPOINT("POST", "/tables", CreateTable, BODY_DTO(TableRequestDto::ObjectWrapper, body)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/tables\'");

    ENDPOINT("POST", "/collections", CreateTable, BODY_DTO(TableRequestDto::ObjectWrapper, body)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/collections\'");

        tr.RecordSection("Received request.");



        WebRequestHandler handler = WebRequestHandler();
@@ -238,8 +238,8 @@ class WebController : public oatpp::web::server::api::ApiController { 


    ADD_CORS(ShowTables)



    ENDPOINT("GET", "/tables", ShowTables, QUERIES(const QueryParams&, query_params)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/tables\'");

    ENDPOINT("GET", "/collections", ShowTables, QUERIES(const QueryParams&, query_params)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/collections\'");

        tr.RecordSection("Received request.");



        WebRequestHandler handler = WebRequestHandler();
@@ -265,21 +265,21 @@ class WebController : public oatpp::web::server::api::ApiController { 


    ADD_CORS(TableOptions)



    ENDPOINT("OPTIONS", "/tables/{table_name}", TableOptions) {

    ENDPOINT("OPTIONS", "/collections/{collection_name}", TableOptions) {

        return createResponse(Status::CODE_204, "No Content");

    }



    ADD_CORS(GetTable)



    ENDPOINT("GET", "/tables/{table_name}", GetTable,

        PATH(String, table_name), QUERIES(const QueryParams&, query_params)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/tables/" + table_name->std_str() + "\'");

    ENDPOINT("GET", "/collections/{collection_name}", GetTable,

        PATH(String, collection_name), QUERIES(const QueryParams&, query_params)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/collections/" + collection_name->std_str() + "\'");

        tr.RecordSection("Received request.");



        WebRequestHandler handler = WebRequestHandler();



        String response_str;

        auto status_dto = handler.GetTable(table_name, query_params, response_str);

        auto status_dto = handler.GetTable(collection_name, query_params, response_str);



        std::shared_ptr<OutgoingResponse> response;

        switch (status_dto->code->getValue()) {
@@ -302,14 +302,14 @@ class WebController : public oatpp::web::server::api::ApiController { 


    ADD_CORS(DropTable)



    ENDPOINT("DELETE", "/tables/{table_name}", DropTable, PATH(String, table_name)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "DELETE \'/tables/" + table_name->std_str() + "\'");

    ENDPOINT("DELETE", "/collections/{collection_name}", DropTable, PATH(String, collection_name)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "DELETE \'/collections/" + collection_name->std_str() + "\'");

        tr.RecordSection("Received request.");



        WebRequestHandler handler = WebRequestHandler();



        std::shared_ptr<OutgoingResponse> response;

        auto status_dto = handler.DropTable(table_name);

        auto status_dto = handler.DropTable(collection_name);

        switch (status_dto->code->getValue()) {

            case StatusCode::SUCCESS:

                response = createDtoResponse(Status::CODE_204, status_dto);
@@ -330,21 +330,21 @@ class WebController : public oatpp::web::server::api::ApiController { 


    ADD_CORS(IndexOptions)



    ENDPOINT("OPTIONS", "/tables/{table_name}/indexes", IndexOptions) {

    ENDPOINT("OPTIONS", "/collections/{collection_name}/indexes", IndexOptions) {

        return createResponse(Status::CODE_204, "No Content");

    }



    ADD_CORS(CreateIndex)



    ENDPOINT("POST", "/tables/{table_name}/indexes", CreateIndex,

             PATH(String, table_name), BODY_STRING(String, body)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/tables/" + table_name->std_str() + "/indexes\'");

    ENDPOINT("POST", "/tables/{collection_name}/indexes", CreateIndex,

             PATH(String, collection_name), BODY_STRING(String, body)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/tables/" + collection_name->std_str() + "/indexes\'");

        tr.RecordSection("Received request.");



        auto handler = WebRequestHandler();



        std::shared_ptr<OutgoingResponse> response;

        auto status_dto = handler.CreateIndex(table_name, body);

        auto status_dto = handler.CreateIndex(collection_name, body);

        switch (status_dto->code->getValue()) {

            case StatusCode::SUCCESS:

                response = createDtoResponse(Status::CODE_201, status_dto);
@@ -365,14 +365,14 @@ class WebController : public oatpp::web::server::api::ApiController { 


    ADD_CORS(GetIndex)



    ENDPOINT("GET", "/tables/{table_name}/indexes", GetIndex, PATH(String, table_name)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/tables/" + table_name->std_str() + "/indexes\'");

    ENDPOINT("GET", "/collections/{collection_name}/indexes", GetIndex, PATH(String, collection_name)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/collections/" + collection_name->std_str() + "/indexes\'");

        tr.RecordSection("Received request.");



        auto handler = WebRequestHandler();



        OString result;

        auto status_dto = handler.GetIndex(table_name, result);

        auto status_dto = handler.GetIndex(collection_name, result);



        std::shared_ptr<OutgoingResponse> response;

        switch (status_dto->code->getValue()) {
@@ -395,14 +395,14 @@ class WebController : public oatpp::web::server::api::ApiController { 


    ADD_CORS(DropIndex)



    ENDPOINT("DELETE", "/tables/{table_name}/indexes", DropIndex, PATH(String, table_name)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "DELETE \'/tables/" + table_name->std_str() + "/indexes\'");

    ENDPOINT("DELETE", "/collections/{collection_name}/indexes", DropIndex, PATH(String, collection_name)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "DELETE \'/collections/" + collection_name->std_str() + "/indexes\'");

        tr.RecordSection("Received request.");



        auto handler = WebRequestHandler();



        std::shared_ptr<OutgoingResponse> response;

        auto status_dto = handler.DropIndex(table_name);

        auto status_dto = handler.DropIndex(collection_name);

        switch (status_dto->code->getValue()) {

            case StatusCode::SUCCESS:

                response = createDtoResponse(Status::CODE_204, status_dto);
@@ -423,21 +423,21 @@ class WebController : public oatpp::web::server::api::ApiController { 


    ADD_CORS(PartitionsOptions)



    ENDPOINT("OPTIONS", "/tables/{table_name}/partitions", PartitionsOptions) {

    ENDPOINT("OPTIONS", "/collections/{collection_name}/partitions", PartitionsOptions) {

        return createResponse(Status::CODE_204, "No Content");

    }



    ADD_CORS(CreatePartition)



    ENDPOINT("POST", "/tables/{table_name}/partitions",

             CreatePartition, PATH(String, table_name), BODY_DTO(PartitionRequestDto::ObjectWrapper, body)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/tables/" + table_name->std_str() + "/partitions\'");

    ENDPOINT("POST", "/collections/{collection_name}/partitions",

             CreatePartition, PATH(String, collection_name), BODY_DTO(PartitionRequestDto::ObjectWrapper, body)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/collections/" + collection_name->std_str() + "/partitions\'");

        tr.RecordSection("Received request.");



        auto handler = WebRequestHandler();



        std::shared_ptr<OutgoingResponse> response;

        auto status_dto = handler.CreatePartition(table_name, body);

        auto status_dto = handler.CreatePartition(collection_name, body);

        switch (status_dto->code->getValue()) {

            case StatusCode::SUCCESS:

                response = createDtoResponse(Status::CODE_201, status_dto);
@@ -457,9 +457,9 @@ class WebController : public oatpp::web::server::api::ApiController { 


    ADD_CORS(ShowPartitions)



    ENDPOINT("GET", "/tables/{table_name}/partitions", ShowPartitions,

             PATH(String, table_name), QUERIES(const QueryParams&, query_params)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/tables/" + table_name->std_str() + "/partitions\'");

    ENDPOINT("GET", "/collections/{collection_name}/partitions", ShowPartitions,

             PATH(String, collection_name), QUERIES(const QueryParams&, query_params)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/collections/" + collection_name->std_str() + "/partitions\'");

        tr.RecordSection("Received request.");



        auto offset = query_params.get("offset");
@@ -469,7 +469,7 @@ class WebController : public oatpp::web::server::api::ApiController { 
        auto handler = WebRequestHandler();



        std::shared_ptr<OutgoingResponse> response;

        auto status_dto = handler.ShowPartitions(table_name, query_params, partition_list_dto);

        auto status_dto = handler.ShowPartitions(collection_name, query_params, partition_list_dto);

        switch (status_dto->code->getValue()) {

            case StatusCode::SUCCESS:

                response = createDtoResponse(Status::CODE_200, partition_list_dto);
@@ -488,16 +488,16 @@ class WebController : public oatpp::web::server::api::ApiController { 


    ADD_CORS(DropPartition)



    ENDPOINT("DELETE", "/tables/{table_name}/partitions", DropPartition,

             PATH(String, table_name), BODY_STRING(String, body)) {

    ENDPOINT("DELETE", "/collections/{collection_name}/partitions", DropPartition,

             PATH(String, collection_name), BODY_STRING(String, body)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) +

                        "DELETE \'/tables/" + table_name->std_str() + "/partitions\'");

                        "DELETE \'/collections/" + collection_name->std_str() + "/partitions\'");

        tr.RecordSection("Received request.");



        auto handler = WebRequestHandler();



        std::shared_ptr<OutgoingResponse> response;

        auto status_dto = handler.DropPartition(table_name, body);

        auto status_dto = handler.DropPartition(collection_name, body);

        switch (status_dto->code->getValue()) {

            case StatusCode::SUCCESS:

                response = createDtoResponse(Status::CODE_204, status_dto);
@@ -517,14 +517,14 @@ class WebController : public oatpp::web::server::api::ApiController { 


    ADD_CORS(ShowSegments)



    ENDPOINT("GET", "/tables/{table_name}/segments", ShowSegments,

             PATH(String, table_name), QUERIES(const QueryParams&, query_params)) {

    ENDPOINT("GET", "/collections/{collection_name}/segments", ShowSegments,

             PATH(String, collection_name), QUERIES(const QueryParams&, query_params)) {

        auto offset = query_params.get("offset");

        auto page_size = query_params.get("page_size");



        auto handler = WebRequestHandler();

        String response;

        auto status_dto = handler.ShowSegments(table_name, query_params, response);

        auto status_dto = handler.ShowSegments(collection_name, query_params, response);



        switch (status_dto->code->getValue()) {

            case StatusCode::SUCCESS:
@@ -541,14 +541,14 @@ class WebController : public oatpp::web::server::api::ApiController { 
     *

     * GetSegmentVector

     */

    ENDPOINT("GET", "/tables/{table_name}/segments/{segment_name}/{info}", GetSegmentInfo,

             PATH(String, table_name), PATH(String, segment_name), PATH(String, info), QUERIES(const QueryParams&, query_params)) {

    ENDPOINT("GET", "/collections/{collection_name}/segments/{segment_name}/{info}", GetSegmentInfo,

             PATH(String, collection_name), PATH(String, segment_name), PATH(String, info), QUERIES(const QueryParams&, query_params)) {

        auto offset = query_params.get("offset");

        auto page_size = query_params.get("page_size");



        auto handler = WebRequestHandler();

        String response;

        auto status_dto = handler.GetSegmentInfo(table_name, segment_name, info, query_params, response);

        auto status_dto = handler.GetSegmentInfo(collection_name, segment_name, info, query_params, response);



        switch (status_dto->code->getValue()) {

            case StatusCode::SUCCESS:
@@ -562,7 +562,7 @@ class WebController : public oatpp::web::server::api::ApiController { 


    ADD_CORS(VectorsOptions)



    ENDPOINT("OPTIONS", "/tables/{table_name}/vectors", VectorsOptions) {

    ENDPOINT("OPTIONS", "/collections/{collection_name}/vectors", VectorsOptions) {

        return createResponse(Status::CODE_204, "No Content");

    }
@@ -571,11 +571,11 @@ class WebController : public oatpp::web::server::api::ApiController { 
     *

     * GetVectorByID ?id=

     */

    ENDPOINT("GET", "/tables/{table_name}/vectors", GetVectors,

             PATH(String, table_name), QUERIES(const QueryParams&, query_params)) {

    ENDPOINT("GET", "/collections/{collection_name}/vectors", GetVectors,

             PATH(String, collection_name), QUERIES(const QueryParams&, query_params)) {

        auto handler = WebRequestHandler();

        String response;

        auto status_dto = handler.GetVector(table_name, query_params, response);

        auto status_dto = handler.GetVector(collection_name, query_params, response);



        switch (status_dto->code->getValue()) {

            case StatusCode::SUCCESS:
@@ -589,16 +589,16 @@ class WebController : public oatpp::web::server::api::ApiController { 


    ADD_CORS(Insert)



    ENDPOINT("POST", "/tables/{table_name}/vectors", Insert,

             PATH(String, table_name), BODY_STRING(String, body)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/tables/" + table_name->std_str() + "/vectors\'");

    ENDPOINT("POST", "/collections/{collection_name}/vectors", Insert,

             PATH(String, collection_name), BODY_STRING(String, body)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/collections/" + collection_name->std_str() + "/vectors\'");

        tr.RecordSection("Received request.");



        auto ids_dto = VectorIdsDto::createShared();

        WebRequestHandler handler = WebRequestHandler();



        std::shared_ptr<OutgoingResponse> response;

        auto status_dto = handler.Insert(table_name, body, ids_dto);

        auto status_dto = handler.Insert(collection_name, body, ids_dto);

        switch (status_dto->code->getValue()) {

            case StatusCode::SUCCESS:

                response = createDtoResponse(Status::CODE_201, ids_dto);
@@ -621,16 +621,16 @@ class WebController : public oatpp::web::server::api::ApiController { 
     * Search

     * Delete by ID

     * */

    ENDPOINT("PUT", "/tables/{table_name}/vectors", VectorsOp,

             PATH(String, table_name), BODY_STRING(String, body)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "PUT \'/tables/" + table_name->std_str() + "/vectors\'");

    ENDPOINT("PUT", "/collections/{collection_name}/vectors", VectorsOp,

             PATH(String, collection_name), BODY_STRING(String, body)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "PUT \'/collections/" + collection_name->std_str() + "/vectors\'");

        tr.RecordSection("Received request.");



        WebRequestHandler handler = WebRequestHandler();



        OString result;

        std::shared_ptr<OutgoingResponse> response;

        auto status_dto = handler.VectorsOp(table_name, body, result);

        auto status_dto = handler.VectorsOp(collection_name, body, result);

        switch (status_dto->code->getValue()) {

            case StatusCode::SUCCESS:

                response = createResponse(Status::CODE_200, result);
@@ -674,11 +674,6 @@ class WebController : public oatpp::web::server::api::ApiController { 


    ADD_CORS(SystemOp)



    /**

    * Load

    * Compact

    * Flush

    */

    ENDPOINT("PUT", "/system/{Op}", SystemOp, PATH(String, Op), BODY_STRING(String, body_str)) {

        TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "PUT \'/system/" + Op->std_str() + "\'");

        tr.RecordSection("Received request.");

core/src/server/web_impl/dto/TableDto.hpp

+4 −4
Original line number Diff line number Diff line
@@ -25,7 +25,7 @@ namespace web { 
class TableRequestDto : public oatpp::data::mapping::type::Object {

 DTO_INIT(TableRequestDto, Object)



    DTO_FIELD(String, table_name, "table_name");

    DTO_FIELD(String, collection_name, "collection_name");

    DTO_FIELD(Int64, dimension, "dimension");

    DTO_FIELD(Int64, index_file_size, "index_file_size") = VALUE_TABLE_INDEX_FILE_SIZE_DEFAULT;

    DTO_FIELD(String, metric_type, "metric_type") = VALUE_TABLE_METRIC_TYPE_DEFAULT;
@@ -34,7 +34,7 @@ class TableRequestDto : public oatpp::data::mapping::type::Object { 
class TableFieldsDto : public oatpp::data::mapping::type::Object {

 DTO_INIT(TableFieldsDto, Object)



    DTO_FIELD(String, table_name);

    DTO_FIELD(String, collection_name);

    DTO_FIELD(Int64, dimension);

    DTO_FIELD(Int64, index_file_size);

    DTO_FIELD(String, metric_type);
@@ -46,13 +46,13 @@ class TableFieldsDto : public oatpp::data::mapping::type::Object { 
class TableListDto : public OObject {

 DTO_INIT(TableListDto, Object)



    DTO_FIELD(List<String>::ObjectWrapper, table_names);

    DTO_FIELD(List<String>::ObjectWrapper, collection_names);

};



class TableListFieldsDto : public OObject {

    DTO_INIT(TableListFieldsDto, Object)



    DTO_FIELD(List<TableFieldsDto::ObjectWrapper>::ObjectWrapper, tables);

    DTO_FIELD(List<TableFieldsDto::ObjectWrapper>::ObjectWrapper, collections);

    DTO_FIELD(Int64, count) = 0;

};