Unverified Commit 2f6920f8 authored by 李盛俊's avatar 李盛俊 Committed by GitHub
Browse files

conf build_index_threshold (#3284) 



* conf build_index_threshold

Signed-off-by: default avatarshengjun.li <shengjun.li@zilliz.com>

* add member

Signed-off-by: default avatarshengjun.li <shengjun.li@zilliz.com>

* add log

Signed-off-by: default avatarshengjun.li <shengjun.li@zilliz.com>
parent 89ca954e

core/src/config/ConfigMgr.cpp

+3 −0
Original line number Diff line number Diff line
@@ -164,6 +164,9 @@ ConfigMgr::ConfigMgr() { 


        /* invisible */

        /* engine */

        {"engine.build_index_threshold",

         CreateIntegerConfig("engine.build_index_threshold", false, 0, std::numeric_limits<int64_t>::max(),

                             &config.engine.build_index_threshold.value, 4096, nullptr, nullptr)},

        {"engine.search_combine_nq",

         CreateIntegerConfig("engine.search_combine_nq", true, 0, std::numeric_limits<int64_t>::max(),

                             &config.engine.search_combine_nq.value, 64, nullptr, nullptr)},

core/src/config/ServerConfig.h

+1 −0
Original line number Diff line number Diff line
@@ -112,6 +112,7 @@ struct ServerConfig { 
    } metric;



    struct Engine {

        Integer build_index_threshold{4096};

        Integer search_combine_nq{0};

        Integer use_blas_threshold{0};

        Integer omp_thread_num{0};

core/src/db/Constants.h

+0 −1
Original line number Diff line number Diff line
@@ -23,7 +23,6 @@ constexpr int64_t TB = 1LL << 40; 


constexpr int64_t MAX_TABLE_FILE_MEM = 128 * MB;



constexpr int64_t BUILD_INDEX_THRESHOLD = 4096;  // row count threshold when building index

constexpr int64_t MAX_NAME_LENGTH = 255;

constexpr int64_t MAX_DIMENSION = 32768;

constexpr int32_t MAX_SEGMENT_ROW_COUNT = 4 * 1024 * 1024;

core/src/db/SnapshotHandlers.cpp

+6 −1
Original line number Diff line number Diff line
@@ -10,6 +10,8 @@ 
// or implied. See the License for the specific language governing permissions and limitations under the License.



#include "db/SnapshotHandlers.h"



#include "config/ServerConfig.h"

#include "db/SnapshotUtils.h"

#include "db/SnapshotVisitor.h"

#include "db/Types.h"
@@ -40,11 +42,14 @@ SegmentsToSearchCollector::Handle(const snapshot::SegmentCommitPtr& segment_comm 
SegmentsToIndexCollector::SegmentsToIndexCollector(snapshot::ScopedSnapshotT ss, const std::string& field_name,

                                                   snapshot::IDS_TYPE& segment_ids)

    : BaseT(ss), field_name_(field_name), segment_ids_(segment_ids) {

    build_index_threshold_ = config.engine.build_index_threshold();

    LOG_ENGINE_DEBUG_ << "Build index threshold is " << build_index_threshold_;

}



Status

SegmentsToIndexCollector::Handle(const snapshot::SegmentCommitPtr& segment_commit) {

    if (segment_commit->GetRowCount() < engine::BUILD_INDEX_THRESHOLD) {

    if (segment_commit->GetRowCount() < build_index_threshold_) {

        LOG_ENGINE_DEBUG_ << "Segment is too small, not to build index, row count " << segment_commit->GetRowCount();

        return Status::OK();

    }

core/src/db/SnapshotHandlers.h

+1 −0
Original line number Diff line number Diff line
@@ -47,6 +47,7 @@ struct SegmentsToIndexCollector : public snapshot::SegmentCommitIterator { 


    std::string field_name_;

    snapshot::IDS_TYPE& segment_ids_;

    int64_t build_index_threshold_;

};



///////////////////////////////////////////////////////////////////////////////