Unverified Commit 5ec1237a authored by Jin Hai's avatar Jin Hai Committed by GitHub
Browse files

Merge pull request #593 from yhmo/preload 

#274 logger the time cost during preloading data
parents 0468b907 2ae57716

CHANGELOG.md

+1 −0
Original line number Diff line number Diff line
@@ -51,6 +51,7 @@ Please mark all change in change log and use the ticket from JIRA. 
- \#255 - Add ivfsq8 test report detailed version

- \#260 - C++ SDK README

- \#266 - Rpc request source code refactor

- \#274 - Logger the time cost during preloading data

- \#275 - Rename C++ SDK IndexType

- \#284 - Change C++ SDK to shared library

- \#306 - Use int64 for all config integer

core/src/db/DBImpl.cpp

+9 −3
Original line number Diff line number Diff line
@@ -182,7 +182,7 @@ DBImpl::PreloadTable(const std::string& table_id) { 
        return SHUTDOWN_ERROR;

    }



    // get all table files from parent table

    // step 1: get all table files from parent table

    meta::DatesT dates;

    std::vector<size_t> ids;

    meta::TableFilesSchema files_array;
@@ -191,7 +191,7 @@ DBImpl::PreloadTable(const std::string& table_id) { 
        return status;

    }



    // get files from partition tables

    // step 2: get files from partition tables

    std::vector<meta::TableSchema> partiton_array;

    status = meta_ptr_->ShowPartitions(table_id, partiton_array);

    for (auto& schema : partiton_array) {
@@ -203,6 +203,10 @@ DBImpl::PreloadTable(const std::string& table_id) { 
    int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage();

    int64_t available_size = cache_total - cache_usage;



    // step 3: load file one by one

    ENGINE_LOG_DEBUG << "Begin pre-load table:" + table_id + ", totally " << files_array.size()

                     << " files need to be pre-loaded";

    TimeRecorderAuto rc("Pre-load table:" + table_id);

    for (auto& file : files_array) {

        ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_,

                                                         (MetricType)file.metric_type_, file.nlist_);
@@ -213,10 +217,12 @@ DBImpl::PreloadTable(const std::string& table_id) { 


        size += engine->PhysicalSize();

        if (size > available_size) {

            ENGINE_LOG_DEBUG << "Pre-load canceled since cache almost full";

            return Status(SERVER_CACHE_FULL, "Cache is full");

        } else {

            try {

                // step 1: load index

                std::string msg = "Pre-loaded file: " + file.file_id_ + " size: " + std::to_string(file.file_size_);

                TimeRecorderAuto rc_1(msg);

                engine->Load(true);

            } catch (std::exception& ex) {

                std::string msg = "Pre-load table encounter exception: " + std::string(ex.what());

core/src/utils/TimeRecorder.cpp

+7 −0
Original line number Diff line number Diff line
@@ -96,4 +96,11 @@ TimeRecorder::ElapseFromBegin(const std::string& msg) { 
    return span;

}



TimeRecorderAuto::TimeRecorderAuto(const std::string& header, int64_t log_level) : TimeRecorder(header, log_level) {

}



TimeRecorderAuto::~TimeRecorderAuto() {

    ElapseFromBegin("totally cost");

}



}  // namespace milvus

core/src/utils/TimeRecorder.h

+8 −1
Original line number Diff line number Diff line
@@ -28,7 +28,7 @@ class TimeRecorder { 
 public:

    explicit TimeRecorder(const std::string& header, int64_t log_level = 1);



    ~TimeRecorder();  // trace = 0, debug = 1, info = 2, warn = 3, error = 4, critical = 5

    virtual ~TimeRecorder();  // trace = 0, debug = 1, info = 2, warn = 3, error = 4, critical = 5



    double

    RecordSection(const std::string& msg);
@@ -50,4 +50,11 @@ class TimeRecorder { 
    int64_t log_level_;

};



class TimeRecorderAuto : public TimeRecorder {

 public:

    explicit TimeRecorderAuto(const std::string& header, int64_t log_level = 1);



    ~TimeRecorderAuto();

};



}  // namespace milvus