Unverified Commit 6e3991d1 authored by shengjun.li's avatar shengjun.li Committed by GitHub
Browse files

#1426 Support to configure whether to enabled autoflush (#1468) 



Signed-off-by: default avatarshengjun.li <shengjun.li@zilliz.com>
parent 23e0816b

CHANGELOG.md

+3 −0
Original line number Diff line number Diff line
@@ -40,6 +40,7 @@ Please mark all change in change log and use the issue from GitHub 
-   \#813 Add push mode for prometheus monitor

-   \#815 Support MinIO storage

-   \#823 Support binary vector tanimoto/jaccard/hamming metric

-   \#830 - Support WAL(write-ahead logging)

-   \#853 Support HNSW

-   \#861 Support DeleteById / SearchByID / GetVectorById / Flush

-   \#910 Change Milvus c++ standard to c++17
@@ -68,8 +69,10 @@ Please mark all change in change log and use the issue from GitHub 
-   \#1234 Do S3 server validation check when Milvus startup

-   \#1263 Allow system conf modifiable and some take effect directly

-   \#1320 Remove debug logging from faiss

-   \#1426 - Support to configure whether to enabled autoflush and the autoflush interval

-   \#1444 Improve delete





## Task

-   \#1327 Exclude third-party code from codebeat

-   \#1331 Exclude third-party code from codacy

core/conf/demo/server_config.yaml

+4 −3
Original line number Diff line number Diff line
@@ -46,13 +46,14 @@ server_config: 
#                      | loaded when Milvus server starts up.                       |            |                 |

#                      | '*' means preload all existing tables.                     |            |                 |

#----------------------+------------------------------------------------------------+------------+-----------------+

# auto_flush_interval  | The interval, in milliseconds, at which Milvus             | Integer    | 1000 (ms)       |

#                      | automatically flushes data to disk.                        |            |                 |

# auto_flush_interval  | The interval, in seconds, at which Milvus automatically    | Integer    | 1 (s)           |

#                      | flushes data to disk.                                      |            |                 |

#                      | 0 means disable the regular flush.                         |            |                 |

#----------------------+------------------------------------------------------------+------------+-----------------+

db_config:

  backend_url: sqlite://:@:/

  preload_table:

  auto_flush_interval: 1000

  auto_flush_interval: 1



#----------------------+------------------------------------------------------------+------------+-----------------+

# Storage Config       | Description                                                | Type       | Default         |

core/conf/server_cpu_config.template

+4 −3
Original line number Diff line number Diff line
@@ -46,13 +46,14 @@ server_config: 
#                      | loaded when Milvus server starts up.                       |            |                 |

#                      | '*' means preload all existing tables.                     |            |                 |

#----------------------+------------------------------------------------------------+------------+-----------------+

# auto_flush_interval  | The interval, in milliseconds, at which Milvus             | Integer    | 1000 (ms)       |

#                      | automatically flushes data to disk.                        |            |                 |

# auto_flush_interval  | The interval, in seconds, at which Milvus automatically    | Integer    | 1 (s)           |

#                      | flushes data to disk.                                      |            |                 |

#                      | 0 means disable the regular flush.                         |            |                 |

#----------------------+------------------------------------------------------------+------------+-----------------+

db_config:

  backend_url: sqlite://:@:/

  preload_table:

  auto_flush_interval: 1000

  auto_flush_interval: 1



#----------------------+------------------------------------------------------------+------------+-----------------+

# Storage Config       | Description                                                | Type       | Default         |

core/conf/server_gpu_config.template

+4 −3
Original line number Diff line number Diff line
@@ -46,13 +46,14 @@ server_config: 
#                      | loaded when Milvus server starts up.                       |            |                 |

#                      | '*' means preload all existing tables.                     |            |                 |

#----------------------+------------------------------------------------------------+------------+-----------------+

# auto_flush_interval  | The interval, in milliseconds, at which Milvus             | Integer    | 1000 (ms)       |

#                      | automatically flushes data to disk.                        |            |                 |

# auto_flush_interval  | The interval, in seconds, at which Milvus automatically    | Integer    | 1 (s)           |

#                      | flushes data to disk.                                      |            |                 |

#                      | 0 means disable the regular flush.                         |            |                 |

#----------------------+------------------------------------------------------------+------------+-----------------+

db_config:

  backend_url: sqlite://:@:/

  preload_table:

  auto_flush_interval: 1000

  auto_flush_interval: 1



#----------------------+------------------------------------------------------------+------------+-----------------+

# Storage Config       | Description                                                | Type       | Default         |

core/src/db/DBImpl.cpp

+27 −13
Original line number Diff line number Diff line
@@ -146,7 +146,7 @@ DBImpl::Stop() { 
    if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) {

        if (options_.wal_enable_) {

            // wait flush merge/buildindex finish

            wal_task_swn_.Notify();

            bg_task_swn_.Notify();

            bg_wal_thread_.join();



        } else {
@@ -156,6 +156,7 @@ DBImpl::Stop() { 
            ExecWalRecord(record);



            // wait merge/buildindex finish

            bg_task_swn_.Notify();

            bg_timer_thread_.join();

        }
@@ -500,7 +501,7 @@ DBImpl::InsertVectors(const std::string& table_id, const std::string& partition_ 
        } else if (!vectors.binary_data_.empty()) {

            wal_mgr_->Insert(table_id, partition_tag, vectors.id_array_, vectors.binary_data_);

        }

        wal_task_swn_.Notify();

        bg_task_swn_.Notify();



    } else {

        wal::MXLogRecord record;
@@ -543,7 +544,7 @@ DBImpl::DeleteVectors(const std::string& table_id, IDNumbers vector_ids) { 
    Status status;

    if (options_.wal_enable_) {

        wal_mgr_->DeleteById(table_id, vector_ids);

        wal_task_swn_.Notify();

        bg_task_swn_.Notify();



    } else {

        wal::MXLogRecord record;
@@ -583,7 +584,7 @@ DBImpl::Flush(const std::string& table_id) { 
        auto lsn = wal_mgr_->Flush(table_id);

        ENGINE_LOG_DEBUG << "wal_mgr_->Flush";

        if (lsn != 0) {

            wal_task_swn_.Notify();

            bg_task_swn_.Notify();

            flush_task_swn_.Wait();

            ENGINE_LOG_DEBUG << "flush_task_swn_.Wait()";

        }
@@ -614,7 +615,7 @@ DBImpl::Flush() { 
        ENGINE_LOG_DEBUG << "WAL flush";

        auto lsn = wal_mgr_->Flush();

        if (lsn != 0) {

            wal_task_swn_.Notify();

            bg_task_swn_.Notify();

            flush_task_swn_.Wait();

        }

    } else {
@@ -1204,7 +1205,11 @@ DBImpl::BackgroundTimerTask() { 
            break;

        }



        std::this_thread::sleep_for(std::chrono::milliseconds(options_.auto_flush_interval_));

        if (options_.auto_flush_interval_ > 0) {

            bg_task_swn_.Wait_For(std::chrono::seconds(options_.auto_flush_interval_));

        } else {

            bg_task_swn_.Wait();

        }



        StartMetricTask();

        StartMergeTask();
@@ -1945,10 +1950,13 @@ void 
DBImpl::BackgroundWalTask() {

    server::SystemInfo::GetInstance().Init();



    std::chrono::system_clock::time_point next_auto_flush_time;

    auto get_next_auto_flush_time = [&]() {

        return std::chrono::system_clock::now() + std::chrono::milliseconds(options_.auto_flush_interval_);

        return std::chrono::system_clock::now() + std::chrono::seconds(options_.auto_flush_interval_);

    };

    auto next_auto_flush_time = get_next_auto_flush_time();

    if (options_.auto_flush_interval_ > 0) {

        next_auto_flush_time = get_next_auto_flush_time();

    }



    wal::MXLogRecord record;
@@ -1963,10 +1971,12 @@ DBImpl::BackgroundWalTask() { 
    };



    while (true) {

        if (options_.auto_flush_interval_ > 0) {

            if (std::chrono::system_clock::now() >= next_auto_flush_time) {

                auto_flush();

                next_auto_flush_time = get_next_auto_flush_time();

            }

        }



        auto error_code = wal_mgr_->GetNextRecord(record);

        if (error_code != WAL_SUCCESS) {
@@ -1981,7 +1991,7 @@ DBImpl::BackgroundWalTask() { 
                flush_task_swn_.Notify();



                // if user flush all manually, update auto flush also

                if (record.table_id.empty()) {

                if (record.table_id.empty() && options_.auto_flush_interval_ > 0) {

                    next_auto_flush_time = get_next_auto_flush_time();

                }

            }
@@ -1995,7 +2005,11 @@ DBImpl::BackgroundWalTask() { 
                break;

            }



            wal_task_swn_.Wait_Until(next_auto_flush_time);

            if (options_.auto_flush_interval_ > 0) {

                bg_task_swn_.Wait_Until(next_auto_flush_time);

            } else {

                bg_task_swn_.Wait();

            }

        }

    }

}