Commit 1164ee6b authored by groot's avatar groot Committed by JinHai-CN
Browse files

#2378 (#2388) 



* return partition lsn

Signed-off-by: default avataryhmo <yihua.mo@zilliz.com>

* fix wal lsn

Signed-off-by: default avatarshengjun.li <shengjun.li@zilliz.com>

* fix wal issue

Signed-off-by: default avataryhmo <yihua.mo@zilliz.com>

* changelog

Signed-off-by: default avataryhmo <yihua.mo@zilliz.com>

* typo

Signed-off-by: default avataryhmo <yihua.mo@zilliz.com>

* all collection include partition

Signed-off-by: default avataryhmo <yihua.mo@zilliz.com>

* fix build error

Signed-off-by: default avataryhmo <yihua.mo@zilliz.com>

* fix flush

Signed-off-by: default avatarshengjun.li <shengjun.li@zilliz.com>

Co-authored-by: default avatarshengjun.li <shengjun.li@zilliz.com>
parent 2b26468c

CHANGELOG.md

+1 −0
Original line number Diff line number Diff line
@@ -4,6 +4,7 @@ Please mark all change in change log and use the issue from GitHub 
# Milvus 0.9.1 (TBD)



## Bug

-   \#2378 Duplicate data after server restart



## Feature

-   \#2363 Update branch version to 0.9.1

core/src/db/DBImpl.cpp

+43 −17
Original line number Diff line number Diff line
@@ -487,7 +487,11 @@ DBImpl::CreatePartition(const std::string& collection_id, const std::string& par 
    }



    uint64_t lsn = 0;

    if (options_.wal_enable_) {

        lsn = wal_mgr_->CreatePartition(collection_id, partition_tag);

    } else {

        meta_ptr_->GetCollectionFlushLSN(collection_id, lsn);

    }

    return meta_ptr_->CreatePartition(collection_id, partition_name, partition_tag, lsn);

}
@@ -545,6 +549,10 @@ DBImpl::DropPartitionByTag(const std::string& collection_id, const std::string& 
        return status;

    }



    if (options_.wal_enable_) {

        wal_mgr_->DropPartition(collection_id, partition_tag);

    }



    return DropPartition(partition_name);

}
@@ -891,7 +899,7 @@ DBImpl::Flush(const std::string& collection_id) { 
            swn_wal_.Notify();

            flush_req_swn_.Wait();

        }



        StartMergeTask();

    } else {

        LOG_ENGINE_DEBUG_ << "MemTable flush";

        InternalFlush(collection_id);
@@ -918,6 +926,7 @@ DBImpl::Flush() { 
            swn_wal_.Notify();

            flush_req_swn_.Wait();

        }

        StartMergeTask();

    } else {

        LOG_ENGINE_DEBUG_ << "MemTable flush";

        InternalFlush();
@@ -1421,7 +1430,9 @@ DBImpl::DropIndex(const std::string& collection_id) { 
    }



    LOG_ENGINE_DEBUG_ << "Drop index for collection: " << collection_id;

    return DropCollectionIndexRecursively(collection_id);

    auto status = DropCollectionIndexRecursively(collection_id);

    StartMergeTask();  // merge small files after drop index

    return status;

}



Status
@@ -2407,30 +2418,39 @@ Status 
DBImpl::ExecWalRecord(const wal::MXLogRecord& record) {

    fiu_return_on("DBImpl.ExexWalRecord.return", Status(););



    auto collections_flushed = [&](const std::set<std::string>& collection_ids) -> uint64_t {

        if (collection_ids.empty()) {

            return 0;

        }



    auto collections_flushed = [&](const std::string collection_id,

                                   const std::set<std::string>& target_collection_names) -> uint64_t {

        uint64_t max_lsn = 0;

        if (options_.wal_enable_) {

            for (auto& collection : collection_ids) {

            uint64_t lsn = 0;

            for (auto& collection : target_collection_names) {

                meta_ptr_->GetCollectionFlushLSN(collection, lsn);

                wal_mgr_->CollectionFlushed(collection, lsn);

                if (lsn > max_lsn) {

                    max_lsn = lsn;

                }

            }

            wal_mgr_->CollectionFlushed(collection_id, lsn);

        }



        std::lock_guard<std::mutex> lck(merge_result_mutex_);

        for (auto& collection : collection_ids) {

        for (auto& collection : target_collection_names) {

            merge_collection_ids_.insert(collection);

        }

        return max_lsn;

    };



    auto partition_flushed = [&](const std::string& collection_id, const std::string& partition,

                                 const std::string& target_collection_name) {

        if (options_.wal_enable_) {

            uint64_t lsn = 0;

            meta_ptr_->GetCollectionFlushLSN(target_collection_name, lsn);

            wal_mgr_->PartitionFlushed(collection_id, partition, lsn);

        }



        std::lock_guard<std::mutex> lck(merge_result_mutex_);

        merge_collection_ids_.insert(target_collection_name);

    };



    Status status;



    switch (record.type) {
@@ -2447,7 +2467,9 @@ DBImpl::ExecWalRecord(const wal::MXLogRecord& record) { 
                                              (record.data_size / record.length / sizeof(float)),

                                              (const float*)record.data, record.attr_nbytes, record.attr_data_size,

                                              record.attr_data, record.lsn, flushed_collections);

            collections_flushed(flushed_collections);

            if (!flushed_collections.empty()) {

                partition_flushed(record.collection_id, record.partition_tag, target_collection_name);

            }



            milvus::server::CollectInsertMetrics metrics(record.length, status);

            break;
@@ -2465,7 +2487,9 @@ DBImpl::ExecWalRecord(const wal::MXLogRecord& record) { 
                                             (record.data_size / record.length / sizeof(uint8_t)),

                                             (const u_int8_t*)record.data, record.lsn, flushed_collections);

            // even though !status.ok, run

            collections_flushed(flushed_collections);

            if (!flushed_collections.empty()) {

                partition_flushed(record.collection_id, record.partition_tag, target_collection_name);

            }



            // metrics

            milvus::server::CollectInsertMetrics metrics(record.length, status);
@@ -2485,7 +2509,9 @@ DBImpl::ExecWalRecord(const wal::MXLogRecord& record) { 
                                             (record.data_size / record.length / sizeof(float)),

                                             (const float*)record.data, record.lsn, flushed_collections);

            // even though !status.ok, run

            collections_flushed(flushed_collections);

            if (!flushed_collections.empty()) {

                partition_flushed(record.collection_id, record.partition_tag, target_collection_name);

            }



            // metrics

            milvus::server::CollectInsertMetrics metrics(record.length, status);
@@ -2548,7 +2574,7 @@ DBImpl::ExecWalRecord(const wal::MXLogRecord& record) { 
                    flushed_collections.insert(collection_id);

                }



                collections_flushed(flushed_collections);

                collections_flushed(record.collection_id, flushed_collections);



            } else {

                // flush all collections
@@ -2558,7 +2584,7 @@ DBImpl::ExecWalRecord(const wal::MXLogRecord& record) { 
                    status = mem_mgr_->Flush(collection_ids);

                }



                uint64_t lsn = collections_flushed(collection_ids);

                uint64_t lsn = collections_flushed("", collection_ids);

                if (options_.wal_enable_) {

                    wal_mgr_->RemoveOldFiles(lsn);

                }

core/src/db/meta/Meta.h

+1 −1
Original line number Diff line number Diff line
@@ -59,7 +59,7 @@ class Meta { 
    HasCollection(const std::string& collection_id, bool& has_or_not, bool is_root = false) = 0;



    virtual Status

    AllCollections(std::vector<CollectionSchema>& table_schema_array) = 0;

    AllCollections(std::vector<CollectionSchema>& table_schema_array, bool is_root = false) = 0;



    virtual Status

    UpdateCollectionFlag(const std::string& collection_id, int64_t flag) = 0;

core/src/db/meta/MySQLMetaImpl.cpp

+14 −8
Original line number Diff line number Diff line
@@ -582,7 +582,7 @@ MySQLMetaImpl::HasCollection(const std::string& collection_id, bool& has_or_not, 
}



Status

MySQLMetaImpl::AllCollections(std::vector<CollectionSchema>& collection_schema_array) {

MySQLMetaImpl::AllCollections(std::vector<CollectionSchema>& collection_schema_array, bool is_root) {

    try {

        server::MetricCollector metric;

        mysqlpp::StoreQueryResult res;
@@ -599,8 +599,12 @@ MySQLMetaImpl::AllCollections(std::vector<CollectionSchema>& collection_schema_a 
            mysqlpp::Query statement = connectionPtr->query();

            statement << "SELECT id, table_id, dimension, engine_type, index_params, index_file_size, metric_type"

                      << " ,owner_table, partition_tag, version, flush_lsn"

                      << " FROM " << META_TABLES << " WHERE state <> " << std::to_string(CollectionSchema::TO_DELETE)

                      << " AND owner_table = \"\";";

                      << " FROM " << META_TABLES << " WHERE state <> " << std::to_string(CollectionSchema::TO_DELETE);

            if (is_root) {

                statement << " AND owner_table = \"\";";

            } else {

                statement << ";";

            }



            LOG_ENGINE_DEBUG_ << "AllCollections: " << statement.str();
@@ -1535,8 +1539,8 @@ MySQLMetaImpl::ShowPartitions(const std::string& collection_id, 


            mysqlpp::Query statement = connectionPtr->query();

            statement << "SELECT table_id, id, state, dimension, created_on, flag, index_file_size,"

                      << " engine_type, index_params, metric_type, partition_tag, version FROM " << META_TABLES

                      << " WHERE owner_table = " << mysqlpp::quote << collection_id << " AND state <> "

                      << " engine_type, index_params, metric_type, partition_tag, version, flush_lsn FROM "

                      << META_TABLES << " WHERE owner_table = " << mysqlpp::quote << collection_id << " AND state <> "

                      << std::to_string(CollectionSchema::TO_DELETE) << ";";



            LOG_ENGINE_DEBUG_ << "ShowPartitions: " << statement.str();
@@ -1559,6 +1563,7 @@ MySQLMetaImpl::ShowPartitions(const std::string& collection_id, 
            partition_schema.owner_collection_ = collection_id;

            resRow["partition_tag"].to_string(partition_schema.partition_tag_);

            resRow["version"].to_string(partition_schema.version_);

            partition_schema.flush_lsn_ = resRow["flush_lsn"];



            partition_schema_array.emplace_back(partition_schema);

        }
@@ -2755,6 +2760,7 @@ MySQLMetaImpl::SetGlobalLastLSN(uint64_t lsn) { 
            }



            bool first_create = false;

            uint64_t last_lsn = 0;

            {

                mysqlpp::StoreQueryResult res;

                mysqlpp::Query statement = connectionPtr->query();
@@ -2762,6 +2768,8 @@ MySQLMetaImpl::SetGlobalLastLSN(uint64_t lsn) { 
                res = statement.store();

                if (res.num_rows() == 0) {

                    first_create = true;

                } else {

                    last_lsn = res[0]["global_lsn"];

                }

            }
@@ -2773,7 +2781,7 @@ MySQLMetaImpl::SetGlobalLastLSN(uint64_t lsn) { 
                if (!statement.exec()) {

                    return HandleException("QUERY ERROR WHEN SET GLOBAL LSN", statement.error());

                }

            } else {

            } else if (lsn > last_lsn) {

                mysqlpp::Query statement = connectionPtr->query();

                statement << "UPDATE " << META_ENVIRONMENT << " SET global_lsn = " << lsn << ";";

                LOG_ENGINE_DEBUG_ << "SetGlobalLastLSN: " << statement.str();
@@ -2783,8 +2791,6 @@ MySQLMetaImpl::SetGlobalLastLSN(uint64_t lsn) { 
                }

            }

        }  // Scoped Connection



        LOG_ENGINE_DEBUG_ << "Successfully update global_lsn: " << lsn;

    } catch (std::exception& e) {

        return HandleException("Failed to set global lsn", e.what());

    }

core/src/db/meta/MySQLMetaImpl.h

+1 −1
Original line number Diff line number Diff line
@@ -42,7 +42,7 @@ class MySQLMetaImpl : public Meta { 
    HasCollection(const std::string& collection_id, bool& has_or_not, bool is_root = false) override;



    Status

    AllCollections(std::vector<CollectionSchema>& collection_schema_array) override;

    AllCollections(std::vector<CollectionSchema>& collection_schema_array, bool is_root = false) override;



    Status

    DropCollection(const std::string& collection_id) override;