Unverified Commit aa2fe726 authored by groot's avatar groot Committed by GitHub
Browse files

prepare for wal (#3297) 



* prepare for wal

Signed-off-by: default avatargroot <yihua.mo@zilliz.com>

* prepare for wal

Signed-off-by: default avatargroot <yihua.mo@zilliz.com>
parent 2f6920f8

core/src/db/DB.h

+5 −2
Original line number Diff line number Diff line
@@ -91,16 +91,19 @@ class DB { 
    virtual Status

    DescribeIndex(const std::string& collection_name, const std::string& field_name, CollectionIndex& index) = 0;



    // op_id is for wal machinery, this id will be used in MemManager

    virtual Status

    Insert(const std::string& collection_name, const std::string& partition_name, DataChunkPtr& data_chunk) = 0;

    Insert(const std::string& collection_name, const std::string& partition_name, DataChunkPtr& data_chunk,

           id_t op_id = 0) = 0;



    virtual Status

    GetEntityByID(const std::string& collection_name, const IDNumbers& id_array,

                  const std::vector<std::string>& field_names, std::vector<bool>& valid_row,

                  DataChunkPtr& data_chunk) = 0;



    // op_id is for wal machinery, this id will be used in MemManager

    virtual Status

    DeleteEntityByID(const std::string& collection_name, const engine::IDNumbers& entity_ids) = 0;

    DeleteEntityByID(const std::string& collection_name, const engine::IDNumbers& entity_ids, id_t op_id = 0) = 0;



    virtual Status

    ListIDInSegment(const std::string& collection_name, int64_t segment_id, IDNumbers& entity_ids) = 0;

core/src/db/DBFactory.cpp

+4 −4
Original line number Diff line number Diff line
@@ -11,8 +11,8 @@ 


#include "db/DBFactory.h"

#include "db/DBImpl.h"

#include "db/transcript/Transcript.h"

#include "db/wal/WriteAheadLog.h"

#include "db/transcript/TranscriptProxy.h"

#include "db/wal/WalProxy.h"



namespace milvus {

namespace engine {
@@ -23,12 +23,12 @@ DBFactory::BuildDB(const DBOptions& options) { 


    // need wal? wal must be after db

    if (options.wal_enable_) {

        db = std::make_shared<WriteAheadLog>(db, options);

        db = std::make_shared<WalProxy>(db, options);

    }



    // need transcript? transcript must be after wal

    if (options.transcript_enable_) {

        db = std::make_shared<Transcript>(db, options);

        db = std::make_shared<TranscriptProxy>(db, options);

    }



    return db;

core/src/db/DBImpl.cpp

+5 −4
Original line number Diff line number Diff line
@@ -412,7 +412,8 @@ DBImpl::DescribeIndex(const std::string& collection_name, const std::string& fie 
}



Status

DBImpl::Insert(const std::string& collection_name, const std::string& partition_name, DataChunkPtr& data_chunk) {

DBImpl::Insert(const std::string& collection_name, const std::string& partition_name, DataChunkPtr& data_chunk,

               id_t op_id) {

    CHECK_INITIALIZED;



    if (data_chunk == nullptr) {
@@ -473,7 +474,7 @@ DBImpl::Insert(const std::string& collection_name, const std::string& partition_ 
    int64_t collection_id = ss->GetCollectionId();

    int64_t partition_id = part->GetID();



    auto status = mem_mgr_->InsertEntities(collection_id, partition_id, data_chunk, 0);

    auto status = mem_mgr_->InsertEntities(collection_id, partition_id, data_chunk, op_id);

    if (!status.ok()) {

        return status;

    }
@@ -509,7 +510,7 @@ DBImpl::GetEntityByID(const std::string& collection_name, const IDNumbers& id_ar 
}



Status

DBImpl::DeleteEntityByID(const std::string& collection_name, const engine::IDNumbers& entity_ids) {

DBImpl::DeleteEntityByID(const std::string& collection_name, const engine::IDNumbers& entity_ids, id_t op_id) {

    CHECK_INITIALIZED;



    snapshot::ScopedSnapshotT ss;
@@ -519,7 +520,7 @@ DBImpl::DeleteEntityByID(const std::string& collection_name, const engine::IDNum 
        return status;

    }



    status = mem_mgr_->DeleteEntities(ss->GetCollectionId(), entity_ids, 0);

    status = mem_mgr_->DeleteEntities(ss->GetCollectionId(), entity_ids, op_id);

    return status;

}

core/src/db/DBImpl.h

+3 −2
Original line number Diff line number Diff line
@@ -86,7 +86,8 @@ class DBImpl : public DB, public ConfigObserver { 
    DescribeIndex(const std::string& collection_name, const std::string& field_name, CollectionIndex& index) override;



    Status

    Insert(const std::string& collection_name, const std::string& partition_name, DataChunkPtr& data_chunk) override;

    Insert(const std::string& collection_name, const std::string& partition_name, DataChunkPtr& data_chunk,

           id_t op_id) override;



    Status

    GetEntityByID(const std::string& collection_name, const IDNumbers& id_array,
@@ -94,7 +95,7 @@ class DBImpl : public DB, public ConfigObserver { 
                  DataChunkPtr& data_chunk) override;



    Status

    DeleteEntityByID(const std::string& collection_name, const engine::IDNumbers& entity_ids) override;

    DeleteEntityByID(const std::string& collection_name, const engine::IDNumbers& entity_ids, id_t op_id) override;



    Status

    Query(const server::ContextPtr& context, const query::QueryPtr& query_ptr, engine::QueryResultPtr& result) override;

core/src/db/DBProxy.cpp

+5 −4
Original line number Diff line number Diff line
@@ -121,9 +121,10 @@ DBProxy::DescribeIndex(const std::string& collection_name, const std::string& fi 
}



Status

DBProxy::Insert(const std::string& collection_name, const std::string& partition_name, DataChunkPtr& data_chunk) {

DBProxy::Insert(const std::string& collection_name, const std::string& partition_name, DataChunkPtr& data_chunk,

                id_t op_id) {

    DB_CHECK

    return db_->Insert(collection_name, partition_name, data_chunk);

    return db_->Insert(collection_name, partition_name, data_chunk, op_id);

}



Status
@@ -135,9 +136,9 @@ DBProxy::GetEntityByID(const std::string& collection_name, const IDNumbers& id_a 
}



Status

DBProxy::DeleteEntityByID(const std::string& collection_name, const engine::IDNumbers& entity_ids) {

DBProxy::DeleteEntityByID(const std::string& collection_name, const engine::IDNumbers& entity_ids, id_t op_id) {

    DB_CHECK

    return db_->DeleteEntityByID(collection_name, entity_ids);

    return db_->DeleteEntityByID(collection_name, entity_ids, op_id);

}



Status