Commit fbe5531b authored by JinHai-CN's avatar JinHai-CN
Browse files

Test 



Signed-off-by: default avatarJinHai-CN <hai.jin@zilliz.com>
parent e7d83d90

core/src/db/insert/MemTable.cpp

+23 −15
Original line number Original line Diff line number Diff line
@@ -179,7 +179,7 @@ MemTable::ApplyDeletes() { 




    auto start_total = std::chrono::high_resolution_clock::now();

    auto start_total = std::chrono::high_resolution_clock::now();





    auto start = std::chrono::high_resolution_clock::now();

//    auto start = std::chrono::high_resolution_clock::now();





    std::vector<int> file_types{meta::TableFileSchema::FILE_TYPE::RAW, meta::TableFileSchema::FILE_TYPE::TO_INDEX,

    std::vector<int> file_types{meta::TableFileSchema::FILE_TYPE::RAW, meta::TableFileSchema::FILE_TYPE::TO_INDEX,

                                meta::TableFileSchema::FILE_TYPE::BACKUP};

                                meta::TableFileSchema::FILE_TYPE::BACKUP};
@@ -221,7 +221,7 @@ MemTable::ApplyDeletes() { 
    OngoingFileChecker::GetInstance().MarkOngoingFiles(files_to_check);

    OngoingFileChecker::GetInstance().MarkOngoingFiles(files_to_check);





    auto end = std::chrono::high_resolution_clock::now();

    auto end = std::chrono::high_resolution_clock::now();

    std::chrono::duration<double> diff = end - start;

    std::chrono::duration<double> diff = end - start_total;

    ENGINE_LOG_DEBUG << "Found " << ids_to_check_map.size() << " segment to apply deletes in " << diff.count() << " s";

    ENGINE_LOG_DEBUG << "Found " << ids_to_check_map.size() << " segment to apply deletes in " << diff.count() << " s";





    meta::TableFilesSchema table_files_to_update;

    meta::TableFilesSchema table_files_to_update;
@@ -230,7 +230,7 @@ MemTable::ApplyDeletes() { 
        auto& table_file = table_files[kv.first];

        auto& table_file = table_files[kv.first];

        ENGINE_LOG_DEBUG << "Applying deletes in segment: " << table_file.segment_id_;

        ENGINE_LOG_DEBUG << "Applying deletes in segment: " << table_file.segment_id_;





        start = std::chrono::high_resolution_clock::now();

        auto time0 = std::chrono::high_resolution_clock::now();





        std::string segment_dir;

        std::string segment_dir;

        utils::GetParentPath(table_file.location_, segment_dir);

        utils::GetParentPath(table_file.location_, segment_dir);
@@ -274,16 +274,14 @@ MemTable::ApplyDeletes() { 




        segment::DeletedDocsPtr deleted_docs = std::make_shared<segment::DeletedDocs>();

        segment::DeletedDocsPtr deleted_docs = std::make_shared<segment::DeletedDocs>();





        end = std::chrono::high_resolution_clock::now();

        auto time1 = std::chrono::high_resolution_clock::now();

        diff = end - start;

        diff = time1 - time0;

        ENGINE_LOG_DEBUG << "Loading uids and deleted docs took " << diff.count() << " s";

        ENGINE_LOG_DEBUG << "Loading uids and deleted docs took " << diff.count() << " s";





        start = std::chrono::high_resolution_clock::now();



        std::sort(ids_to_check.begin(), ids_to_check.end());

        std::sort(ids_to_check.begin(), ids_to_check.end());





        end = std::chrono::high_resolution_clock::now();

        auto time2 = std::chrono::high_resolution_clock::now();

        diff = end - start;

        diff = time2 - time1;

        ENGINE_LOG_DEBUG << "Sorting " << ids_to_check.size() << " ids took " << diff.count() << " s";

        ENGINE_LOG_DEBUG << "Sorting " << ids_to_check.size() << " ids took " << diff.count() << " s";





        size_t delete_count = 0;

        size_t delete_count = 0;
@@ -324,11 +322,13 @@ MemTable::ApplyDeletes() { 
                         << find_diff.count() << " s in total";

                         << find_diff.count() << " s in total";

        ENGINE_LOG_DEBUG << "Setting deleted docs and bloom filter took " << set_diff.count() << " s in total";

        ENGINE_LOG_DEBUG << "Setting deleted docs and bloom filter took " << set_diff.count() << " s in total";





        auto time3 = std::chrono::high_resolution_clock::now();



        for (auto i = 0; i < indexes.size(); ++i) {

        for (auto i = 0; i < indexes.size(); ++i) {

            indexes[i]->SetBlacklist(blacklists[i]);

            indexes[i]->SetBlacklist(blacklists[i]);

        }

        }





        start = std::chrono::high_resolution_clock::now();

//        start = std::chrono::high_resolution_clock::now();





        segment::Segment tmp_segment;

        segment::Segment tmp_segment;

        segment::SegmentWriter segment_writer(segment_dir);

        segment::SegmentWriter segment_writer(segment_dir);
@@ -337,20 +337,20 @@ MemTable::ApplyDeletes() { 
            break;

            break;

        }

        }





        end = std::chrono::high_resolution_clock::now();

        auto time4 = std::chrono::high_resolution_clock::now();

        diff = end - start;

        diff = time4 - time3;

        ENGINE_LOG_DEBUG << "Appended " << deleted_docs->GetSize()

        ENGINE_LOG_DEBUG << "Appended " << deleted_docs->GetSize()

                         << " offsets to deleted docs in segment: " << table_file.segment_id_ << " in " << diff.count()

                         << " offsets to deleted docs in segment: " << table_file.segment_id_ << " in " << diff.count()

                         << " s";

                         << " s";





        start = std::chrono::high_resolution_clock::now();

//        start = std::chrono::high_resolution_clock::now();





        status = segment_writer.WriteBloomFilter(id_bloom_filter_ptr);

        status = segment_writer.WriteBloomFilter(id_bloom_filter_ptr);

        if (!status.ok()) {

        if (!status.ok()) {

            break;

            break;

        }

        }

        end = std::chrono::high_resolution_clock::now();

        auto time5 = std::chrono::high_resolution_clock::now();

        diff = end - start;

        diff = time5 - time4;

        ENGINE_LOG_DEBUG << "Updated bloom filter in segment: " << table_file.segment_id_ << " in " << diff.count()

        ENGINE_LOG_DEBUG << "Updated bloom filter in segment: " << table_file.segment_id_ << " in " << diff.count()

                         << " s";

                         << " s";
@@ -362,8 +362,15 @@ MemTable::ApplyDeletes() { 
                table_files_to_update.emplace_back(file);

                table_files_to_update.emplace_back(file);

            }

            }

        }

        }

        auto time6 = std::chrono::high_resolution_clock::now();

        diff = time6 - time5;

        ENGINE_LOG_DEBUG << "Update table file row count in vector of segment: " << table_file.segment_id_ << " in " << diff.count()

                         << " s";

    }

    }





    auto time7 = std::chrono::high_resolution_clock::now();





    status = meta_->UpdateTableFilesRowCount(table_files_to_update);

    status = meta_->UpdateTableFilesRowCount(table_files_to_update);





    if (!status.ok()) {

    if (!status.ok()) {
@@ -375,6 +382,7 @@ MemTable::ApplyDeletes() { 
    doc_ids_to_delete_.clear();

    doc_ids_to_delete_.clear();





    auto end_total = std::chrono::high_resolution_clock::now();

    auto end_total = std::chrono::high_resolution_clock::now();

    ENGINE_LOG_DEBUG << "Update deletes to meta in table " << table_id_ << " in " << (end_total - time7).count() << " s";

    std::chrono::duration<double> diff_total = end_total - start_total;

    std::chrono::duration<double> diff_total = end_total - start_total;

    ENGINE_LOG_DEBUG << "Finished applying deletes in table " << table_id_ << " in " << diff_total.count() << " s";

    ENGINE_LOG_DEBUG << "Finished applying deletes in table " << table_id_ << " in " << diff_total.count() << " s";

sdk/examples/simple/src/ClientTest.cpp

+13 −9
Original line number Original line Diff line number Diff line
@@ -31,7 +31,7 @@ constexpr int64_t NQ = 5; 
constexpr int64_t TOP_K = 10;

constexpr int64_t TOP_K = 10;

constexpr int64_t NPROBE = 32;

constexpr int64_t NPROBE = 32;

constexpr int64_t SEARCH_TARGET = 5000;  // change this value, result is different

constexpr int64_t SEARCH_TARGET = 5000;  // change this value, result is different

constexpr int64_t ADD_ENTITY_LOOP = 5;

constexpr int64_t ADD_ENTITY_LOOP = 50;

constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8;

constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8;

constexpr int32_t NLIST = 16384;

constexpr int32_t NLIST = 16384;
@@ -245,15 +245,19 @@ ClientTest::Test() { 
    Flush(collection_name);

    Flush(collection_name);

    ShowCollectionInfo(collection_name);

    ShowCollectionInfo(collection_name);





    GetEntityById(collection_name, search_id_array_[0]);

//    GetEntityById(collection_name, search_id_array_[0]);

    SearchEntities(collection_name, TOP_K, NPROBE);

//    SearchEntities(collection_name, TOP_K, NPROBE);



//

    CreateIndex(collection_name, INDEX_TYPE, NLIST);

//    CreateIndex(collection_name, INDEX_TYPE, NLIST);

    ShowCollectionInfo(collection_name);

//    ShowCollectionInfo(collection_name);



//

    PreloadCollection(collection_name);

//    PreloadCollection(collection_name);





    std::vector<int64_t> delete_ids = {search_id_array_[0], search_id_array_[1]};

    constexpr long vector_size = 2000000;

    std::vector<int64_t> delete_ids(vector_size); //  = {search_id_array_[0], search_id_array_[1]};

    for(long i= 0; i < vector_size;  ++ i) {

        delete_ids[i] = i;

    }

    DeleteByIds(collection_name, delete_ids);

    DeleteByIds(collection_name, delete_ids);

    SearchEntities(collection_name, TOP_K, NPROBE); // this line get two search error since we delete two entities

    SearchEntities(collection_name, TOP_K, NPROBE); // this line get two search error since we delete two entities