Unverified Commit 9938c636 authored by Jin Hai's avatar Jin Hai Committed by GitHub
Browse files

Merge pull request #1581 from youny626/fix#1577 

fix #1577
parents 7cd727aa 281d8ec5

CHANGELOG.md

+1 −0
Original line number Diff line number Diff line
@@ -47,6 +47,7 @@ Please mark all change in change log and use the issue from GitHub 
-   \#1556 Index file not created after table and index created

-   \#1560 Search crashed with Super-high dimensional binary vector

-   \#1574 Set all existing bitset in cache when applying deletes

-   \#1577 Row count incorrect if delete vectors then create index



## Feature

-   \#216 Add CLI to get server info

core/src/scheduler/task/BuildIndexTask.cpp

+2 −1
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@ 
#include "scheduler/task/BuildIndexTask.h"



#include <fiu-local.h>



#include <memory>

#include <string>

#include <thread>
@@ -206,7 +207,7 @@ XBuildIndexTask::Execute() { 
        // step 6: update meta

        table_file.file_type_ = engine::meta::TableFileSchema::INDEX;

        table_file.file_size_ = index->PhysicalSize();

        table_file.row_count_ = index->Count();

        table_file.row_count_ = file_->row_count_;  // index->Count();



        auto origin_file = *file_;

        origin_file.file_type_ = engine::meta::TableFileSchema::BACKUP;

core/unittest/db/test_delete.cpp

+76 −0
Original line number Diff line number Diff line
@@ -264,6 +264,82 @@ TEST_F(DeleteTest, delete_multiple_times) { 
    }

}



TEST_F(DeleteTest, delete_before_create_index) {

    milvus::engine::meta::TableSchema table_info = BuildTableSchema();

    table_info.engine_type_ = (int32_t)milvus::engine::EngineType::FAISS_IVFFLAT;

    auto stat = db_->CreateTable(table_info);



    milvus::engine::meta::TableSchema table_info_get;

    table_info_get.table_id_ = table_info.table_id_;

    stat = db_->DescribeTable(table_info_get);

    ASSERT_TRUE(stat.ok());

    ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);



    int64_t nb = 10000;

    milvus::engine::VectorsData xb;

    BuildVectors(nb, xb);



    for (int64_t i = 0; i < nb; i++) {

        xb.id_array_.push_back(i);

    }



    stat = db_->InsertVectors(table_info.table_id_, "", xb);

    ASSERT_TRUE(stat.ok());



    stat = db_->Flush();

    ASSERT_TRUE(stat.ok());



    std::random_device rd;

    std::mt19937 gen(rd());

    std::uniform_int_distribution<int64_t> dis(0, nb - 1);



    int64_t num_query = 10;

    std::map<int64_t, milvus::engine::VectorsData> search_vectors;

    for (int64_t i = 0; i < num_query; ++i) {

        int64_t index = dis(gen);

        milvus::engine::VectorsData search;

        search.vector_count_ = 1;

        for (int64_t j = 0; j < TABLE_DIM; j++) {

            search.float_data_.push_back(xb.float_data_[index * TABLE_DIM + j]);

        }

        search_vectors.insert(std::make_pair(xb.id_array_[index], search));

    }



    milvus::engine::IDNumbers ids_to_delete;

    for (auto& kv : search_vectors) {

        ids_to_delete.emplace_back(kv.first);

    }

    stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete);



    stat = db_->Flush();

    ASSERT_TRUE(stat.ok());



    milvus::engine::TableIndex index;

    index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8;

    index.extra_params_ = {{"nlist", 100}};

    stat = db_->CreateIndex(table_info.table_id_, index);

    ASSERT_TRUE(stat.ok());



    uint64_t row_count;

    stat = db_->GetTableRowCount(table_info.table_id_, row_count);

    ASSERT_TRUE(stat.ok());

    ASSERT_EQ(row_count, nb - ids_to_delete.size());



    int topk = 10, nprobe = 10;

    for (auto& pair : search_vectors) {

        auto& search = pair.second;



        std::vector<std::string> tags;

        milvus::engine::ResultIds result_ids;

        milvus::engine::ResultDistances result_distances;

        stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids,

                          result_distances);

        ASSERT_NE(result_ids[0], pair.first);

        //        ASSERT_LT(result_distances[0], 1e-4);

        ASSERT_GT(result_distances[0], 1);

    }

}



TEST_F(DeleteTest, delete_with_index) {

    milvus::engine::meta::TableSchema table_info = BuildTableSchema();

    table_info.engine_type_ = (int32_t)milvus::engine::EngineType::FAISS_IVFFLAT;