Unverified Commit 01b6ad36 authored by Zhiru Zhu's avatar Zhiru Zhu Committed by GitHub
Browse files

Fix#1484 (#1490) 



* update

Signed-off-by: default avatarZhiru Zhu <zzhu@fandm.edu>

* #1426 Support to configure whether to enabled autoflush (#1468)

Signed-off-by: default avatarshengjun.li <shengjun.li@zilliz.com>
Signed-off-by: default avatarZhiru Zhu <zzhu@fandm.edu>

* fix cases: test_search_id/test_wal (#1486)

Signed-off-by: default avatardel-zhenwu <zw@zilliz.com>
Signed-off-by: default avatarZhiru Zhu <zzhu@fandm.edu>

* #1480 add return code for AVX512 selection (#1482)

Signed-off-by: default avataryudong.cai <yudong.cai@zilliz.com>
Signed-off-by: default avatarZhiru Zhu <zzhu@fandm.edu>

* update

Signed-off-by: default avatarZhiru Zhu <zzhu@fandm.edu>

* update

Signed-off-by: default avatarZhiru Zhu <zzhu@fandm.edu>

Co-authored-by: default avatarshengjun.li <49774184+shengjun1985@users.noreply.github.com>
Co-authored-by: default avatardel-zhenwu <56623710+del-zhenwu@users.noreply.github.com>
Co-authored-by: default avatarCai Yudong <yudong.cai@zilliz.com>
parent d460f48c

CHANGELOG.md

+1 −0
Original line number Diff line number Diff line
@@ -28,6 +28,7 @@ Please mark all change in change log and use the issue from GitHub 
-   \#1298 Unittest failed when on CPU2GPU case

-   \#1359 Negative distance value returned when searching with HNSW index type

-   \#1429 Server crashed when searching vectors using GPU

-   \#1484 Index type changed to IDMAP after compacted 



## Feature

-   \#216 Add CLI to get server info

core/src/db/DBImpl.cpp

+52 −14
Original line number Diff line number Diff line
@@ -658,12 +658,23 @@ DBImpl::Compact(const std::string& table_id) { 


    const std::lock_guard<std::mutex> lock(flush_merge_compact_mutex_);



    // Save table index

    TableIndex table_index;

    status = DescribeIndex(table_id, table_index);

    if (!status.ok()) {

        return status;

    }



    // Drop all index

    status = DropIndex(table_id);

    if (!status.ok()) {

        std::string err_msg = "Failed to drop index in compact: " + status.message();

        ENGINE_LOG_ERROR << err_msg;

        return Status(DB_ERROR, err_msg);

        return status;

    }



    // Then update table index to the previous index

    status = UpdateTableIndexRecursively(table_id, table_index);

    if (!status.ok()) {

        return status;

    }



    // Get files to compact from meta.
@@ -679,23 +690,51 @@ DBImpl::Compact(const std::string& table_id) { 
    ENGINE_LOG_DEBUG << "Found " << files_to_compact.size() << " segment to compact";



    OngoingFileChecker::GetInstance().MarkOngoingFiles(files_to_compact);



    meta::TableFilesSchema files_to_update;

    Status compact_status;

    for (auto& file : files_to_compact) {

        status = CompactFile(table_id, file);

        compact_status = CompactFile(table_id, file, files_to_update);



        if (!compact_status.ok()) {

            ENGINE_LOG_ERROR << "Compact failed for file " << file.file_id_ << ": " << compact_status.message();

            break;

        }

    }



    if (compact_status.ok()) {

        ENGINE_LOG_DEBUG << "Finished compacting table: " << table_id;

    }



    ENGINE_LOG_ERROR << "Updating meta after compaction...";



    // Drop index again, in case some files were in the index building process during compacting

    status = DropIndex(table_id);

    if (!status.ok()) {

            OngoingFileChecker::GetInstance().UnmarkOngoingFiles(files_to_compact);

        return status;

    }



    // Update index

    status = UpdateTableIndexRecursively(table_id, table_index);

    if (!status.ok()) {

        return status;

    }



    status = meta_ptr_->UpdateTableFiles(files_to_update);

    if (!status.ok()) {

        return status;

    }



    OngoingFileChecker::GetInstance().UnmarkOngoingFiles(files_to_compact);



    ENGINE_LOG_DEBUG << "Finished compacting table: " << table_id;

    ENGINE_LOG_DEBUG << "Finished updating meta after compaction";



    return status;

}



Status

DBImpl::CompactFile(const std::string& table_id, const milvus::engine::meta::TableFileSchema& file) {

DBImpl::CompactFile(const std::string& table_id, const meta::TableFileSchema& file,

                    meta::TableFilesSchema& files_to_update) {

    ENGINE_LOG_DEBUG << "Compacting segment " << file.segment_id_ << " for table: " << table_id;



    // Create new table file
@@ -740,10 +779,6 @@ DBImpl::CompactFile(const std::string& table_id, const milvus::engine::meta::Tab 
        return status;

    }



    // Drop index again, in case some files were in the index building process during merging

    // TODO: might be too frequent?

    DropIndex(table_id);



    // Update table files state

    // if index type isn't IDMAP, set file type to TO_INDEX if file size exceed index_file_size

    // else set file type to RAW, no need to build index
@@ -763,7 +798,10 @@ DBImpl::CompactFile(const std::string& table_id, const milvus::engine::meta::Tab 
    }



    updated.emplace_back(compacted_file);

    status = meta_ptr_->UpdateTableFiles(updated);



    for (auto& f : updated) {

        files_to_update.emplace_back(f);

    }



    ENGINE_LOG_DEBUG << "Compacted segment " << compacted_file.segment_id_ << " from "

                     << std::to_string(file_to_compact.file_size_) << " bytes to "

core/src/db/DBImpl.h

+2 −1
Original line number Diff line number Diff line
@@ -182,7 +182,8 @@ class DBImpl : public DB { 
    BackgroundBuildIndex();



    Status

    CompactFile(const std::string& table_id, const milvus::engine::meta::TableFileSchema& file);

    CompactFile(const std::string& table_id, const meta::TableFileSchema& file,

                meta::TableFilesSchema& files_to_update);



    /*

    Status

core/unittest/db/test_delete.cpp

+5 −1
Original line number Diff line number Diff line
@@ -571,7 +571,11 @@ TEST_F(DeleteTest, compact_with_index) { 


    stat = db_->Compact(GetTableName());

    ASSERT_TRUE(stat.ok());

    // std::this_thread::sleep_for(std::chrono::seconds(5));  // wait for build index to finish



    milvus::engine::TableIndex table_index;

    stat = db_->DescribeIndex(GetTableName(), table_index);

    ASSERT_TRUE(stat.ok());

    ASSERT_FLOAT_EQ(table_index.engine_type_, index.engine_type_);



    int topk = 10, nprobe = 10;

    for (auto& pair : search_vectors) {