Unverified Commit c8a59b27 authored by groot's avatar groot Committed by GitHub
Browse files

#1276 SQLite throw exception when create 50000+ partitions in a table (#1879) 



* #1276 SQLite throw exception when create 50000+ partitions in a table

Signed-off-by: default avatargroot <yihua.mo@zilliz.com>

* #1276 SQLite throw exception when create 50000+ partitions in a table

Signed-off-by: default avatargroot <yihua.mo@zilliz.com>

* remove some log

Signed-off-by: default avatargroot <yihua.mo@zilliz.com>

* remove some log

Signed-off-by: default avatargroot <yihua.mo@zilliz.com>

* typo

Signed-off-by: default avatargroot <yihua.mo@zilliz.com>

* typo

Signed-off-by: default avatargroot <yihua.mo@zilliz.com>

* fix unittest

Signed-off-by: default avatargroot <yihua.mo@zilliz.com>

Co-authored-by: default avatarJinHai-CN <hai.jin@zilliz.com>
parent e25c5af2

CHANGELOG.md

+2 −1
Original line number Diff line number Diff line
@@ -5,9 +5,10 @@ Please mark all change in change log and use the issue from GitHub 
# Milvus 0.8.0 (TBD)



## Bug

-   \#1276 SQLite throw exception after create 50000+ partitions in a table

-   \#1762 Server is not forbidden to create new partition which tag is `_default`

-   \#1873 Fix index file serialize to incorrect path

-   \#1881 Fix Annoy index search fail

-   \#1881 Fix Annoy index search failure



## Feature

-   \#261  Integrate ANNOY into Milvus

core/src/db/DBImpl.cpp

+6 −4
Original line number Diff line number Diff line
@@ -171,6 +171,8 @@ DBImpl::Stop() { 
            bg_flush_thread_.join();

        }



        WaitMergeFileFinish();



        swn_index_.Notify();

        bg_index_thread_.join();
@@ -1262,22 +1264,22 @@ DBImpl::BackgroundIndexThread() { 


void

DBImpl::WaitMergeFileFinish() {

    ENGINE_LOG_DEBUG << "Begin WaitMergeFileFinish";

    //    ENGINE_LOG_DEBUG << "Begin WaitMergeFileFinish";

    std::lock_guard<std::mutex> lck(merge_result_mutex_);

    for (auto& iter : merge_thread_results_) {

        iter.wait();

    }

    ENGINE_LOG_DEBUG << "End WaitMergeFileFinish";

    //    ENGINE_LOG_DEBUG << "End WaitMergeFileFinish";

}



void

DBImpl::WaitBuildIndexFinish() {

    ENGINE_LOG_DEBUG << "Begin WaitBuildIndexFinish";

    //    ENGINE_LOG_DEBUG << "Begin WaitBuildIndexFinish";

    std::lock_guard<std::mutex> lck(index_result_mutex_);

    for (auto& iter : index_thread_results_) {

        iter.wait();

    }

    ENGINE_LOG_DEBUG << "End WaitBuildIndexFinish";

    //    ENGINE_LOG_DEBUG << "End WaitBuildIndexFinish";

}



void

core/src/server/delivery/request/CreatePartitionRequest.cpp

+10 −0
Original line number Diff line number Diff line
@@ -18,10 +18,13 @@ 
#include <fiu-local.h>

#include <memory>

#include <string>

#include <vector>



namespace milvus {

namespace server {



constexpr uint64_t MAX_PARTITION_LIMIT = 5000;



CreatePartitionRequest::CreatePartitionRequest(const std::shared_ptr<milvus::server::Context>& context,

                                               const std::string& collection_name, const std::string& tag)

    : BaseRequest(context, BaseRequest::kCreatePartition), collection_name_(collection_name), tag_(tag) {
@@ -76,6 +79,13 @@ CreatePartitionRequest::OnExecute() { 
            }

        }



        // check partition total count

        std::vector<engine::meta::CollectionSchema> schema_array;

        status = DBWrapper::DB()->ShowPartitions(collection_name_, schema_array);

        if (schema_array.size() >= MAX_PARTITION_LIMIT) {

            return Status(SERVER_UNSUPPORTED_ERROR, "The number of partitions exceeds the upper limit(5000)");

        }



        rc.RecordSection("check validation");



        // step 2: create partition