Unverified Commit 48bd34b0 authored by Jin Hai's avatar Jin Hai Committed by GitHub
Browse files

Merge pull request #555 from yhmo/clean_cache 

#530 BuildIndex stop when do build index and search simultaneously
parents 69418f86 87a36ebd

CHANGELOG.md

+1 −0
Original line number Diff line number Diff line
@@ -24,6 +24,7 @@ Please mark all change in change log and use the ticket from JIRA. 
- \#486 - gpu no usage during index building

- \#509 - IVF_PQ index build trapped into dead loop caused by invalid params

- \#513 - Unittest DELETE_BY_RANGE sometimes failed

- \#523 - Erase file data from cache once the file is marked as deleted

- \#527 - faiss benchmark not compatible with faiss 1.6.0

- \#530 - BuildIndex stop when do build index and search simultaneously

- \#532 - assigin value to `table_name` from confest shell

core/src/cache/Cache.inl

+4 −3
Original line number Diff line number Diff line
@@ -100,7 +100,7 @@ Cache<ItemObj>::insert(const std::string& key, const ItemObj& item) { 


        lru_.put(key, item);

        SERVER_LOG_DEBUG << "Insert " << key << " size: " << item->Size() << " bytes into cache, usage: " << usage_

                         << " bytes";

                         << " bytes," << " capacity: " << capacity_ << " bytes";

    }

}
@@ -115,7 +115,8 @@ Cache<ItemObj>::erase(const std::string& key) { 
    const ItemObj& old_item = lru_.get(key);

    usage_ -= old_item->Size();



    SERVER_LOG_DEBUG << "Erase " << key << " size: " << old_item->Size();

    SERVER_LOG_DEBUG << "Erase " << key << " size: " << old_item->Size() << " bytes from cache, usage: " << usage_

                     << " bytes," << " capacity: " << capacity_ << " bytes";



    lru_.erase(key);

}

core/src/db/DBImpl.cpp

+12 −5
Original line number Diff line number Diff line
@@ -112,7 +112,7 @@ DBImpl::Stop() { 
    bg_timer_thread_.join();



    if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) {

        meta_ptr_->CleanUp();

        meta_ptr_->CleanUpShadowFiles();

    }



    // ENGINE_LOG_TRACE << "DB service stop";
@@ -777,11 +777,18 @@ DBImpl::BackgroundCompaction(std::set<std::string> table_ids) { 


    meta_ptr_->Archive();



    int ttl = 5 * meta::M_SEC;  // default: file will be deleted after 5 minutes

    {

        uint64_t ttl = 10 * meta::SECOND;  // default: file data will be erase from cache after few seconds

        meta_ptr_->CleanUpCacheWithTTL(ttl);

    }



    {

        uint64_t ttl = 5 * meta::M_SEC;  // default: file will be deleted after few minutes

        if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) {

            ttl = meta::D_SEC;

        }

        meta_ptr_->CleanUpFilesWithTTL(ttl);

    }



    // ENGINE_LOG_TRACE << " Background compaction thread exit";

}

core/src/db/engine/ExecutionEngineImpl.cpp

+5 −0
Original line number Diff line number Diff line
@@ -257,6 +257,11 @@ ExecutionEngineImpl::PhysicalSize() const { 
Status

ExecutionEngineImpl::Serialize() {

    auto status = write_index(index_, location_);



    // here we reset index size by file size,

    // since some index type(such as SQ8) data size become smaller after serialized

    index_->set_size(PhysicalSize());



    return status;

}

core/src/db/meta/Meta.h

+6 −2
Original line number Diff line number Diff line
@@ -118,9 +118,13 @@ class Meta { 
    Archive() = 0;



    virtual Status

    CleanUp() = 0;

    CleanUpShadowFiles() = 0;



    virtual Status CleanUpFilesWithTTL(uint16_t) = 0;

    virtual Status

    CleanUpCacheWithTTL(uint64_t seconds) = 0;



    virtual Status

    CleanUpFilesWithTTL(uint64_t seconds) = 0;



    virtual Status

    DropAll() = 0;