Commit 95d085bc authored by xiaojun.lin's avatar xiaojun.lin
Browse files

Merge remote-tracking branch 'gitlab/0.5.1' into upgrade_faiss 


Former-commit-id: 5a643c79da8baa6c7667368f961a1bb4ab8e2220
parents d4fe7607 5a45c9b4

ci/jenkinsfile/milvus_build.groovy

+1 −1
Original line number Diff line number Diff line
@@ -14,7 +14,7 @@ container('milvus-build-env') { 
                            sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' \

                            && export JFROG_USER_NAME='${USERNAME}' \

                            && export JFROG_PASSWORD='${PASSWORD}' \

                            && export FAISS_URL='http://192.168.1.105:6060/jinhai/faiss/-/archive/branch-0.2.1/faiss-branch-0.2.1.tar.gz' \

                            && export FAISS_URL='http://192.168.1.105:6060/jinhai/faiss/-/archive/branch-0.3.0/faiss-branch-0.3.0.tar.gz' \

                            && ./build.sh -t ${params.BUILD_TYPE} -d /opt/milvus -j -u -c"



                            sh "./coverage.sh -u root -p 123456 -t \$POD_IP"

ci/jenkinsfile/milvus_build_no_ut.groovy

+1 −1
Original line number Diff line number Diff line
@@ -14,7 +14,7 @@ container('milvus-build-env') { 
                            sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' \

                            && export JFROG_USER_NAME='${USERNAME}' \

                            && export JFROG_PASSWORD='${PASSWORD}' \

                            && export FAISS_URL='http://192.168.1.105:6060/jinhai/faiss/-/archive/branch-0.2.1/faiss-branch-0.2.1.tar.gz' \

                            && export FAISS_URL='http://192.168.1.105:6060/jinhai/faiss/-/archive/branch-0.3.0/faiss-branch-0.3.0.tar.gz' \

                            && ./build.sh -t ${params.BUILD_TYPE} -j -d /opt/milvus"

                        }

                    }

core/src/db/engine/ExecutionEngineImpl.cpp

+53 −6
Original line number Diff line number Diff line
@@ -18,16 +18,13 @@ 
#include "db/engine/ExecutionEngineImpl.h"

#include "cache/CpuCacheMgr.h"

#include "cache/GpuCacheMgr.h"

#include "knowhere/common/Config.h"

#include "metrics/Metrics.h"

#include "scheduler/Utils.h"

#include "server/Config.h"

#include "utils/CommonUtil.h"

#include "utils/Exception.h"

#include "utils/Log.h"



#include "knowhere/common/Config.h"

#include "knowhere/common/Exception.h"

#include "knowhere/index/vector_index/IndexIVFSQHybrid.h"

#include "scheduler/Utils.h"

#include "server/Config.h"

#include "wrapper/ConfAdapter.h"

#include "wrapper/ConfAdapterMgr.h"

#include "wrapper/VecImpl.h"
@@ -260,6 +257,56 @@ ExecutionEngineImpl::Load(bool to_cache) { 
Status

ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) {

    if (hybrid) {

#if 1

        const std::string key = location_ + ".quantizer";

        std::vector<uint64_t> gpus = scheduler::get_gpu_pool();



        const int64_t NOT_FOUND = -1;

        int64_t device_id = NOT_FOUND;



        // cache hit

        {

            knowhere::QuantizerPtr quantizer = nullptr;



            for (auto& gpu : gpus) {

                auto cache = cache::GpuCacheMgr::GetInstance(gpu);

                if (auto cached_quantizer = cache->GetIndex(key)) {

                    device_id = gpu;

                    quantizer = std::static_pointer_cast<CachedQuantizer>(cached_quantizer)->Data();

                }

            }



            if (device_id != NOT_FOUND) {

                // cache hit

                auto config = std::make_shared<knowhere::QuantizerCfg>();

                config->gpu_id = device_id;

                config->mode = 2;

                auto new_index = index_->LoadData(quantizer, config);

                index_ = new_index;

            }

        }



        if (device_id == NOT_FOUND) {

            // cache miss

            std::vector<int64_t> all_free_mem;

            for (auto& gpu : gpus) {

                auto cache = cache::GpuCacheMgr::GetInstance(gpu);

                auto free_mem = cache->CacheCapacity() - cache->CacheUsage();

                all_free_mem.push_back(free_mem);

            }



            auto max_e = std::max_element(all_free_mem.begin(), all_free_mem.end());

            auto best_index = std::distance(all_free_mem.begin(), max_e);

            device_id = gpus[best_index];



            auto pair = index_->CopyToGpuWithQuantizer(device_id);

            index_ = pair.first;



            // cache

            auto cached_quantizer = std::make_shared<CachedQuantizer>(pair.second);

            cache::GpuCacheMgr::GetInstance(device_id)->InsertItem(key, cached_quantizer);

        }

#endif

        return Status::OK();

    }

core/src/index/cmake/ThirdPartyPackagesCore.cmake

+2 −1
Original line number Diff line number Diff line
@@ -244,7 +244,8 @@ if(CUSTOMIZATION) 
        # set(FAISS_MD5 "21deb1c708490ca40ecb899122c01403") # commit-id 643e48f479637fd947e7b93fa4ca72b38ecc9a39 branch-0.2.0

        # set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1

        # set(FAISS_MD5 "c89ea8e655f5cdf58f42486f13614714") # commit-id 9c28a1cbb88f41fa03b03d7204106201ad33276b branch-0.2.1

        set(FAISS_MD5 "87fdd86351ffcaf3f80dc26ade63c44b") # commit-id 841a156e67e8e22cd8088e1b58c00afbf2efc30b branch-0.2.1

        # set(FAISS_MD5 "87fdd86351ffcaf3f80dc26ade63c44b") # commit-id 841a156e67e8e22cd8088e1b58c00afbf2efc30b branch-0.2.1

        set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0

    endif()

else()

    set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/v1.5.3.tar.gz")

core/src/index/knowhere/knowhere/index/vector_index/FaissBaseIndex.h

+1 −1
Original line number Diff line number Diff line
@@ -38,7 +38,7 @@ class FaissBaseIndex { 
    virtual void

    SealImpl();



 protected:

 public:

    std::shared_ptr<faiss::Index> index_ = nullptr;

};