Commit 88e5f77a authored by jinhai's avatar jinhai
Browse files

Merge branch 'branch-0.5.1' into 'branch-0.5.0' 

MS-627 Integrate new index: IVFSQHybrid

See merge request megasearch/milvus!683

Former-commit-id: e04bb5f5c273c15e0f5ec52904cb0cedfad7080e
parents 360aa1b2 0b834ea9

cpp/CHANGELOG.md

+1 −0
Original line number Diff line number Diff line
@@ -35,6 +35,7 @@ Please mark all change in change log and use the ticket from JIRA. 
- MS-626 - Refactor DataObj to support cache any type data



## New Feature

- MS-627 - Integrate new index: IVFSQHybrid



## Task

- MS-554 - Change license to Apache 2.0

cpp/src/core/cmake/ThirdPartyPackagesCore.cmake

+4 −2
Original line number Diff line number Diff line
@@ -235,8 +235,10 @@ else() 
    message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}")

endif()

# set(FAISS_MD5 "a589663865a8558205533c8ac414278c")

# set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c

set(FAISS_MD5 "21deb1c708490ca40ecb899122c01403") # commit-id 643e48f479637fd947e7b93fa4ca72b38ecc9a39

# set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c branch-0.2.0

# set(FAISS_MD5 "21deb1c708490ca40ecb899122c01403") # commit-id 643e48f479637fd947e7b93fa4ca72b38ecc9a39 branch-0.2.0

# set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1

set(FAISS_MD5 "5af237d77947ee632f169bcb36feee2b") # commit-id 2c8affd0da60354e4322fa4c0224519e7912b9c4 branch-0.2.1



if(DEFINED ENV{KNOWHERE_ARROW_URL})

    set(ARROW_SOURCE_URL "$ENV{KNOWHERE_ARROW_URL}")

cpp/src/core/knowhere/CMakeLists.txt

+1 −0
Original line number Diff line number Diff line
@@ -50,6 +50,7 @@ set(index_srcs 
        knowhere/index/vector_index/helpers/FaissGpuResourceMgr.cpp

        knowhere/index/vector_index/IndexIVFSQ.cpp

        knowhere/index/vector_index/IndexGPUIVFSQ.cpp

        knowhere/index/vector_index/IndexIVFSQHybrid.cpp

        knowhere/index/vector_index/IndexIVFPQ.cpp

        knowhere/index/vector_index/IndexGPUIVFPQ.cpp

        knowhere/index/vector_index/FaissBaseIndex.cpp

cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp

+13 −19
Original line number Diff line number Diff line
@@ -126,32 +126,22 @@ GPUIVF::LoadImpl(const BinarySet& index_binary) { 
    }

}



IVFIndexPtr

GPUIVF::Copy_index_gpu_to_cpu() {

    std::lock_guard<std::mutex> lk(mutex_);



    faiss::Index* device_index = index_.get();

    faiss::Index* host_index = faiss::gpu::index_gpu_to_cpu(device_index);



    std::shared_ptr<faiss::Index> new_index;

    new_index.reset(host_index);

    return std::make_shared<IVF>(new_index);

}



void

GPUIVF::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) {

    std::lock_guard<std::mutex> lk(mutex_);



    // TODO(linxj): gpu index support GenParams

    if (auto device_index = std::static_pointer_cast<faiss::gpu::GpuIndexIVF>(index_)) {

    if (auto device_index = std::dynamic_pointer_cast<faiss::gpu::GpuIndexIVF>(index_)) {

        auto search_cfg = std::dynamic_pointer_cast<IVFCfg>(cfg);

        device_index->setNumProbes(search_cfg->nprobe);



        {

            // TODO(linxj): allocate mem

            // TODO(linxj): allocate gpu mem

            ResScope rs(res_, gpu_id_);

            device_index->search(n, (float*)data, k, distances, labels);

        }

    } else {

        KNOWHERE_THROW_MSG("Not a GpuIndexIVF type.");

    }

}
@@ -159,12 +149,16 @@ VectorIndexPtr 
GPUIVF::CopyGpuToCpu(const Config& config) {

    std::lock_guard<std::mutex> lk(mutex_);



    if (auto device_idx = std::dynamic_pointer_cast<faiss::gpu::GpuIndexIVF>(index_)) {

        faiss::Index* device_index = index_.get();

        faiss::Index* host_index = faiss::gpu::index_gpu_to_cpu(device_index);



        std::shared_ptr<faiss::Index> new_index;

        new_index.reset(host_index);

        return std::make_shared<IVF>(new_index);

    } else {

        return std::make_shared<IVF>(index_);

    }

}



VectorIndexPtr

cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.h

+0 −4
Original line number Diff line number Diff line
@@ -78,10 +78,6 @@ class GPUIVF : public IVF, public GPUIndex { 
    VectorIndexPtr

    Clone() final;



    // TODO(linxj): Deprecated

    virtual IVFIndexPtr

    Copy_index_gpu_to_cpu();



 protected:

    void

    search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) override;