Commit f604a61c authored by jinhai's avatar jinhai
Browse files

Merge branch 'update_unittest' into 'branch-0.5.0' 

MS-648 Update unittest

See merge request megasearch/milvus!702

Former-commit-id: afa3bc636fb4c061034bcbab59da72d3f1bd6fd7
parents d9b49447 324a967f

cpp/CHANGELOG.md

+1 −0
Original line number Diff line number Diff line
@@ -36,6 +36,7 @@ Please mark all change in change log and use the ticket from JIRA. 
- MS-619 - Add optimizer class in scheduler

- MS-614 - Preload table at startup

- MS-626 - Refactor DataObj to support cache any type data

- MS-648 - Improve unittest



## New Feature

- MS-627 - Integrate new index: IVFSQHybrid

cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp

+9 −0
Original line number Diff line number Diff line
@@ -71,4 +71,13 @@ GPUIVFSQ::CopyGpuToCpu(const Config& config) { 
    return std::make_shared<IVFSQ>(new_index);

}



void

GPUIVFSQ::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) {

#ifdef CUSTOMIZATION

    GPUIVF::search_impl(n, data, k, distances, labels, cfg);

#else

    IVF::search_impl(n, data, k, distances, labels, cfg);

#endif

}



}  // namespace knowhere

cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.h

+4 −0
Original line number Diff line number Diff line
@@ -38,6 +38,10 @@ class GPUIVFSQ : public GPUIVF { 


    VectorIndexPtr

    CopyGpuToCpu(const Config& config) override;



 protected:

    void

    search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) override;

};



}  // namespace knowhere

cpp/src/core/unittest/test_ivf.cpp

+24 −21
Original line number Diff line number Diff line
@@ -154,8 +154,8 @@ class IVFTest : public DataGen, public TestWithParam<::std::tuple<std::string, P 
INSTANTIATE_TEST_CASE_P(IVFParameters, IVFTest,

                        Values(std::make_tuple("IVF", ParameterType::ivf),

                               std::make_tuple("GPUIVF", ParameterType::ivf),

                               //                            std::make_tuple("IVFPQ", ParameterType::ivfpq),

                               //                            std::make_tuple("GPUIVFPQ", ParameterType::ivfpq),

                               std::make_tuple("IVFPQ", ParameterType::ivfpq),

                               std::make_tuple("GPUIVFPQ", ParameterType::ivfpq),

                               std::make_tuple("IVFSQ", ParameterType::ivfsq),

#ifdef CUSTOMIZATION

                               std::make_tuple("IVFSQHybrid", ParameterType::ivfsq),
@@ -240,25 +240,26 @@ TEST_P(IVFTest, hybrid) { 
        auto result = hybrid_1_idx->Search(query_dataset, conf);

        AssertAnns(result, nq, conf->k);

        PrintResult(result, nq, k);

        hybrid_1_idx->UnsetQuantizer();

    }



    {

        auto hybrid_2_idx = std::make_shared<knowhere::IVFSQHybrid>(device_id);



        auto binaryset = index_->Serialize();

        hybrid_2_idx->Load(binaryset);



        auto quantizer_conf = std::make_shared<knowhere::QuantizerCfg>();

        quantizer_conf->mode = 1;

        quantizer_conf->gpu_id = device_id;

        auto q = hybrid_2_idx->LoadQuantizer(quantizer_conf);

        quantizer_conf->mode = 2;

        hybrid_2_idx->LoadData(q, quantizer_conf);



        auto result = hybrid_2_idx->Search(query_dataset, conf);

        AssertAnns(result, nq, conf->k);

        PrintResult(result, nq, k);

    }

    //    {

    //        auto hybrid_2_idx = std::make_shared<knowhere::IVFSQHybrid>(device_id);

    //

    //        auto binaryset = index_->Serialize();

    //        hybrid_2_idx->Load(binaryset);

    //

    //        auto quantizer_conf = std::make_shared<knowhere::QuantizerCfg>();

    //        quantizer_conf->mode = 1;

    //        quantizer_conf->gpu_id = device_id;

    //        auto q = hybrid_2_idx->LoadQuantizer(quantizer_conf);

    //        quantizer_conf->mode = 2;

    //        hybrid_2_idx->LoadData(q, quantizer_conf);

    //

    //        auto result = hybrid_2_idx->Search(query_dataset, conf);

    //        AssertAnns(result, nq, conf->k);

    //        PrintResult(result, nq, k);

    //    }

}



// TEST_P(IVFTest, gpu_to_cpu) {
@@ -438,6 +439,7 @@ TEST_P(IVFTest, clone_test) { 
    }

}



#ifdef CUSTOMIZATION

TEST_P(IVFTest, seal_test) {

    // FaissGpuResourceMgr::GetInstance().InitDevice(device_id);
@@ -472,6 +474,7 @@ TEST_P(IVFTest, seal_test) { 
    auto with_seal = tc.RecordSection("With seal");

    ASSERT_GE(without_seal, with_seal);

}

#endif



class GPURESTEST : public DataGen, public ::testing::Test {

 protected:
@@ -637,7 +640,7 @@ TEST_F(GPURESTEST, copyandsearch) { 
    // search and copy at the same time

    printf("==================\n");



    index_type = "GPUIVFSQ";

    index_type = "GPUIVF";

    index_ = IndexFactory(index_type);



    auto conf = std::make_shared<knowhere::IVFSQCfg>();
@@ -699,7 +702,7 @@ TEST_F(GPURESTEST, copyandsearch) { 
}



TEST_F(GPURESTEST, TrainAndSearch) {

    index_type = "GPUIVFSQ";

    index_type = "GPUIVF";

    index_ = IndexFactory(index_type);



    auto conf = std::make_shared<knowhere::IVFSQCfg>();

cpp/src/core/unittest/test_kdt.cpp

+1 −0
Original line number Diff line number Diff line
@@ -36,6 +36,7 @@ class KDTTest : public DataGen, public ::testing::Test { 
 protected:

    void

    SetUp() override {

        Generate(96, 1000, 10);

        index_ = std::make_shared<knowhere::CPUKDTRNG>();



        auto tempconf = std::make_shared<knowhere::KDTCfg>();