Commit 993558a0 authored by xiaojun.lin's avatar xiaojun.lin
Browse files

MS-565 remove jsoncons v2 


Former-commit-id: 64e514a130640eae43c4e9657a3fcbe2efac3074
parent a03e7ed2

cpp/src/core/knowhere/CMakeLists.txt

+0 −3
Original line number Diff line number Diff line
@@ -117,7 +117,6 @@ set(CORE_INCLUDE_DIRS 
        ${CORE_SOURCE_DIR}/knowhere

        ${CORE_SOURCE_DIR}/thirdparty

        ${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService

        ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include

        ${ARROW_INCLUDE_DIR}

        ${FAISS_INCLUDE_DIR}

        ${OPENBLAS_INCLUDE_DIR}
@@ -129,8 +128,6 @@ set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE) 


#INSTALL(DIRECTORY

#        ${CORE_SOURCE_DIR}/include/knowhere

#        ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons

#        ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons_ext

#        ${ARROW_INCLUDE_DIR}/arrow

#        ${FAISS_PREFIX}/include/faiss

#        ${OPENBLAS_INCLUDE_DIR}/

cpp/src/core/test/test_ivf.cpp

+2 −2
Original line number Diff line number Diff line
@@ -146,8 +146,8 @@ INSTANTIATE_TEST_CASE_P(IVFParameters, IVFTest, 
                        Values(

                            std::make_tuple("IVF", ParameterType::ivf),

                            std::make_tuple("GPUIVF", ParameterType::ivf),

                            std::make_tuple("IVFPQ", ParameterType::ivfpq),

                            std::make_tuple("GPUIVFPQ", ParameterType::ivfpq),

//                            std::make_tuple("IVFPQ", ParameterType::ivfpq),

//                            std::make_tuple("GPUIVFPQ", ParameterType::ivfpq),

                            std::make_tuple("IVFSQ", ParameterType::ivfsq),

                            std::make_tuple("GPUIVFSQ", ParameterType::ivfsq)

                        )

cpp/src/db/engine/ExecutionEngineImpl.cpp

+35 −16
Original line number Diff line number Diff line
@@ -26,6 +26,9 @@ 
#include "src/wrapper/VecIndex.h"

#include "src/wrapper/VecImpl.h"

#include "knowhere/common/Exception.h"

#include "knowhere/common/Config.h"

#include "wrapper/ConfAdapterMgr.h"

#include "wrapper/ConfAdapter.h"

#include "server/Config.h"



#include <stdexcept>
@@ -50,11 +53,15 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, 
        throw Exception(DB_ERROR, "Could not create VecIndex");

    }



    Config build_cfg;

    build_cfg["dim"] = dimension;

    build_cfg["metric_type"] = (metric_type_ == MetricType::IP) ? "IP" : "L2";

    AutoGenParams(index_->GetType(), 0, build_cfg);

    auto ec = std::static_pointer_cast<BFIndex>(index_)->Build(build_cfg);

    TempMetaConf temp_conf;

    temp_conf.gpu_id = gpu_num_;

    temp_conf.dim = dimension;

    temp_conf.metric_type = (metric_type_ == MetricType::IP) ?

                            knowhere::METRICTYPE::IP : knowhere::METRICTYPE::L2;

    auto adapter = AdapterMgr::GetInstance().GetAdapter(index_->GetType());

    auto conf = adapter->Match(temp_conf);



    auto ec = std::static_pointer_cast<BFIndex>(index_)->Build(conf);

    if (ec != KNOWHERE_SUCCESS) {

        throw Exception(DB_ERROR, "Build index error");

    }
@@ -274,17 +281,21 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t 
        throw Exception(DB_ERROR, "Could not create VecIndex");

    }



    Config build_cfg;

    build_cfg["dim"] = Dimension();

    build_cfg["metric_type"] = (metric_type_ == MetricType::IP) ? "IP" : "L2";

    build_cfg["gpu_id"] = gpu_num_;

    build_cfg["nlist"] = nlist_;

    AutoGenParams(to_index->GetType(), Count(), build_cfg);

    TempMetaConf temp_conf;

    temp_conf.gpu_id = gpu_num_;

    temp_conf.dim = Dimension();

    temp_conf.nlist = nlist_;

    temp_conf.metric_type = (metric_type_ == MetricType::IP) ?

        knowhere::METRICTYPE::IP : knowhere::METRICTYPE::L2;

    temp_conf.size = Count();



    auto adapter = AdapterMgr::GetInstance().GetAdapter(to_index->GetType());

    auto conf = adapter->Match(temp_conf);



    auto status = to_index->BuildAll(Count(),

                                     from_index->GetRawVectors(),

                                     from_index->GetRawIds(),

                                 build_cfg);

                                     conf);

    if (!status.ok()) { throw Exception(DB_ERROR, status.message()); }



    return std::make_shared<ExecutionEngineImpl>(to_index, location, engine_type, metric_type_, nlist_);
@@ -302,8 +313,16 @@ Status ExecutionEngineImpl::Search(long n, 
    }



    ENGINE_LOG_DEBUG << "Search Params: [k]  " << k << " [nprobe] " << nprobe;

    auto cfg = Config::object{{"k", k}, {"nprobe", nprobe}};

    auto status = index_->Search(n, data, distances, labels, cfg);



    // TODO(linxj): remove here. Get conf from function

    TempMetaConf temp_conf;

    temp_conf.k = k;

    temp_conf.nprobe = nprobe;



    auto adapter = AdapterMgr::GetInstance().GetAdapter(index_->GetType());

    auto conf = adapter->MatchSearch(temp_conf, index_->GetType());



    auto status = index_->Search(n, data, distances, labels, conf);

    if (!status.ok()) {

        ENGINE_LOG_ERROR << "Search error";

    }

cpp/src/wrapper/ConfAdapter.cpp

0 → 100644
+161 −0
Original line number Diff line number Diff line 
// Licensed to the Apache Software Foundation (ASF) under one

// or more contributor license agreements.  See the NOTICE file

// distributed with this work for additional information

// regarding copyright ownership.  The ASF licenses this file

// to you under the Apache License, Version 2.0 (the

// "License"); you may not use this file except in compliance

// with the License.  You may obtain a copy of the License at

//

//   http://www.apache.org/licenses/LICENSE-2.0

//

// Unless required by applicable law or agreed to in writing,

// software distributed under the License is distributed on an

// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

// KIND, either express or implied.  See the License for the

// specific language governing permissions and limitations

// under the License.





#include <cmath>

#include "ConfAdapter.h"

#include "src/utils/Log.h"

#include "knowhere/index/vector_index/helpers/IndexParameter.h"



// TODO: add conf checker



namespace zilliz {

namespace milvus {

namespace engine {



#if CUDA_VERSION > 9000

#define GPU_MAX_NRPOBE 2048

#else

#define GPU_MAX_NRPOBE 1024

#endif



void

ConfAdapter::MatchBase(knowhere::Config conf) {

    if (conf->metric_type == knowhere::DEFAULT_TYPE)

        conf->metric_type = knowhere::METRICTYPE::L2;

    if (conf->gpu_id == knowhere::INVALID_VALUE)

        conf->gpu_id = 0;

}



knowhere::Config

ConfAdapter::Match(const TempMetaConf &metaconf) {

    auto conf = std::make_shared<knowhere::Cfg>();

    conf->d = metaconf.dim;

    conf->metric_type = metaconf.metric_type;

    conf->gpu_id = conf->gpu_id;

    MatchBase(conf);

    return conf;

}



knowhere::Config

ConfAdapter::MatchSearch(const TempMetaConf &metaconf, const IndexType &type) {

    auto conf = std::make_shared<knowhere::Cfg>();

    conf->k = metaconf.k;

    return conf;

}



knowhere::Config

IVFConfAdapter::Match(const TempMetaConf &metaconf) {

    auto conf = std::make_shared<knowhere::IVFCfg>();

    conf->nlist = MatchNlist(metaconf.size, metaconf.nlist);

    conf->d = metaconf.dim;

    conf->metric_type = metaconf.metric_type;

    conf->gpu_id = conf->gpu_id;

    MatchBase(conf);

    return conf;

}



static constexpr float TYPICAL_COUNT = 1000000.0;



int64_t

IVFConfAdapter::MatchNlist(const int64_t &size, const int64_t &nlist) {

    if (size <= TYPICAL_COUNT / 16384 + 1) {

        // handle less row count, avoid nlist set to 0

        return 1;

    } else if (int(size / TYPICAL_COUNT) * nlist == 0) {

        // calculate a proper nlist if nlist not specified or size less than TYPICAL_COUNT

        return int(size / TYPICAL_COUNT * 16384);

    }

    return 0;

}



knowhere::Config

IVFConfAdapter::MatchSearch(const TempMetaConf &metaconf, const IndexType &type) {

    auto conf = std::make_shared<knowhere::IVFCfg>();

    conf->k = metaconf.k;

    conf->nprobe = metaconf.nprobe;



    switch (type) {

        case IndexType::FAISS_IVFFLAT_GPU:

        case IndexType::FAISS_IVFSQ8_GPU:

        case IndexType::FAISS_IVFPQ_GPU:

            if (conf->nprobe > GPU_MAX_NRPOBE) {

                WRAPPER_LOG_WARNING << "When search with GPU, nprobe shoud be no more than " << GPU_MAX_NRPOBE

                                    << ", but you passed " << conf->nprobe

                                    << ". Search with " << GPU_MAX_NRPOBE << " instead";

                conf->nprobe = GPU_MAX_NRPOBE;



            }

    }

    return conf;

}



knowhere::Config

IVFSQConfAdapter::Match(const TempMetaConf &metaconf) {

    auto conf = std::make_shared<knowhere::IVFSQCfg>();

    conf->nlist = MatchNlist(metaconf.size, metaconf.nlist);

    conf->d = metaconf.dim;

    conf->metric_type = metaconf.metric_type;

    conf->gpu_id = conf->gpu_id;

    conf->nbits = 8;

    MatchBase(conf);

    return conf;

}



knowhere::Config

IVFPQConfAdapter::Match(const TempMetaConf &metaconf) {

    auto conf = std::make_shared<knowhere::IVFPQCfg>();

    conf->nlist = MatchNlist(metaconf.size, metaconf.nlist);

    conf->d = metaconf.dim;

    conf->metric_type = metaconf.metric_type;

    conf->gpu_id = conf->gpu_id;

    conf->nbits = 8;

    conf->m = 8;

    MatchBase(conf);

    return conf;

}



knowhere::Config

NSGConfAdapter::Match(const TempMetaConf &metaconf) {

    auto conf = std::make_shared<knowhere::NSGCfg>();

    conf->nlist = MatchNlist(metaconf.size, metaconf.nlist);

    conf->d = metaconf.dim;

    conf->metric_type = metaconf.metric_type;

    conf->gpu_id = conf->gpu_id;



    auto scale_factor = round(metaconf.dim / 128.0);

    scale_factor = scale_factor >= 4 ? 4 : scale_factor;

    conf->nprobe = 6 + 10 * scale_factor;

    conf->knng = 100 + 100 * scale_factor;

    conf->search_length = 40 + 5 * scale_factor;

    conf->out_degree = 50 + 5 * scale_factor;

    conf->candidate_pool_size = 200 + 100 * scale_factor;

    MatchBase(conf);

    return conf;

}



knowhere::Config

NSGConfAdapter::MatchSearch(const TempMetaConf &metaconf, const IndexType &type) {

    auto conf = std::make_shared<knowhere::NSGCfg>();

    conf->k = metaconf.k;

    conf->search_length = metaconf.search_length;

    return conf;

}



}

}

}

cpp/src/wrapper/ConfAdapter.h

0 → 100644
+91 −0
Original line number Diff line number Diff line 
// Licensed to the Apache Software Foundation (ASF) under one

// or more contributor license agreements.  See the NOTICE file

// distributed with this work for additional information

// regarding copyright ownership.  The ASF licenses this file

// to you under the Apache License, Version 2.0 (the

// "License"); you may not use this file except in compliance

// with the License.  You may obtain a copy of the License at

//

//   http://www.apache.org/licenses/LICENSE-2.0

//

// Unless required by applicable law or agreed to in writing,

// software distributed under the License is distributed on an

// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

// KIND, either express or implied.  See the License for the

// specific language governing permissions and limitations

// under the License.





#pragma once



#include "knowhere/common/Config.h"

#include "VecIndex.h"





namespace zilliz {

namespace milvus {

namespace engine {



// TODO(linxj): remove later, replace with real metaconf

constexpr int64_t TEMPMETA_DEFAULT_VALUE = -1;

struct TempMetaConf {

    int64_t size = TEMPMETA_DEFAULT_VALUE;

    int64_t nlist = TEMPMETA_DEFAULT_VALUE;

    int64_t dim = TEMPMETA_DEFAULT_VALUE;

    int64_t gpu_id = TEMPMETA_DEFAULT_VALUE;

    int64_t k = TEMPMETA_DEFAULT_VALUE;

    int64_t nprobe = TEMPMETA_DEFAULT_VALUE;

    int64_t search_length = TEMPMETA_DEFAULT_VALUE;

    knowhere::METRICTYPE metric_type = knowhere::DEFAULT_TYPE;

};



class ConfAdapter {

 public:

    virtual knowhere::Config

    Match(const TempMetaConf &metaconf);



    virtual knowhere::Config

    MatchSearch(const TempMetaConf &metaconf, const IndexType &type);



 protected:

    static void MatchBase(knowhere::Config conf);

};



using ConfAdapterPtr = std::shared_ptr<ConfAdapter>;



class IVFConfAdapter : public ConfAdapter {

 public:

    knowhere::Config

    Match(const TempMetaConf &metaconf) override;



    knowhere::Config

    MatchSearch(const TempMetaConf &metaconf, const IndexType &type) override;



 protected:

    static int64_t MatchNlist(const int64_t &size, const int64_t &nlist);

};



class IVFSQConfAdapter : public IVFConfAdapter {

 public:

    knowhere::Config

    Match(const TempMetaConf &metaconf) override;

};



class IVFPQConfAdapter : public IVFConfAdapter {

 public:

    knowhere::Config

    Match(const TempMetaConf &metaconf) override;

};



class NSGConfAdapter : public IVFConfAdapter {

 public:

    knowhere::Config

    Match(const TempMetaConf &metaconf) override;



    knowhere::Config

    MatchSearch(const TempMetaConf &metaconf, const IndexType &type) final;

};



}

}

}