Commit cc47ad4f authored by groot's avatar groot
Browse files

#353 Rename config.h.in to version.h.in

parents 29586ba7 fa4f0a07

CHANGELOG.md

+4 −1
Original line number Diff line number Diff line
@@ -8,13 +8,16 @@ Please mark all change in change log and use the ticket from JIRA. 
- \#246 - Exclude src/external folder from code coverage for jenkin ci

- \#248 - Reside src/external in thirdparty

- \#316 - Some files not merged after vectors added

- \#327 - Search does not use GPU when index type is FLAT

- \#340 - Test cases run failed on 0.6.0

- \#353 - Rename config.h.in to version.h.in

- \#374 - sdk_simple return empty result



## Feature

- \#12 - Pure CPU version for Milvus

- \#77 - Support table partition

- \#226 - Experimental shards middleware for Milvus

- \#127 - Support new Index type IVFPQ



## Improvement

- \#275 - Rename C++ SDK IndexType

core/src/db/engine/ExecutionEngine.h

+3 −1
Original line number Diff line number Diff line
@@ -26,6 +26,7 @@ 
namespace milvus {

namespace engine {



// TODO(linxj): replace with VecIndex::IndexType

enum class EngineType {

    INVALID = 0,

    FAISS_IDMAP = 1,
@@ -33,7 +34,8 @@ enum class EngineType { 
    FAISS_IVFSQ8,

    NSG_MIX,

    FAISS_IVFSQ8H,

    MAX_VALUE = FAISS_IVFSQ8H,

    FAISS_PQ,

    MAX_VALUE = FAISS_PQ,

};



enum class MetricType {

core/src/db/engine/ExecutionEngineImpl.cpp

+8 −0
Original line number Diff line number Diff line
@@ -116,6 +116,14 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) { 
            index = GetVecIndexFactory(IndexType::FAISS_IVFSQ8_HYBRID);

            break;

        }

        case EngineType::FAISS_PQ: {

#ifdef MILVUS_CPU_VERSION

            index = GetVecIndexFactory(IndexType::FAISS_IVFPQ_CPU);

#else

            index = GetVecIndexFactory(IndexType::FAISS_IVFPQ_MIX);

#endif

            break;

        }

        default: {

            ENGINE_LOG_ERROR << "Unsupported index type";

            return nullptr;

core/src/grpc/gen-milvus/milvus.pb.h

+41 −16
Original line number Diff line number Diff line
@@ -1321,7 +1321,8 @@ class RowRecord : 
  void clear_vector_data();

  float vector_data(int index) const;

  void set_vector_data(int index, float value);

  void add_vector_data(float value);

//  void add_vector_data(float value);

  void add_vector_data(std::vector<float>::const_iterator begin, std::vector<float>::const_iterator end);

  const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >&

      vector_data() const;

  ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >*
@@ -1473,7 +1474,9 @@ class InsertParam : 
  void clear_row_id_array();

  ::PROTOBUF_NAMESPACE_ID::int64 row_id_array(int index) const;

  void set_row_id_array(int index, ::PROTOBUF_NAMESPACE_ID::int64 value);

  void add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value);

//  void add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value);

  void add_row_id_array(std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator begin,

                        std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator end);

  const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >&

      row_id_array() const;

  ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >*
@@ -2136,7 +2139,9 @@ class TopKQueryResult : 
  void clear_ids();

  ::PROTOBUF_NAMESPACE_ID::int64 ids(int index) const;

  void set_ids(int index, ::PROTOBUF_NAMESPACE_ID::int64 value);

  void add_ids(::PROTOBUF_NAMESPACE_ID::int64 value);

//  void add_ids(::PROTOBUF_NAMESPACE_ID::int64 value);

  void add_ids(std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator begin,

               std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator end);

  const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >&

      ids() const;

  ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >*
@@ -2147,7 +2152,8 @@ class TopKQueryResult : 
  void clear_distances();

  float distances(int index) const;

  void set_distances(int index, float value);

  void add_distances(float value);

//  void add_distances(float value);

  void add_distances(std::vector<float>::const_iterator begin, std::vector<float>::const_iterator end);

  const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >&

      distances() const;

  ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >*
@@ -3922,8 +3928,13 @@ inline void RowRecord::set_vector_data(int index, float value) { 
  vector_data_.Set(index, value);

  // @@protoc_insertion_point(field_set:milvus.grpc.RowRecord.vector_data)

}

inline void RowRecord::add_vector_data(float value) {

  vector_data_.Add(value);

//inline void RowRecord::add_vector_data(float value) {

//  vector_data_.Add(value);

//  // @@protoc_insertion_point(field_add:milvus.grpc.RowRecord.vector_data)

//}

inline void RowRecord::add_vector_data(std::vector<float>::const_iterator begin,

                                       std::vector<float>::const_iterator end) {

    vector_data_.Add(begin, end);

    // @@protoc_insertion_point(field_add:milvus.grpc.RowRecord.vector_data)

}

inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >&
@@ -4037,8 +4048,13 @@ inline void InsertParam::set_row_id_array(int index, ::PROTOBUF_NAMESPACE_ID::in 
  row_id_array_.Set(index, value);

  // @@protoc_insertion_point(field_set:milvus.grpc.InsertParam.row_id_array)

}

inline void InsertParam::add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value) {

  row_id_array_.Add(value);

//inline void InsertParam::add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value) {

//  row_id_array_.Add(value);

//  // @@protoc_insertion_point(field_add:milvus.grpc.InsertParam.row_id_array)

//}

inline void InsertParam::add_row_id_array(std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator begin,

                                          std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator end) {

    row_id_array_.Add(begin, end);

    // @@protoc_insertion_point(field_add:milvus.grpc.InsertParam.row_id_array)

}

inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >&
@@ -4588,8 +4604,13 @@ inline void TopKQueryResult::set_ids(int index, ::PROTOBUF_NAMESPACE_ID::int64 v 
  ids_.Set(index, value);

  // @@protoc_insertion_point(field_set:milvus.grpc.TopKQueryResult.ids)

}

inline void TopKQueryResult::add_ids(::PROTOBUF_NAMESPACE_ID::int64 value) {

  ids_.Add(value);

//inline void TopKQueryResult::add_ids(::PROTOBUF_NAMESPACE_ID::int64 value) {

//  ids_.Add(value);

//  // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.ids)

//}

inline void TopKQueryResult::add_ids(std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator begin,

                                     std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator end) {

    ids_.Add(begin,end);

    // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.ids)

}

inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >&
@@ -4618,8 +4639,12 @@ inline void TopKQueryResult::set_distances(int index, float value) { 
  distances_.Set(index, value);

  // @@protoc_insertion_point(field_set:milvus.grpc.TopKQueryResult.distances)

}

inline void TopKQueryResult::add_distances(float value) {

  distances_.Add(value);

//inline void TopKQueryResult::add_distances(float value) {

//  distances_.Add(value);

//  // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.distances)

//}

inline void TopKQueryResult::add_distances(std::vector<float>::const_iterator begin, std::vector<float>::const_iterator end) {

    distances_.Add(begin, end);

    // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.distances)

}

inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >&

core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp

+21 −12
Original line number Diff line number Diff line
@@ -39,17 +39,19 @@ GPUIVFPQ::Train(const DatasetPtr& dataset, const Config& config) { 


    GETTENSOR(dataset)



    // TODO(linxj): set device here.

    // TODO(linxj): set gpu resource here.

    faiss::gpu::StandardGpuResources res;

    faiss::gpu::GpuIndexIVFPQ device_index(&res, dim, build_cfg->nlist, build_cfg->m, build_cfg->nbits,

    auto temp_resource = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_);

    if (temp_resource != nullptr) {

        ResScope rs(temp_resource, gpu_id_, true);

        auto device_index = new faiss::gpu::GpuIndexIVFPQ(temp_resource->faiss_res.get(), dim, build_cfg->nlist,

                                                          build_cfg->m, build_cfg->nbits,

                                                          GetMetricType(build_cfg->metric_type));  // IP not support

    device_index.train(rows, (float*)p_data);



        device_index->train(rows, (float*)p_data);

        std::shared_ptr<faiss::Index> host_index = nullptr;

    host_index.reset(faiss::gpu::index_gpu_to_cpu(&device_index));



        host_index.reset(faiss::gpu::index_gpu_to_cpu(device_index));

        return std::make_shared<IVFIndexModel>(host_index);

    } else {

        KNOWHERE_THROW_MSG("Build IVFSQ can't get gpu resource");

    }

}



std::shared_ptr<faiss::IVFSearchParameters>
@@ -66,7 +68,14 @@ GPUIVFPQ::GenParams(const Config& config) { 


VectorIndexPtr

GPUIVFPQ::CopyGpuToCpu(const Config& config) {

    KNOWHERE_THROW_MSG("not support yet");

    std::lock_guard<std::mutex> lk(mutex_);



    faiss::Index* device_index = index_.get();

    faiss::Index* host_index = faiss::gpu::index_gpu_to_cpu(device_index);



    std::shared_ptr<faiss::Index> new_index;

    new_index.reset(host_index);

    return std::make_shared<IVFPQ>(new_index);

}



}  // namespace knowhere