Unverified Commit db75a9ae authored by Jin Hai's avatar Jin Hai Committed by GitHub
Browse files

Merge pull request #335 from fishpenguin/0.6.0-yk 

memory usage increased slowly during searching vectors
parents 2e62a052 4840aa33

CHANGELOG.md

+1 −0
Original line number Diff line number Diff line
@@ -4,6 +4,7 @@ Please mark all change in change log and use the ticket from JIRA. 
# Milvus 0.6.0 (TODO)



## Bug

- \#228 - memory usage increased slowly during searching vectors

- \#246 - Exclude src/external folder from code coverage for jenkin ci

- \#248 - Reside src/external in thirdparty

core/src/index/knowhere/knowhere/adapter/SptagAdapter.cpp

+29 −28
Original line number Diff line number Diff line
@@ -89,34 +89,35 @@ ConvertToDataset(std::vector<SPTAG::QueryResult> query_results) { 
        }

    }



    auto id_buf = MakeMutableBufferSmart((uint8_t*)p_id, sizeof(int64_t) * elems);

    auto dist_buf = MakeMutableBufferSmart((uint8_t*)p_dist, sizeof(float) * elems);



    // TODO: magic

    std::vector<BufferPtr> id_bufs{nullptr, id_buf};

    std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};



    auto int64_type = std::make_shared<arrow::Int64Type>();

    auto float_type = std::make_shared<arrow::FloatType>();



    auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);

    auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);

    //    auto id_array_data = std::make_shared<ArrayData>(int64_type, sizeof(int64_t) * elems, id_bufs);

    //    auto dist_array_data = std::make_shared<ArrayData>(float_type, sizeof(float) * elems, dist_bufs);



    //    auto ids = ConstructInt64Array((uint8_t*)p_id, sizeof(int64_t) * elems);

    //    auto dists = ConstructFloatArray((uint8_t*)p_dist, sizeof(float) * elems);



    auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);

    auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);

    std::vector<ArrayPtr> array{ids, dists};



    auto field_id = std::make_shared<Field>("id", std::make_shared<arrow::Int64Type>());

    auto field_dist = std::make_shared<Field>("dist", std::make_shared<arrow::FloatType>());

    std::vector<FieldPtr> fields{field_id, field_dist};

    auto schema = std::make_shared<Schema>(fields);



    return std::make_shared<Dataset>(array, schema);

    //    auto id_buf = MakeMutableBufferSmart((uint8_t*)p_id, sizeof(int64_t) * elems);

    //    auto dist_buf = MakeMutableBufferSmart((uint8_t*)p_dist, sizeof(float) * elems);

    //

    //    // TODO: magic

    //    std::vector<BufferPtr> id_bufs{nullptr, id_buf};

    //    std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};

    //

    //    auto int64_type = std::make_shared<arrow::Int64Type>();

    //    auto float_type = std::make_shared<arrow::FloatType>();

    //

    //    auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);

    //    auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);

    //    //    auto id_array_data = std::make_shared<ArrayData>(int64_type, sizeof(int64_t) * elems, id_bufs);

    //    //    auto dist_array_data = std::make_shared<ArrayData>(float_type, sizeof(float) * elems, dist_bufs);

    //

    //    //    auto ids = ConstructInt64Array((uint8_t*)p_id, sizeof(int64_t) * elems);

    //    //    auto dists = ConstructFloatArray((uint8_t*)p_dist, sizeof(float) * elems);

    //

    //    auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);

    //    auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);

    //    std::vector<ArrayPtr> array{ids, dists};

    //

    //    auto field_id = std::make_shared<Field>("id", std::make_shared<arrow::Int64Type>());

    //    auto field_dist = std::make_shared<Field>("dist", std::make_shared<arrow::FloatType>());

    //    std::vector<FieldPtr> fields{field_id, field_dist};

    //    auto schema = std::make_shared<Schema>(fields);

    //

    //    return std::make_shared<Dataset>(array, schema);

    return std::make_shared<Dataset>((void*)p_id, (void*)p_dist);

}



}  // namespace knowhere

core/src/index/knowhere/knowhere/common/Dataset.h

+16 −0
Original line number Diff line number Diff line
@@ -54,6 +54,9 @@ class Dataset { 
        : tensor_(std::move(tensor)), tensor_schema_(std::move(tensor_schema)) {

    }



    Dataset(void* ids, void* dists) : ids_(ids), dists_(dists) {

    }



    Dataset(const Dataset&) = delete;

    Dataset&

    operator=(const Dataset&) = delete;
@@ -128,6 +131,16 @@ class Dataset { 
        tensor_schema_ = std::move(tensor_schema);

    }



    void*

    ids() {

        return ids_;

    }



    void*

    dist() {

        return dists_;

    }



    // const Config &

    // meta() const { return meta_; }
@@ -141,6 +154,9 @@ class Dataset { 
    SchemaPtr array_schema_;

    std::vector<TensorPtr> tensor_;

    SchemaPtr tensor_schema_;

    // TODO(yukun): using smart pointer

    void* ids_;

    void* dists_;

    // Config meta_;

};

core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp

+18 −17
Original line number Diff line number Diff line
@@ -80,23 +80,24 @@ IDMAP::Search(const DatasetPtr& dataset, const Config& config) { 


    search_impl(rows, (float*)p_data, config->k, res_dis, res_ids, Config());



    auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems);

    auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems);



    std::vector<BufferPtr> id_bufs{nullptr, id_buf};

    std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};



    auto int64_type = std::make_shared<arrow::Int64Type>();

    auto float_type = std::make_shared<arrow::FloatType>();



    auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);

    auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);



    auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);

    auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);

    std::vector<ArrayPtr> array{ids, dists};



    return std::make_shared<Dataset>(array, nullptr);

    //    auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems);

    //    auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems);

    //

    //    std::vector<BufferPtr> id_bufs{nullptr, id_buf};

    //    std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};

    //

    //    auto int64_type = std::make_shared<arrow::Int64Type>();

    //    auto float_type = std::make_shared<arrow::FloatType>();

    //

    //    auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);

    //    auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);

    //

    //    auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);

    //    auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);

    //    std::vector<ArrayPtr> array{ids, dists};

    //

    //    return std::make_shared<Dataset>(array, nullptr);

    return std::make_shared<Dataset>((void*)res_ids, (void*)res_dis);

}



void

core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp

+17 −17
Original line number Diff line number Diff line
@@ -139,23 +139,23 @@ IVF::Search(const DatasetPtr& dataset, const Config& config) { 
    //    std::cout << ss_res_id.str() << std::endl;

    //    std::cout << ss_res_dist.str() << std::endl << std::endl;



    auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems);

    auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems);



    std::vector<BufferPtr> id_bufs{nullptr, id_buf};

    std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};



    auto int64_type = std::make_shared<arrow::Int64Type>();

    auto float_type = std::make_shared<arrow::FloatType>();



    auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);

    auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);



    auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);

    auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);

    std::vector<ArrayPtr> array{ids, dists};

    //    auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems);

    //    auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems);

    //

    //    std::vector<BufferPtr> id_bufs{nullptr, id_buf};

    //    std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};

    //

    //    auto int64_type = std::make_shared<arrow::Int64Type>();

    //    auto float_type = std::make_shared<arrow::FloatType>();

    //

    //    auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);

    //    auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);

    //

    //    auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);

    //    auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);

    //    std::vector<ArrayPtr> array{ids, dists};



    return std::make_shared<Dataset>(array, nullptr);

    return std::make_shared<Dataset>((void*)res_ids, (void*)res_dis);

}



void