Unverified Commit 244d0e34 authored by groot's avatar groot Committed by GitHub
Browse files

Remove unnecessary memcpy (#2337) 



* fix #2264

Signed-off-by: default avataryhmo <yihua.mo@zilliz.com>

* typo

Signed-off-by: default avataryhmo <yihua.mo@zilliz.com>

* fix ut

Signed-off-by: default avataryhmo <yihua.mo@zilliz.com>

* typo

Signed-off-by: default avataryhmo <yihua.mo@zilliz.com>

* Remove unnecessary memcpy

Signed-off-by: default avataryhmo <yihua.mo@zilliz.com>
parent 312df5af

core/src/db/insert/VectorSource.cpp

+11 −28
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ 
#include "db/engine/ExecutionEngine.h"

#include "metrics/Metrics.h"

#include "utils/Log.h"

#include "utils/TimeRecorder.h"



namespace milvus {

namespace engine {
@@ -36,9 +37,8 @@ VectorSource::VectorSource(milvus::engine::VectorsData vectors, 
}



Status

VectorSource::Add(/*const ExecutionEnginePtr& execution_engine,*/ const segment::SegmentWriterPtr& segment_writer_ptr,

                  const meta::SegmentSchema& table_file_schema, const size_t& num_vectors_to_add,

                  size_t& num_vectors_added) {

VectorSource::Add(const segment::SegmentWriterPtr& segment_writer_ptr, const meta::SegmentSchema& table_file_schema,

                  const size_t& num_vectors_to_add, size_t& num_vectors_added) {

    uint64_t n = vectors_.vector_count_;

    server::CollectAddMetrics metrics(n, table_file_schema.dimension_);
@@ -61,35 +61,17 @@ VectorSource::Add(/*const ExecutionEnginePtr& execution_engine,*/ const segment: 


    Status status;

    if (!vectors_.float_data_.empty()) {

        /*

        status = execution_engine->AddWithIds(

            num_vectors_added, vectors_.float_data_.data() + current_num_vectors_added * table_file_schema.dimension_,

            vector_ids_to_add.data());

        */

        std::vector<uint8_t> vectors;

        LOG_ENGINE_DEBUG_ << LogOut("[%s][%ld]", "insert", 0) << "Insert float data into segment";

        auto size = num_vectors_added * table_file_schema.dimension_ * sizeof(float);

        vectors.resize(size);

        memcpy(vectors.data(), vectors_.float_data_.data() + current_num_vectors_added * table_file_schema.dimension_,

               size);

        LOG_ENGINE_DEBUG_ << LogOut("[%s][%ld]", "insert", 0) << "Insert into segment";

        status = segment_writer_ptr->AddVectors(table_file_schema.file_id_, vectors, vector_ids_to_add);



        float* ptr = vectors_.float_data_.data() + current_num_vectors_added * table_file_schema.dimension_;

        status = segment_writer_ptr->AddVectors(table_file_schema.file_id_, (uint8_t*)ptr, size, vector_ids_to_add);

    } else if (!vectors_.binary_data_.empty()) {

        /*

        status = execution_engine->AddWithIds(

            num_vectors_added,

            vectors_.binary_data_.data() + current_num_vectors_added * SingleVectorSize(table_file_schema.dimension_),

            vector_ids_to_add.data());

        */

        LOG_ENGINE_DEBUG_ << LogOut("[%s][%ld]", "insert", 0) << "Insert binary data into segment";

        std::vector<uint8_t> vectors;

        auto size = num_vectors_added * SingleVectorSize(table_file_schema.dimension_) * sizeof(uint8_t);

        vectors.resize(size);

        memcpy(

            vectors.data(),

            vectors_.binary_data_.data() + current_num_vectors_added * SingleVectorSize(table_file_schema.dimension_),

            size);

        LOG_ENGINE_DEBUG_ << LogOut("[%s][%ld]", "insert", 0) << "Insert into segment";

        status = segment_writer_ptr->AddVectors(table_file_schema.file_id_, vectors, vector_ids_to_add);

        uint8_t* ptr =

            vectors_.binary_data_.data() + current_num_vectors_added * SingleVectorSize(table_file_schema.dimension_);

        status = segment_writer_ptr->AddVectors(table_file_schema.file_id_, ptr, size, vector_ids_to_add);

    }



    // Clear vector data
@@ -175,6 +157,7 @@ VectorSource::SingleVectorSize(uint16_t dimension) { 


    return 0;

}



size_t

VectorSource::SingleEntitySize(uint16_t dimension) {

    // TODO(yukun) add entity type and size compute

core/src/db/insert/VectorSource.h

+2 −2
Original line number Diff line number Diff line
@@ -36,8 +36,8 @@ class VectorSource { 
                 const std::unordered_map<std::string, std::vector<uint8_t>>& attr_data);



    Status

    Add(/*const ExecutionEnginePtr& execution_engine,*/ const segment::SegmentWriterPtr& segment_writer_ptr,

        const meta::SegmentSchema& table_file_schema, const size_t& num_vectors_to_add, size_t& num_vectors_added);

    Add(const segment::SegmentWriterPtr& segment_writer_ptr, const meta::SegmentSchema& table_file_schema,

        const size_t& num_vectors_to_add, size_t& num_vectors_added);



    Status

    AddEntities(const segment::SegmentWriterPtr& segment_writer_ptr, const meta::SegmentSchema& collection_file_schema,

core/src/db/merge/MergeLayeredStrategy.cpp

+1 −1
Original line number Diff line number Diff line
@@ -20,7 +20,7 @@ 
namespace milvus {

namespace engine {



const int64_t FORCE_MERGE_THREASHOLD = 10;  // force merge files older this time(in second)

const int64_t FORCE_MERGE_THREASHOLD = 30;  // force merge files older this time(in second)



Status

MergeLayeredStrategy::RegroupFiles(meta::FilesHolder& files_holder, MergeFilesGroups& files_groups) {

core/src/segment/SegmentWriter.cpp

+10 −0
Original line number Diff line number Diff line
@@ -52,6 +52,16 @@ SegmentWriter::AddVectors(const std::string& name, const std::vector<uint8_t>& d 
    return Status::OK();

}



Status

SegmentWriter::AddVectors(const std::string& name, const uint8_t* data, uint64_t size,

                          const std::vector<doc_id_t>& uids) {

    segment_ptr_->vectors_ptr_->AddData(data, size);

    segment_ptr_->vectors_ptr_->AddUids(uids);

    segment_ptr_->vectors_ptr_->SetName(name);



    return Status::OK();

}



Status

SegmentWriter::AddAttrs(const std::string& name, const std::unordered_map<std::string, uint64_t>& attr_nbytes,

                        const std::unordered_map<std::string, std::vector<uint8_t>>& attr_data,

core/src/segment/SegmentWriter.h

+3 −0
Original line number Diff line number Diff line
@@ -36,6 +36,9 @@ class SegmentWriter { 
    Status

    AddVectors(const std::string& name, const std::vector<uint8_t>& data, const std::vector<doc_id_t>& uids);



    Status

    AddVectors(const std::string& name, const uint8_t* data, uint64_t size, const std::vector<doc_id_t>& uids);



    Status

    AddAttrs(const std::string& name, const std::unordered_map<std::string, uint64_t>& attr_nbytes,

             const std::unordered_map<std::string, std::vector<uint8_t>>& attr_data, const std::vector<doc_id_t>& uids);