Unverified Commit bf6d22e2 authored by 蔡宇东's avatar 蔡宇东 Committed by GitHub
Browse files

#1873 fix index file serialize to incorrect path (#1874) 



* #1873 fix index file serialize to incorrect path

Signed-off-by: default avataryudong.cai <yudong.cai@zilliz.com>

* not create sq8h index when gpu disabled

Signed-off-by: default avataryudong.cai <yudong.cai@zilliz.com>
parent c6f4660b

CHANGELOG.md

+2 −1
Original line number Diff line number Diff line
@@ -5,7 +5,8 @@ Please mark all change in change log and use the issue from GitHub 
# Milvus 0.8.0 (TBD)



## Bug

-   \#1762 Server is not forbidden to create new partition which tag is "_default"

-   \#1762 Server is not forbidden to create new partition which tag is `_default`

-   \#1873 Fix index file serialize to incorrect path



## Feature

-   \#261  Integrate ANNOY into Milvus

core/src/codecs/VectorIndexFormat.h

+3 −2
Original line number Diff line number Diff line
@@ -29,10 +29,11 @@ namespace codec { 
class VectorIndexFormat {

 public:

    virtual void

    read(const storage::FSHandlerPtr& fs_ptr, segment::VectorIndexPtr& vector_index) = 0;

    read(const storage::FSHandlerPtr& fs_ptr, const std::string& location, segment::VectorIndexPtr& vector_index) = 0;



    virtual void

    write(const storage::FSHandlerPtr& fs_ptr, const segment::VectorIndexPtr& vector_index) = 0;

    write(const storage::FSHandlerPtr& fs_ptr, const std::string& location,

          const segment::VectorIndexPtr& vector_index) = 0;

};



using VectorIndexFormatPtr = std::shared_ptr<VectorIndexFormat>;

core/src/codecs/default/DefaultVectorIndexFormat.cpp

+8 −32
Original line number Diff line number Diff line
@@ -98,7 +98,8 @@ DefaultVectorIndexFormat::read_internal(const storage::FSHandlerPtr& fs_ptr, con 
}



void

DefaultVectorIndexFormat::read(const storage::FSHandlerPtr& fs_ptr, segment::VectorIndexPtr& vector_index) {

DefaultVectorIndexFormat::read(const storage::FSHandlerPtr& fs_ptr, const std::string& location,

                               segment::VectorIndexPtr& vector_index) {

    const std::lock_guard<std::mutex> lock(mutex_);



    std::string dir_path = fs_ptr->operation_ptr_->GetDirectory();
@@ -108,42 +109,17 @@ DefaultVectorIndexFormat::read(const storage::FSHandlerPtr& fs_ptr, segment::Vec 
        throw Exception(SERVER_INVALID_ARGUMENT, err_msg);

    }



    boost::filesystem::path target_path(dir_path);

    typedef boost::filesystem::directory_iterator d_it;

    d_it it_end;

    d_it it(target_path);



    for (; it != it_end; ++it) {

        const auto& path = it->path();



        // if (path.extension().string() == vector_index_extension_) {

        /* tmp solution, should be replaced when use .idx as index extension name */

        const std::string& location = path.string();

        if (location.substr(location.length() - 3) == "000") {

    knowhere::VecIndexPtr index = read_internal(fs_ptr, location);

    vector_index->SetVectorIndex(index);

            vector_index->SetName(path.stem().string());

            return;

        }

    }

}



std::string

GenerateFileName() {

    auto now = std::chrono::system_clock::now();

    auto micros = std::chrono::duration_cast<std::chrono::microseconds>(now.time_since_epoch()).count();

    return std::to_string(micros * 1000);

}



void

DefaultVectorIndexFormat::write(const storage::FSHandlerPtr& fs_ptr, const segment::VectorIndexPtr& vector_index) {

DefaultVectorIndexFormat::write(const storage::FSHandlerPtr& fs_ptr, const std::string& location,

                                const segment::VectorIndexPtr& vector_index) {

    const std::lock_guard<std::mutex> lock(mutex_);



    std::string dir_path = fs_ptr->operation_ptr_->GetDirectory();



    const std::string index_file_path = dir_path + "/" + GenerateFileName();

    // const std::string index_file_path = dir_path + "/" + vector_index->GetName() + vector_index_extension_;



    milvus::TimeRecorder recorder("write_index");



    knowhere::VecIndexPtr index = vector_index->GetVectorIndex();
@@ -152,7 +128,7 @@ DefaultVectorIndexFormat::write(const storage::FSHandlerPtr& fs_ptr, const segme 
    int32_t index_type = knowhere::StrToOldIndexType(index->index_type());



    recorder.RecordSection("Start");

    fs_ptr->writer_ptr_->open(index_file_path);

    fs_ptr->writer_ptr_->open(location);



    fs_ptr->writer_ptr_->write(&index_type, sizeof(index_type));
@@ -171,7 +147,7 @@ DefaultVectorIndexFormat::write(const storage::FSHandlerPtr& fs_ptr, const segme 


    double span = recorder.RecordSection("End");

    double rate = fs_ptr->writer_ptr_->length() * 1000000.0 / span / 1024 / 1024;

    ENGINE_LOG_DEBUG << "write_index(" << index_file_path << ") rate " << rate << "MB/s";

    ENGINE_LOG_DEBUG << "write_index(" << location << ") rate " << rate << "MB/s";

}



}  // namespace codec

core/src/codecs/default/DefaultVectorIndexFormat.h

+4 −2
Original line number Diff line number Diff line
@@ -30,10 +30,12 @@ class DefaultVectorIndexFormat : public VectorIndexFormat { 
    DefaultVectorIndexFormat() = default;



    void

    read(const storage::FSHandlerPtr& fs_ptr, segment::VectorIndexPtr& vector_index) override;

    read(const storage::FSHandlerPtr& fs_ptr, const std::string& location,

         segment::VectorIndexPtr& vector_index) override;



    void

    write(const storage::FSHandlerPtr& fs_ptr, const segment::VectorIndexPtr& vector_index) override;

    write(const storage::FSHandlerPtr& fs_ptr, const std::string& location,

          const segment::VectorIndexPtr& vector_index) override;



    // No copy and move

    DefaultVectorIndexFormat(const DefaultVectorIndexFormat&) = delete;

core/src/db/engine/ExecutionEngineImpl.cpp

+2 −2
Original line number Diff line number Diff line
@@ -358,7 +358,7 @@ ExecutionEngineImpl::Serialize() { 
    utils::GetParentPath(location_, segment_dir);

    auto segment_writer_ptr = std::make_shared<segment::SegmentWriter>(segment_dir);

    segment_writer_ptr->SetVectorIndex(index_);

    segment_writer_ptr->WriteVectorIndex();

    segment_writer_ptr->WriteVectorIndex(location_);



    // here we reset index size by file size,

    // since some index type(such as SQ8) data size become smaller after serialized
@@ -443,7 +443,7 @@ ExecutionEngineImpl::Load(bool to_cache) { 
            try {

                segment::SegmentPtr segment_ptr;

                segment_reader_ptr->GetSegment(segment_ptr);

                auto status = segment_reader_ptr->LoadVectorIndex(segment_ptr->vector_index_ptr_);

                auto status = segment_reader_ptr->LoadVectorIndex(location_, segment_ptr->vector_index_ptr_);

                index_ = segment_ptr->vector_index_ptr_->GetVectorIndex();



                if (index_ == nullptr) {