Unverified Commit 66e38d20 authored by 蔡宇东's avatar 蔡宇东 Committed by GitHub
Browse files

Caiyd 1881 bad alloc (#1892) 



* #1881 update storage APIs, use int64_t for size

Signed-off-by: default avataryudong.cai <yudong.cai@zilliz.com>

* update changelog

Signed-off-by: default avataryudong.cai <yudong.cai@zilliz.com>
parent cc3b75b6

CHANGELOG.md

+1 −0
Original line number Diff line number Diff line
@@ -7,6 +7,7 @@ Please mark all change in change log and use the issue from GitHub 
## Bug

-   \#1762 Server is not forbidden to create new partition which tag is `_default`

-   \#1873 Fix index file serialize to incorrect path

-   \#1881 Fix Annoy index search fail



## Feature

-   \#261  Integrate ANNOY into Milvus

core/src/codecs/default/DefaultVectorIndexFormat.cpp

+9 −3
Original line number Diff line number Diff line
@@ -36,9 +36,12 @@ DefaultVectorIndexFormat::read_internal(const storage::FSHandlerPtr& fs_ptr, con 
    knowhere::BinarySet load_data_list;



    recorder.RecordSection("Start");

    fs_ptr->reader_ptr_->open(path);

    if (!fs_ptr->reader_ptr_->open(path)) {

        ENGINE_LOG_ERROR << "Fail to open vector index: " << path;

        return nullptr;

    }



    size_t length = fs_ptr->reader_ptr_->length();

    int64_t length = fs_ptr->reader_ptr_->length();

    if (length <= 0) {

        ENGINE_LOG_ERROR << "Invalid vector index length: " << path;

        return nullptr;
@@ -128,7 +131,10 @@ DefaultVectorIndexFormat::write(const storage::FSHandlerPtr& fs_ptr, const std:: 
    int32_t index_type = knowhere::StrToOldIndexType(index->index_type());



    recorder.RecordSection("Start");

    fs_ptr->writer_ptr_->open(location);

    if (!fs_ptr->writer_ptr_->open(location)) {

        ENGINE_LOG_ERROR << "Fail to open vector index: " << location;

        return;

    }



    fs_ptr->writer_ptr_->write(&index_type, sizeof(index_type));

core/src/storage/IOReader.h

+4 −4
Original line number Diff line number Diff line
@@ -19,16 +19,16 @@ namespace storage { 


class IOReader {

 public:

    virtual void

    virtual bool

    open(const std::string& name) = 0;



    virtual void

    read(void* ptr, size_t size) = 0;

    read(void* ptr, int64_t size) = 0;



    virtual void

    seekg(size_t pos) = 0;

    seekg(int64_t pos) = 0;



    virtual size_t

    virtual int64_t

    length() = 0;



    virtual void

core/src/storage/IOWriter.h

+3 −3
Original line number Diff line number Diff line
@@ -19,13 +19,13 @@ namespace storage { 


class IOWriter {

 public:

    virtual void

    virtual bool

    open(const std::string& name) = 0;



    virtual void

    write(void* ptr, size_t size) = 0;

    write(void* ptr, int64_t size) = 0;



    virtual size_t

    virtual int64_t

    length() = 0;



    virtual void

core/src/storage/disk/DiskIOReader.cpp

+6 −5
Original line number Diff line number Diff line
@@ -14,26 +14,27 @@ 
namespace milvus {

namespace storage {



void

bool

DiskIOReader::open(const std::string& name) {

    name_ = name;

    fs_ = std::fstream(name_, std::ios::in | std::ios::binary);

    return fs_.good();

}



void

DiskIOReader::read(void* ptr, size_t size) {

DiskIOReader::read(void* ptr, int64_t size) {

    fs_.read(reinterpret_cast<char*>(ptr), size);

}



void

DiskIOReader::seekg(size_t pos) {

DiskIOReader::seekg(int64_t pos) {

    fs_.seekg(pos);

}



size_t

int64_t

DiskIOReader::length() {

    fs_.seekg(0, fs_.end);

    size_t len = fs_.tellg();

    int64_t len = fs_.tellg();

    fs_.seekg(0, fs_.beg);

    return len;

}