Commit d599e596 authored by peng.xu's avatar peng.xu
Browse files

Merge branch 'branch-0.5.0' into '0.5.0' 

#39 Create SQ8H index hang if using github server version

See merge request megasearch/milvus!762

Former-commit-id: cfd239ddc25a9222dd2740c629b7c35aae0b8932
parents 0741b76b 6b475613

CHANGELOG.md

+1 −0
Original line number Diff line number Diff line
@@ -29,6 +29,7 @@ Please mark all change in change log and use the ticket from JIRA. 
- \#20 - C++ sdk example get grpc error 

- \#23 - Add unittest to improve code coverage

- \#31 - make clang-format failed after run build.sh -l

- \#39 - Create SQ8H index hang if using github server version



## Improvement

- MS-552 - Add and change the easylogging library

core/src/db/engine/ExecutionEngineImpl.cpp

+3 −3
Original line number Diff line number Diff line
@@ -65,7 +65,7 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, const std::string& 
    : location_(location), dim_(dimension), index_type_(index_type), metric_type_(metric_type), nlist_(nlist) {

    index_ = CreatetVecIndex(EngineType::FAISS_IDMAP);

    if (!index_) {

        throw Exception(DB_ERROR, "Could not create VecIndex");

        throw Exception(DB_ERROR, "Unsupported index type");

    }



    TempMetaConf temp_conf;
@@ -111,7 +111,7 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) { 
            break;

        }

        default: {

            ENGINE_LOG_ERROR << "Invalid engine type";

            ENGINE_LOG_ERROR << "Unsupported index type";

            return nullptr;

        }

    }
@@ -373,7 +373,7 @@ ExecutionEngineImpl::BuildIndex(const std::string& location, EngineType engine_t 


    auto to_index = CreatetVecIndex(engine_type);

    if (!to_index) {

        throw Exception(DB_ERROR, "Could not create VecIndex");

        throw Exception(DB_ERROR, "Unsupported index type");

    }



    TempMetaConf temp_conf;

core/src/scheduler/task/BuildIndexTask.cpp

+6 −0
Original line number Diff line number Diff line
@@ -134,6 +134,7 @@ XBuildIndexTask::Execute() { 
                ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_

                                 << " to to_delete";



                build_index_job->BuildIndexDone(to_index_id_);

                to_index_engine_ = nullptr;

                return;

            }
@@ -148,6 +149,7 @@ XBuildIndexTask::Execute() { 
            std::cout << "ERROR: failed to build index, index file is too large or gpu memory is not enough"

                      << std::endl;



            build_index_job->BuildIndexDone(to_index_id_);

            build_index_job->GetStatus() = Status(DB_ERROR, msg);

            to_index_engine_ = nullptr;

            return;
@@ -158,6 +160,9 @@ XBuildIndexTask::Execute() { 
        meta_ptr->HasTable(file_->table_id_, has_table);

        if (!has_table) {

            meta_ptr->DeleteTableFiles(file_->table_id_);



            build_index_job->BuildIndexDone(to_index_id_);

            build_index_job->GetStatus() = Status(DB_ERROR, "Table has been deleted, discard index file.");

            to_index_engine_ = nullptr;

            return;

        }
@@ -177,6 +182,7 @@ XBuildIndexTask::Execute() { 
            std::cout << "ERROR: failed to persist index file: " << table_file.location_

                      << ", possible out of disk space" << std::endl;



            build_index_job->BuildIndexDone(to_index_id_);

            build_index_job->GetStatus() = Status(DB_ERROR, msg);

            to_index_engine_ = nullptr;

            return;

core/src/utils/ValidationUtil.cpp

+9 −0
Original line number Diff line number Diff line
@@ -94,6 +94,15 @@ ValidationUtil::ValidateTableIndexType(int32_t index_type) { 
        return Status(SERVER_INVALID_INDEX_TYPE, msg);

    }



#ifndef CUSTOMIZATION

    // special case, hybird index only available in customize faiss library

    if (engine_type == static_cast<int>(engine::EngineType::FAISS_IVFSQ8H)) {

        std::string msg = "Unsupported index type: " + std::to_string(index_type);

        SERVER_LOG_ERROR << msg;

        return Status(SERVER_INVALID_INDEX_TYPE, msg);

    }

#endif



    return Status::OK();

}