Commit f330d9f9 authored by 蔡宇东's avatar 蔡宇东
Browse files

#658 Milvus error out when building SQ8H index without GPU resources

parent c974392b

CHANGELOG.md

+1 −0
Original line number Diff line number Diff line
@@ -47,6 +47,7 @@ Please mark all change in change log and use the ticket from JIRA. 
- \#631 - FAISS isn't compiled with O3 option

- \#649 - Typo "partiton" should be "partition"

- \#654 - Random crash when frequently insert vector one by one

- \#658 - Milvus error out when building SQ8H index without GPU resources



## Feature

- \#12 - Pure CPU version for Milvus

core/src/db/engine/ExecutionEngineImpl.cpp

+5 −1
Original line number Diff line number Diff line
@@ -121,7 +121,11 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) { 
        }

#ifdef CUSTOMIZATION

        case EngineType::FAISS_IVFSQ8H: {

            if (gpu_resource_enable) {

                index = GetVecIndexFactory(IndexType::FAISS_IVFSQ8_HYBRID);

            } else {

                throw Exception(DB_ERROR, "No GPU resources for IVFSQ8H");

            }

            break;

        }

#endif

core/src/scheduler/task/BuildIndexTask.cpp

+4 −12
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ 
#include "db/engine/EngineFactory.h"

#include "metrics/Metrics.h"

#include "scheduler/job/BuildIndexJob.h"

#include "utils/Exception.h"

#include "utils/Log.h"

#include "utils/TimeRecorder.h"
@@ -129,24 +130,15 @@ XBuildIndexTask::Execute() { 
        try {

            index = to_index_engine_->BuildIndex(table_file.location_, (EngineType)table_file.engine_type_);

            if (index == nullptr) {

                table_file.file_type_ = engine::meta::TableFileSchema::TO_DELETE;

                status = meta_ptr->UpdateTableFile(table_file);

                ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_

                                 << " to to_delete";



                build_index_job->BuildIndexDone(to_index_id_);

                to_index_engine_ = nullptr;

                return;

                throw Exception(DB_ERROR, "index NULL");

            }

        } catch (std::exception& ex) {

            std::string msg = "BuildIndex encounter exception: " + std::string(ex.what());

            std::string msg = "Build index exception: " + std::string(ex.what());

            ENGINE_LOG_ERROR << msg;



            table_file.file_type_ = engine::meta::TableFileSchema::TO_DELETE;

            status = meta_ptr->UpdateTableFile(table_file);

            ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete";



            ENGINE_LOG_ERROR << "Failed to build index, index file is too large or gpu memory is not enough";

            ENGINE_LOG_DEBUG << "Build index fail, mark file: " << table_file.file_id_ << " to to_delete";



            build_index_job->BuildIndexDone(to_index_id_);

            build_index_job->GetStatus() = Status(DB_ERROR, msg);

core/src/sdk/examples/partition/src/ClientTest.cpp

+2 −5
Original line number Diff line number Diff line
@@ -17,15 +17,12 @@ 


#include "sdk/examples/partition/src/ClientTest.h"

#include "MilvusApi.h"

#include "sdk/examples/utils/TimeRecorder.h"

#include "sdk/examples/utils/Utils.h"



#include <src/sdk/examples/utils/TimeRecorder.h>

#include <time.h>

#include <unistd.h>

#include <chrono>

#include <iostream>

#include <memory>

#include <thread>

#include <utility>

#include <vector>
@@ -42,7 +39,7 @@ constexpr int64_t TOP_K = 10; 
constexpr int64_t NPROBE = 32;

constexpr int64_t SEARCH_TARGET = 5000;  // change this value, result is different

constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8;

constexpr int32_t N_LIST = 15000;

constexpr int32_t N_LIST = 16384;

constexpr int32_t PARTITION_COUNT = 5;

constexpr int32_t TARGET_PARTITION = 3;

core/src/sdk/examples/simple/src/ClientTest.cpp

+1 −4
Original line number Diff line number Diff line
@@ -20,11 +20,8 @@ 
#include "sdk/examples/utils/TimeRecorder.h"

#include "sdk/examples/utils/Utils.h"



#include <time.h>

#include <unistd.h>

#include <iostream>

#include <memory>

#include <thread>

#include <utility>

#include <vector>
@@ -42,7 +39,7 @@ constexpr int64_t NPROBE = 32; 
constexpr int64_t SEARCH_TARGET = 5000;  // change this value, result is different

constexpr int64_t ADD_VECTOR_LOOP = 5;

constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8;

constexpr int32_t N_LIST = 15000;

constexpr int32_t N_LIST = 16384;



milvus::TableSchema

BuildTableSchema() {