Loading cpp/src/db/DBImpl.cpp +22 −23 Original line number Diff line number Diff line Loading @@ -898,34 +898,33 @@ DBImpl::BackgroundBuildIndex() { meta_ptr_->FilesToIndex(to_index_files); Status status; // scheduler::BuildIndexJobPtr // job = std::make_shared<scheduler::BuildIndexJob>(0, meta_ptr_, options_); scheduler::BuildIndexJobPtr job = std::make_shared<scheduler::BuildIndexJob>(0, meta_ptr_, options_); // step 2: put build index task to scheduler // for (auto &file : to_index_files) { // scheduler::TableFileSchemaPtr file_ptr = std::make_shared<meta::TableFileSchema>(file); // job->AddToIndexFiles(file_ptr); // } // scheduler::JobMgrInst::GetInstance()->Put(job); // job->WaitBuildIndexFinish(); // if (!job->GetStatus().ok()) { // Status status = job->GetStatus(); // ENGINE_LOG_ERROR << "Building index failed: " << status.ToString(); // } for (auto &file : to_index_files) { std::cout << "get to index file" << std::endl; status = BuildIndex(file); if (!status.ok()) { ENGINE_LOG_ERROR << "Building index for " << file.id_ << " failed: " << status.ToString(); } if (shutting_down_.load(std::memory_order_acquire)) { ENGINE_LOG_DEBUG << "Server will shutdown, skip build index action"; break; scheduler::TableFileSchemaPtr file_ptr = std::make_shared<meta::TableFileSchema>(file); job->AddToIndexFiles(file_ptr); } scheduler::JobMgrInst::GetInstance()->Put(job); job->WaitBuildIndexFinish(); if (!job->GetStatus().ok()) { Status status = job->GetStatus(); ENGINE_LOG_ERROR << "Building index failed: " << status.ToString(); } // for (auto &file : to_index_files) { // status = BuildIndex(file); // if (!status.ok()) { // ENGINE_LOG_ERROR << "Building index for " << file.id_ << " failed: " << status.ToString(); // } // // if (shutting_down_.load(std::memory_order_acquire)) { // ENGINE_LOG_DEBUG << "Server will shutdown, skip build index action"; // break; // } // } ENGINE_LOG_TRACE << "Background build index thread exit"; } Loading cpp/src/scheduler/job/BuildIndexJob.cpp +3 −6 Original line number Diff line number Diff line Loading @@ -15,16 +15,14 @@ // specific language governing permissions and limitations // under the License. #include "BuildIndexJob.h" #include "utils/Log.h" #include "BuildIndexJob.h" namespace milvus { namespace scheduler { BuildIndexJob::BuildIndexJob(JobId id, engine::meta::MetaPtr meta_ptr, engine::DBOptions options) : Job(id, JobType::BUILD), meta_ptr_(std::move(meta_ptr)), options_(std::move(options)) { } bool Loading Loading @@ -54,6 +52,5 @@ BuildIndexJob::BuildIndexDone(size_t to_index_id) { SERVER_LOG_DEBUG << "BuildIndexJob " << id() << " finish index file: " << to_index_id; } } } No newline at end of file } // namespace scheduler } // namespace milvus cpp/src/scheduler/job/BuildIndexJob.h +2 −2 Original line number Diff line number Diff line Loading @@ -87,5 +87,5 @@ class BuildIndexJob : public Job { using BuildIndexJobPtr = std::shared_ptr<BuildIndexJob>; } } } // namespace scheduler } // namespace milvus cpp/src/scheduler/task/BuildIndexTask.cpp +7 −5 Original line number Diff line number Diff line Loading @@ -26,6 +26,7 @@ #include <thread> #include <utility> namespace milvus { namespace scheduler { Loading Loading @@ -116,7 +117,7 @@ XBuildIndexTask::Execute() { table_file.table_id_ = file_->table_id_; table_file.date_ = file_->date_; table_file.file_type_ = engine::meta::TableFileSchema::NEW_INDEX; // for multi-db-path, distribute index file averagely to each path engine::meta::TableFileSchema::NEW_INDEX; engine::meta::MetaPtr meta_ptr = build_index_job->meta(); Status status = build_index_job->meta()->CreateTableFile(table_file); Loading Loading @@ -204,7 +205,8 @@ XBuildIndexTask::Execute() { table_file.file_type_ = engine::meta::TableFileSchema::TO_DELETE; status = meta_ptr->UpdateTableFile(table_file); ENGINE_LOG_DEBUG << "Failed to up date file to index, mark file: " << table_file.file_id_ << " to to_delete"; ENGINE_LOG_DEBUG << "Failed to up date file to index, mark file: " << table_file.file_id_ << " to to_delete"; } build_index_job->BuildIndexDone(to_index_id_); Loading cpp/src/scheduler/task/BuildIndexTask.h +0 −1 Original line number Diff line number Diff line Loading @@ -41,7 +41,6 @@ class XBuildIndexTask : public Task { size_t to_index_id_ = 0; int to_index_type_ = 0; ExecutionEnginePtr to_index_engine_ = nullptr; }; } // namespace scheduler Loading Loading
cpp/src/db/DBImpl.cpp +22 −23 Original line number Diff line number Diff line Loading @@ -898,34 +898,33 @@ DBImpl::BackgroundBuildIndex() { meta_ptr_->FilesToIndex(to_index_files); Status status; // scheduler::BuildIndexJobPtr // job = std::make_shared<scheduler::BuildIndexJob>(0, meta_ptr_, options_); scheduler::BuildIndexJobPtr job = std::make_shared<scheduler::BuildIndexJob>(0, meta_ptr_, options_); // step 2: put build index task to scheduler // for (auto &file : to_index_files) { // scheduler::TableFileSchemaPtr file_ptr = std::make_shared<meta::TableFileSchema>(file); // job->AddToIndexFiles(file_ptr); // } // scheduler::JobMgrInst::GetInstance()->Put(job); // job->WaitBuildIndexFinish(); // if (!job->GetStatus().ok()) { // Status status = job->GetStatus(); // ENGINE_LOG_ERROR << "Building index failed: " << status.ToString(); // } for (auto &file : to_index_files) { std::cout << "get to index file" << std::endl; status = BuildIndex(file); if (!status.ok()) { ENGINE_LOG_ERROR << "Building index for " << file.id_ << " failed: " << status.ToString(); } if (shutting_down_.load(std::memory_order_acquire)) { ENGINE_LOG_DEBUG << "Server will shutdown, skip build index action"; break; scheduler::TableFileSchemaPtr file_ptr = std::make_shared<meta::TableFileSchema>(file); job->AddToIndexFiles(file_ptr); } scheduler::JobMgrInst::GetInstance()->Put(job); job->WaitBuildIndexFinish(); if (!job->GetStatus().ok()) { Status status = job->GetStatus(); ENGINE_LOG_ERROR << "Building index failed: " << status.ToString(); } // for (auto &file : to_index_files) { // status = BuildIndex(file); // if (!status.ok()) { // ENGINE_LOG_ERROR << "Building index for " << file.id_ << " failed: " << status.ToString(); // } // // if (shutting_down_.load(std::memory_order_acquire)) { // ENGINE_LOG_DEBUG << "Server will shutdown, skip build index action"; // break; // } // } ENGINE_LOG_TRACE << "Background build index thread exit"; } Loading
cpp/src/scheduler/job/BuildIndexJob.cpp +3 −6 Original line number Diff line number Diff line Loading @@ -15,16 +15,14 @@ // specific language governing permissions and limitations // under the License. #include "BuildIndexJob.h" #include "utils/Log.h" #include "BuildIndexJob.h" namespace milvus { namespace scheduler { BuildIndexJob::BuildIndexJob(JobId id, engine::meta::MetaPtr meta_ptr, engine::DBOptions options) : Job(id, JobType::BUILD), meta_ptr_(std::move(meta_ptr)), options_(std::move(options)) { } bool Loading Loading @@ -54,6 +52,5 @@ BuildIndexJob::BuildIndexDone(size_t to_index_id) { SERVER_LOG_DEBUG << "BuildIndexJob " << id() << " finish index file: " << to_index_id; } } } No newline at end of file } // namespace scheduler } // namespace milvus
cpp/src/scheduler/job/BuildIndexJob.h +2 −2 Original line number Diff line number Diff line Loading @@ -87,5 +87,5 @@ class BuildIndexJob : public Job { using BuildIndexJobPtr = std::shared_ptr<BuildIndexJob>; } } } // namespace scheduler } // namespace milvus
cpp/src/scheduler/task/BuildIndexTask.cpp +7 −5 Original line number Diff line number Diff line Loading @@ -26,6 +26,7 @@ #include <thread> #include <utility> namespace milvus { namespace scheduler { Loading Loading @@ -116,7 +117,7 @@ XBuildIndexTask::Execute() { table_file.table_id_ = file_->table_id_; table_file.date_ = file_->date_; table_file.file_type_ = engine::meta::TableFileSchema::NEW_INDEX; // for multi-db-path, distribute index file averagely to each path engine::meta::TableFileSchema::NEW_INDEX; engine::meta::MetaPtr meta_ptr = build_index_job->meta(); Status status = build_index_job->meta()->CreateTableFile(table_file); Loading Loading @@ -204,7 +205,8 @@ XBuildIndexTask::Execute() { table_file.file_type_ = engine::meta::TableFileSchema::TO_DELETE; status = meta_ptr->UpdateTableFile(table_file); ENGINE_LOG_DEBUG << "Failed to up date file to index, mark file: " << table_file.file_id_ << " to to_delete"; ENGINE_LOG_DEBUG << "Failed to up date file to index, mark file: " << table_file.file_id_ << " to to_delete"; } build_index_job->BuildIndexDone(to_index_id_); Loading
cpp/src/scheduler/task/BuildIndexTask.h +0 −1 Original line number Diff line number Diff line Loading @@ -41,7 +41,6 @@ class XBuildIndexTask : public Task { size_t to_index_id_ = 0; int to_index_type_ = 0; ExecutionEnginePtr to_index_engine_ = nullptr; }; } // namespace scheduler Loading