buildindex to scheduler run ok (b87ddb9f) · Commits · SUMMER2020 / students / proj-2018085

cpp/src/db/DBImpl.cpp

+22 −40

Original line number	Diff line number	Diff line
		@@ -242,11 +242,7 @@ DBImpl::GetTableRowCount(const std::string &table_id, uint64_t &row_count) {
		}

		Status
		<<<<<<< HEAD
		DBImpl::InsertVectors(const std::string &table_id_, uint64_t n, const float *vectors, IDNumbers &vector_ids_) {
		=======
		DBImpl::InsertVectors(const std::string& table_id, uint64_t n, const float* vectors, IDNumbers& vector_ids) {
		>>>>>>> upstream/branch-0.5.0
		// ENGINE_LOG_DEBUG << "Insert " << n << " vectors to cache";
		if (shutting_down_.load(std::memory_order_acquire)) {
		return Status(DB_ERROR, "Milsvus server is shutdown!");
		@@ -299,17 +295,6 @@ DBImpl::CreateIndex(const std::string &table_id, const TableIndex &index) {
		// for IDMAP type, only wait all NEW file converted to RAW file
		// for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files
		std::vector<int> file_types;
		<<<<<<< HEAD
		if (index.engine_type_ == (int) EngineType::FAISS_IDMAP) {
		file_types = {
		(int) meta::TableFileSchema::NEW, (int) meta::TableFileSchema::NEW_MERGE,
		};
		} else {
		file_types = {
		(int) meta::TableFileSchema::RAW, (int) meta::TableFileSchema::NEW,
		(int) meta::TableFileSchema::NEW_MERGE, (int) meta::TableFileSchema::NEW_INDEX,
		(int) meta::TableFileSchema::TO_INDEX,
		=======
		if (index.engine_type_ == static_cast<int32_t>(EngineType::FAISS_IDMAP)) {
		file_types = {
		static_cast<int32_t>(meta::TableFileSchema::NEW), static_cast<int32_t>(meta::TableFileSchema::NEW_MERGE),
		@@ -321,7 +306,6 @@ DBImpl::CreateIndex(const std::string &table_id, const TableIndex &index) {
		static_cast<int32_t>(meta::TableFileSchema::NEW_MERGE),
		static_cast<int32_t>(meta::TableFileSchema::NEW_INDEX),
		static_cast<int32_t>(meta::TableFileSchema::TO_INDEX),
		>>>>>>> upstream/branch-0.5.0
		};
		}

		@@ -915,38 +899,36 @@ DBImpl::BackgroundBuildIndex() {
		Status status;

		scheduler::BuildIndexJobPtr
		job = std::make_shared<scheduler::BuildIndexJob>(0);
		job = std::make_shared<scheduler::BuildIndexJob>(0, meta_ptr_);

		// step 2: put build index task to scheduler
		scheduler::JobMgrInst::GetInstance()->Put(job);
		// for (auto &file : to_index_files) {
		// std::cout << "get to index file" << std::endl;
		//
		// scheduler::TableFileSchemaPtr file_ptr = std::make_shared<meta::TableFileSchema>(file);
		// job->AddToIndexFiles(file_ptr);
		//
		// if (!job->GetStatus().ok()) {
		// Status status = job->GetStatus();
		// ENGINE_LOG_ERROR << "Building index for " << file.id_ << " failed: " << status.ToString();
		// }
		//
		// }
		// scheduler::JobMgrInst::GetInstance()->Put(job);
		// job->WaitBuildIndexFinish();

		for (auto &file : to_index_files) {
		std::cout << "get to index file" << std::endl;
		meta::TableFileSchema table_file;
		table_file.table_id_ = file.table_id_;
		table_file.date_ = file.date_;
		table_file.file_type_ =
		meta::TableFileSchema::NEW_INDEX; // for multi-db-path, distribute index file averagely to each path
		status = meta_ptr_->CreateTableFile(table_file);
		status = BuildIndex(file);
		if (!status.ok()) {
		ENGINE_LOG_ERROR << "Failed to create table file: " << status.ToString();
		ENGINE_LOG_ERROR << "Building index for " << file.id_ << " failed: " << status.ToString();
		}

		scheduler::TableFileSchemaPtr file_ptr = std::make_shared<meta::TableFileSchema>(file);
		job->AddToIndexFiles(file_ptr, table_file);
		if (shutting_down_.load(std::memory_order_acquire)) {
		ENGINE_LOG_DEBUG << "Server will shutdown, skip build index action";
		break;
		}
		}
		job->WaitBuildIndexFinish();

		// for (auto &file : to_index_files) {
		// status = BuildIndex(file);
		// if (!status.ok()) {
		// ENGINE_LOG_ERROR << "Building index for " << file.id_ << " failed: " << status.ToString();
		// }
		//
		// if (shutting_down_.load(std::memory_order_acquire)) {
		// ENGINE_LOG_DEBUG << "Server will shutdown, skip build index action";
		// break;
		// }
		// }

		ENGINE_LOG_TRACE << "Background build index thread exit";
		}

cpp/src/db/engine/ExecutionEngineImpl.cpp

+2 −0

Original line number	Diff line number	Diff line
		@@ -133,6 +133,7 @@ ExecutionEngineImpl::Serialize() {

		Status
		ExecutionEngineImpl::Load(bool to_cache) {
		std::cout << "load" << std::endl;
		index_ = cache::CpuCacheMgr::GetInstance()->GetIndex(location_);
		bool already_in_cache = (index_ != nullptr);
		if (!already_in_cache) {
		@@ -161,6 +162,7 @@ ExecutionEngineImpl::Load(bool to_cache) {

		Status
		ExecutionEngineImpl::CopyToGpu(uint64_t device_id) {
		std::cout << "copy2gpu" << std::endl;
		auto index = cache::GpuCacheMgr::GetInstance(device_id)->GetIndex(location_);
		bool already_in_cache = (index != nullptr);
		if (already_in_cache) {

cpp/src/scheduler/TaskCreator.cpp

+1 −1

Original line number	Diff line number	Diff line
		@@ -69,7 +69,7 @@ TaskCreator::Create(const DeleteJobPtr &job) {
		}

		std::vector<TaskPtr>
		TaskCreator::Create(const zilliz::milvus::scheduler::BuildIndexJobPtr &job) {
		TaskCreator::Create(const BuildIndexJobPtr &job) {
		std::vector<TaskPtr> tasks;
		//TODO(yukun): remove "disk" hardcode here
		ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource("disk");

cpp/src/scheduler/action/PushTaskToNeighbour.cpp

+2 −1

Original line number	Diff line number	Diff line
		@@ -172,7 +172,8 @@ Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr
		Status stat = config.GetDBConfigBuildIndexGPU(build_index_gpu);

		for (uint64_t i = 0; i < compute_resources.size(); ++i) {
		if (compute_resources[i]->device_id() == build_index_gpu) {
		if (compute_resources[i]->name()
		== res_mgr.lock()->GetResource(ResourceType::GPU, build_index_gpu)->name()) {
		Path task_path(paths[i], paths[i].size() - 1);
		task->path() = task_path;
		break;

cpp/src/scheduler/job/BuildIndexJob.cpp

+4 −8

Original line number	Diff line number	Diff line
		@@ -19,27 +19,24 @@
		#include "utils/Log.h"


		namespace zilliz {
		namespace milvus {
		namespace scheduler {

		BuildIndexJob::BuildIndexJob(zilliz::milvus::scheduler::JobId id)
		: Job(id, JobType::BUILD){
		BuildIndexJob::BuildIndexJob(JobId id, engine::meta::MetaPtr meta_ptr)
		: Job(id, JobType::BUILD), meta_ptr_(std::move(meta_ptr)) {

		}

		bool
		BuildIndexJob::AddToIndexFiles(const engine::meta::TableFileSchemaPtr &to_index_file,
		const TableFileSchema table_file) {
		BuildIndexJob::AddToIndexFiles(const engine::meta::TableFileSchemaPtr &to_index_file) {
		std::unique_lock<std::mutex> lock(mutex_);
		if (to_index_file == nullptr) {
		if (to_index_file == nullptr \|\| to_index_files_.find(to_index_file->id_) != to_index_files_.end()) {
		return false;
		}

		SERVER_LOG_DEBUG << "BuildIndexJob " << id() << " add to_index file: " << to_index_file->id_;

		to_index_files_[to_index_file->id_] = to_index_file;
		table_files_[table_file.id_] = table_file;
		}

		Status&
		@@ -60,4 +57,3 @@ BuildIndexJob::BuildIndexDone(size_t to_index_id) {

		}
		}
		No newline at end of file
		}
		No newline at end of file