format sdk code (a4f1a639) · Commits · SUMMER2020 / students / proj-2018085

cpp/src/main.cpp

+0 −2

Original line number	Diff line number	Diff line
		@@ -76,7 +76,6 @@ main(int argc, char *argv[]) {
		std::cout << "Initial log config from: " << log_config_file << std::endl;
		break;
		}

		case 'p': {
		char *pid_filename_ptr = strdup(optarg);
		pid_filename = pid_filename_ptr;
		@@ -84,7 +83,6 @@ main(int argc, char *argv[]) {
		std::cout << pid_filename << std::endl;
		break;
		}

		case 'd':
		start_daemonized = 1;
		break;

cpp/src/sdk/examples/grpcsimple/main.cpp

+4 −5

Original line number	Diff line number	Diff line
		@@ -23,8 +23,8 @@

		#include "src/ClientTest.h"

		void print_help(const std::string &app_name);

		void
		print_help(const std::string &app_name);

		int
		main(int argc, char *argv[]) {
		@@ -56,8 +56,7 @@ main(int argc, char *argv[]) {
		break;
		}
		case 'h':
		default:
		print_help(app_name);
		default:print_help(app_name);
		return EXIT_SUCCESS;
		}
		}

cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp

+99 −83

Original line number	Diff line number	Diff line
		@@ -15,7 +15,7 @@
		// specific language governing permissions and limitations
		// under the License.

		#include "ClientTest.h"
		#include "sdk/examples/grpcsimple/src/ClientTest.h"
		#include "MilvusApi.h"
		#include "cache/CpuCacheMgr.h"

		@@ -24,15 +24,17 @@
		#include <chrono>
		#include <thread>
		#include <unistd.h>

		using namespace milvus;
		#include <memory>
		#include <vector>
		#include <utility>

		//#define SET_VECTOR_IDS;

		namespace {
		std::string GetTableName();
		const std::string&
		GetTableName();

		const std::string TABLE_NAME = GetTableName();
		const char* TABLE_NAME = GetTableName().c_str();
		constexpr int64_t TABLE_DIMENSION = 512;
		constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024;
		constexpr int64_t BATCH_ROW_COUNT = 100000;
		@@ -44,15 +46,17 @@ constexpr int64_t SECONDS_EACH_HOUR = 3600;

		#define BLOCK_SPLITER std::cout << "===========================================" << std::endl;

		void PrintTableSchema(const TableSchema& tb_schema) {
		void
		PrintTableSchema(const milvus::TableSchema &tb_schema) {
		BLOCK_SPLITER
		std::cout << "Table name: " << tb_schema.table_name << std::endl;
		std::cout << "Table dimension: " << tb_schema.dimension << std::endl;
		BLOCK_SPLITER
		}

		void PrintSearchResult(const std::vector<std::pair<int64_t, RowRecord>>& search_record_array,
		const std::vector<TopKQueryResult>& topk_query_result_array) {
		void
		PrintSearchResult(const std::vector<std::pair<int64_t, milvus::RowRecord>> &search_record_array,
		const std::vector<milvus::TopKQueryResult> &topk_query_result_array) {
		BLOCK_SPLITER
		std::cout << "Returned result count: " << topk_query_result_array.size() << std::endl;

		@@ -72,56 +76,63 @@ void PrintSearchResult(const std::vector<std::pair<int64_t, RowRecord>>& search_
		BLOCK_SPLITER
		}

		std::string CurrentTime() {
		std::string
		CurrentTime() {
		time_t tt;
		time(&tt);
		tt = tt + 8 * SECONDS_EACH_HOUR;
		tm* t= gmtime( &tt );
		tm t;
		gmtime_r(&tt, &t);

		std::string str = std::to_string(t->tm_year + 1900) + "_" + std::to_string(t->tm_mon + 1)
		+ "_" + std::to_string(t->tm_mday) + "_" + std::to_string(t->tm_hour)
		+ "_" + std::to_string(t->tm_min) + "_" + std::to_string(t->tm_sec);
		std::string str = std::to_string(t.tm_year + 1900) + "_" + std::to_string(t.tm_mon + 1)
		+ "_" + std::to_string(t.tm_mday) + "_" + std::to_string(t.tm_hour)
		+ "_" + std::to_string(t.tm_min) + "_" + std::to_string(t.tm_sec);

		return str;
		}

		std::string CurrentTmDate(int64_t offset_day = 0) {
		std::string
		CurrentTmDate(int64_t offset_day = 0) {
		time_t tt;
		time(&tt);
		tt = tt + 8 * SECONDS_EACH_HOUR;
		tt = tt + 24 * SECONDS_EACH_HOUR * offset_day;
		tm* t= gmtime( &tt );
		tm t;
		gmtime_r(&tt, &t);

		std::string str = std::to_string(t->tm_year + 1900) + "-" + std::to_string(t->tm_mon + 1)
		+ "-" + std::to_string(t->tm_mday);
		std::string str = std::to_string(t.tm_year + 1900) + "-" + std::to_string(t.tm_mon + 1)
		+ "-" + std::to_string(t.tm_mday);

		return str;
		}

		std::string GetTableName() {
		static std::string s_id(CurrentTime());
		return "tbl_" + s_id;
		const std::string&
		GetTableName() {
		static std::string s_id("tbl_" + CurrentTime());
		return s_id;
		}

		TableSchema BuildTableSchema() {
		TableSchema tb_schema;
		milvus::TableSchema
		BuildTableSchema() {
		milvus::TableSchema tb_schema;
		tb_schema.table_name = TABLE_NAME;
		tb_schema.dimension = TABLE_DIMENSION;
		tb_schema.index_file_size = TABLE_INDEX_FILE_SIZE;
		tb_schema.metric_type = MetricType::L2;
		tb_schema.metric_type = milvus::MetricType::L2;

		return tb_schema;
		}

		void BuildVectors(int64_t from, int64_t to,
		std::vector<RowRecord>& vector_record_array) {
		void
		BuildVectors(int64_t from, int64_t to,
		std::vector<milvus::RowRecord> &vector_record_array) {
		if (to <= from) {
		return;
		}

		vector_record_array.clear();
		for (int64_t k = from; k < to; k++) {
		RowRecord record;
		milvus::RowRecord record;
		record.data.resize(TABLE_DIMENSION);
		for (int64_t i = 0; i < TABLE_DIMENSION; i++) {
		record.data[i] = (float) (k % (i + 1));
		@@ -131,7 +142,8 @@ void BuildVectors(int64_t from, int64_t to,
		}
		}

		void Sleep(int seconds) {
		void
		Sleep(int seconds) {
		std::cout << "Waiting " << seconds << " seconds ..." << std::endl;
		sleep(seconds);
		}
		@@ -145,7 +157,7 @@ class TimeRecorder {

		~TimeRecorder() {
		std::chrono::system_clock::time_point end = std::chrono::system_clock::now();
		long span = (std::chrono::duration_cast<std::chrono::milliseconds> (end - start_)).count();
		int64_t span = (std::chrono::duration_cast<std::chrono::milliseconds>(end - start_)).count();
		std::cout << title_ << " totally cost: " << span << " ms" << std::endl;
		}

		@@ -154,8 +166,9 @@ class TimeRecorder {
		std::chrono::system_clock::time_point start_;
		};

		void CheckResult(const std::vector<std::pair<int64_t, RowRecord>>& search_record_array,
		const std::vector<TopKQueryResult>& topk_query_result_array) {
		void
		CheckResult(const std::vector<std::pair<int64_t, milvus::RowRecord>> &search_record_array,
		const std::vector<milvus::TopKQueryResult> &topk_query_result_array) {
		BLOCK_SPLITER
		int64_t index = 0;
		for (auto &result : topk_query_result_array) {
		@@ -171,42 +184,45 @@ void CheckResult(const std::vector<std::pair<int64_t, RowRecord>>& search_record
		BLOCK_SPLITER
		}

		void DoSearch(std::shared_ptr<Connection> conn,
		const std::vector<std::pair<int64_t, RowRecord>>& search_record_array,
		void
		DoSearch(std::shared_ptr<milvus::Connection> conn,
		const std::vector<std::pair<int64_t, milvus::RowRecord>> &search_record_array,
		const std::string &phase_name) {
		std::vector<Range> query_range_array;
		Range rg;
		std::vector<milvus::Range> query_range_array;
		milvus::Range rg;
		rg.start_value = CurrentTmDate();
		rg.end_value = CurrentTmDate(1);
		query_range_array.emplace_back(rg);

		std::vector<RowRecord> record_array;
		std::vector<milvus::RowRecord> record_array;
		for (auto &pair : search_record_array) {
		record_array.push_back(pair.second);
		}

		auto start = std::chrono::high_resolution_clock::now();
		std::vector<TopKQueryResult> topk_query_result_array;
		std::vector<milvus::TopKQueryResult> topk_query_result_array;
		{
		TimeRecorder rc(phase_name);
		Status stat = conn->Search(TABLE_NAME, record_array, query_range_array, TOP_K, 32, topk_query_result_array);
		milvus::Status stat =
		conn->Search(TABLE_NAME, record_array, query_range_array, TOP_K, 32, topk_query_result_array);
		std::cout << "SearchVector function call status: " << stat.message() << std::endl;
		}
		auto finish = std::chrono::high_resolution_clock::now();
		std::cout << "SEARCHVECTOR COST: " << std::chrono::duration_cast<std::chrono::duration<double>>(finish - start).count() << "s\n";
		std::cout << "SEARCHVECTOR COST: "
		<< std::chrono::duration_cast<std::chrono::duration<double>>(finish - start).count() << "s\n";

		PrintSearchResult(search_record_array, topk_query_result_array);
		CheckResult(search_record_array, topk_query_result_array);
		}
		}
		} // namespace

		void
		ClientTest::Test(const std::string &address, const std::string &port) {
		std::shared_ptr<Connection> conn = Connection::Create();
		std::shared_ptr<milvus::Connection> conn = milvus::Connection::Create();

		{//connect server
		ConnectParam param = {address, port};
		Status stat = conn->Connect(param);
		milvus::ConnectParam param = {address, port};
		milvus::Status stat = conn->Connect(param);
		std::cout << "Connect function call status: " << stat.message() << std::endl;
		}

		@@ -222,7 +238,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {

		{
		std::vector<std::string> tables;
		Status stat = conn->ShowTables(tables);
		milvus::Status stat = conn->ShowTables(tables);
		std::cout << "ShowTables function call status: " << stat.message() << std::endl;
		std::cout << "All tables: " << std::endl;
		for (auto &table : tables) {
		@@ -234,8 +250,8 @@ ClientTest::Test(const std::string& address, const std::string& port) {
		}

		{//create table
		TableSchema tb_schema = BuildTableSchema();
		Status stat = conn->CreateTable(tb_schema);
		milvus::TableSchema tb_schema = BuildTableSchema();
		milvus::Status stat = conn->CreateTable(tb_schema);
		std::cout << "CreateTable function call status: " << stat.message() << std::endl;
		PrintTableSchema(tb_schema);

		@@ -246,16 +262,16 @@ ClientTest::Test(const std::string& address, const std::string& port) {
		}

		{//describe table
		TableSchema tb_schema;
		Status stat = conn->DescribeTable(TABLE_NAME, tb_schema);
		milvus::TableSchema tb_schema;
		milvus::Status stat = conn->DescribeTable(TABLE_NAME, tb_schema);
		std::cout << "DescribeTable function call status: " << stat.message() << std::endl;
		PrintTableSchema(tb_schema);
		}

		std::vector<std::pair<int64_t, RowRecord>> search_record_array;
		std::vector<std::pair<int64_t, milvus::RowRecord>> search_record_array;
		{//insert vectors
		for (int i = 0; i < ADD_VECTOR_LOOP; i++) {//add vectors
		std::vector<RowRecord> record_array;
		std::vector<milvus::RowRecord> record_array;
		int64_t begin_index = i * BATCH_ROW_COUNT;
		BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array);

		@@ -274,10 +290,10 @@ ClientTest::Test(const std::string& address, const std::string& port) {

		auto start = std::chrono::high_resolution_clock::now();

		Status stat = conn->Insert(TABLE_NAME, record_array, record_ids);
		milvus::Status stat = conn->Insert(TABLE_NAME, record_array, record_ids);
		auto finish = std::chrono::high_resolution_clock::now();
		std::cout << "InsertVector cost: " << std::chrono::duration_cast<std::chrono::duration<double>>(finish - start).count() << "s\n";

		std::cout << "InsertVector cost: "
		<< std::chrono::duration_cast<std::chrono::duration<double>>(finish - start).count() << "s\n";

		std::cout << "InsertVector function call status: " << stat.message() << std::endl;
		std::cout << "Returned id array count: " << record_ids.size() << std::endl;
		@@ -293,27 +309,27 @@ ClientTest::Test(const std::string& address, const std::string& port) {
		Sleep(2);

		int64_t row_count = 0;
		Status stat = conn->CountTable(TABLE_NAME, row_count);
		milvus::Status stat = conn->CountTable(TABLE_NAME, row_count);
		std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl;
		// DoSearch(conn, search_record_array, "Search without index");
		}

		{//wait unit build index finish
		std::cout << "Wait until create all index done" << std::endl;
		IndexParam index;
		milvus::IndexParam index;
		index.table_name = TABLE_NAME;
		index.index_type = IndexType::gpu_ivfsq8;
		index.index_type = milvus::IndexType::gpu_ivfsq8;
		index.nlist = 16384;
		Status stat = conn->CreateIndex(index);
		milvus::Status stat = conn->CreateIndex(index);
		std::cout << "CreateIndex function call status: " << stat.message() << std::endl;

		IndexParam index2;
		milvus::IndexParam index2;
		stat = conn->DescribeIndex(TABLE_NAME, index2);
		std::cout << "DescribeIndex function call status: " << stat.message() << std::endl;
		}

		{//preload table
		Status stat = conn->PreloadTable(TABLE_NAME);
		milvus::Status stat = conn->PreloadTable(TABLE_NAME);
		std::cout << "PreloadTable function call status: " << stat.message() << std::endl;
		}

		@@ -325,7 +341,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
		}

		{//delete index
		Status stat = conn->DropIndex(TABLE_NAME);
		milvus::Status stat = conn->DropIndex(TABLE_NAME);
		std::cout << "DropIndex function call status: " << stat.message() << std::endl;

		int64_t row_count = 0;
		@@ -334,11 +350,11 @@ ClientTest::Test(const std::string& address, const std::string& port) {
		}

		{//delete by range
		Range rg;
		milvus::Range rg;
		rg.start_value = CurrentTmDate(-2);
		rg.end_value = CurrentTmDate(-3);

		Status stat = conn->DeleteByRange(rg, TABLE_NAME);
		milvus::Status stat = conn->DeleteByRange(rg, TABLE_NAME);
		std::cout << "DeleteByRange function call status: " << stat.message() << std::endl;
		}

		@@ -351,7 +367,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
		std::string status = conn->ServerStatus();
		std::cout << "Server status before disconnect: " << status << std::endl;
		}
		Connection::Destroy(conn);
		milvus::Connection::Destroy(conn);
		{//server status
		std::string status = conn->ServerStatus();
		std::cout << "Server status after disconnect: " << status << std::endl;

cpp/src/sdk/grpc/ClientProxy.cpp

+26 −28

Original line number	Diff line number	Diff line
		@@ -15,13 +15,17 @@
		// specific language governing permissions and limitations
		// under the License.

		#include "ClientProxy.h"
		#include "version.h"
		#include "milvus.grpc.pb.h"
		#include "sdk/grpc/ClientProxy.h"
		#include "../../../version.h"
		#include "grpc/gen-milvus/milvus.grpc.pb.h"

		#include <memory>
		#include <vector>
		#include <string>

		//#define GRPC_MULTIPLE_THREAD;

		namespace milvus {

		bool
		UriCheck(const std::string &uri) {
		size_t index = uri.find_first_of(':', 0);
		@@ -133,7 +137,6 @@ ClientProxy::CreateIndex(const IndexParam &index_param) {
		grpc_index_param.mutable_index()->set_index_type((int32_t) index_param.index_type);
		grpc_index_param.mutable_index()->set_nlist(index_param.nlist);
		return client_ptr_->CreateIndex(grpc_index_param);

		} catch (std::exception &ex) {
		return Status(StatusCode::UnknownError, "failed to build index: " + std::string(ex.what()));
		}
		@@ -181,7 +184,9 @@ ClientProxy::Insert(const std::string &table_name,
		}
		std::for_each(threads.begin(), threads.end(), std::mem_fn(&std::thread::join));
		auto finish = std::chrono::high_resolution_clock::now();
		std::cout << "InsertVector cost: " << std::chrono::duration_cast<std::chrono::duration<double>>(finish - start).count() << "s\n";
		std::cout <<
		"InsertVector cost: " << std::chrono::duration_cast<std::chrono::duration<double>>(finish - start).count()
		<< "s\n";
		std::cout << "*****************************************************\n";

		for (size_t i = 0; i < thread_count; i++) {
		@@ -213,9 +218,7 @@ ClientProxy::Insert(const std::string &table_name,
		id_array.push_back(vector_ids.vector_id_array(i));
		}
		}

		#endif

		} catch (std::exception &ex) {
		return Status(StatusCode::UnknownError, "fail to add vector: " + std::string(ex.what()));
		}
		@@ -267,11 +270,9 @@ ClientProxy::Search(const std::string &table_name,
		topk_query_result_array.emplace_back(result);
		}
		return status;

		} catch (std::exception &ex) {
		return Status(StatusCode::UnknownError, "fail to search vectors: " + std::string(ex.what()));
		}

		}

		Status
		@@ -290,7 +291,6 @@ ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_sch
		} catch (std::exception &ex) {
		return Status(StatusCode::UnknownError, "fail to describe table: " + std::string(ex.what()));
		}

		}

		Status
		@@ -316,7 +316,6 @@ ClientProxy::ShowTables(std::vector<std::string> &table_array) {
		table_array[i] = table_name_list.table_names(i);
		}
		return status;

		} catch (std::exception &ex) {
		return Status(StatusCode::UnknownError, "fail to show tables: " + std::string(ex.what()));
		}
		@@ -400,7 +399,6 @@ ClientProxy::DescribeIndex(const std::string &table_name, IndexParam &index_para
		index_param.nlist = grpc_index_param.mutable_index()->nlist();

		return status;

		} catch (std::exception &ex) {
		return Status(StatusCode::UnknownError, "fail to describe index: " + std::string(ex.what()));
		}
		@@ -418,4 +416,4 @@ ClientProxy::DropIndex(const std::string &table_name) const {
		}
		}

		}
		} // namespace milvus

cpp/src/sdk/grpc/ClientProxy.h

+36 −32

Original line number	Diff line number	Diff line
		@@ -20,41 +20,45 @@
		#include "MilvusApi.h"
		#include "GrpcClient.h"

		#include <vector>
		#include <string>
		#include <memory>

		namespace milvus {

		class ClientProxy : public Connection {
		public:
		// Implementations of the Connection interface
		virtual Status
		Status
		Connect(const ConnectParam &param) override;

		virtual Status
		Status
		Connect(const std::string &uri) override;

		virtual Status
		Status
		Connected() const override;

		virtual Status
		Status
		Disconnect() override;

		virtual Status
		Status
		CreateTable(const TableSchema &param) override;

		virtual bool
		bool
		HasTable(const std::string &table_name) override;

		virtual Status
		Status
		DropTable(const std::string &table_name) override;

		virtual Status
		Status
		CreateIndex(const IndexParam &index_param) override;

		virtual Status
		Status
		Insert(const std::string &table_name,
		const std::vector<RowRecord> &record_array,
		std::vector<int64_t> &id_array) override;

		virtual Status
		Status
		Search(const std::string &table_name,
		const std::vector<RowRecord> &query_record_array,
		const std::vector<Range> &query_range_array,
		@@ -62,38 +66,38 @@ public:
		int64_t nprobe,
		std::vector<TopKQueryResult> &topk_query_result_array) override;

		virtual Status
		Status
		DescribeTable(const std::string &table_name, TableSchema &table_schema) override;

		virtual Status
		Status
		CountTable(const std::string &table_name, int64_t &row_count) override;

		virtual Status
		Status
		ShowTables(std::vector<std::string> &table_array) override;

		virtual std::string
		std::string
		ClientVersion() const override;

		virtual std::string
		std::string
		ServerVersion() const override;

		virtual std::string
		std::string
		ServerStatus() const override;

		virtual std::string
		std::string
		DumpTaskTables() const override;

		virtual Status
		Status
		DeleteByRange(Range &range,
		const std::string &table_name) override;

		virtual Status
		Status
		PreloadTable(const std::string &table_name) const override;

		virtual Status
		Status
		DescribeIndex(const std::string &table_name, IndexParam &index_param) const override;

		virtual Status
		Status
		DropIndex(const std::string &table_name) const override;

		private:
		@@ -104,4 +108,4 @@ private:
		bool connected_ = false;
		};

		}
		} // namespace milvus