Merge branch 'branch-0.5.0' into 'branch-0.5.0' (7268cfb3) · Commits · SUMMER2020 / students / proj-2018085

cpp/build-support/lint_exclusions.txt

+2 −1

Original line number	Diff line number	Diff line
		@@ -5,3 +5,4 @@
		src/core/thirdparty
		src/grpc
		easylogging++
		*SqliteMetaImpl.cpp
		No newline at end of file

cpp/src/metrics/MetricBase.h

+136 −54

Original line number	Diff line number	Diff line
		@@ -21,6 +21,8 @@
		#include "utils/Error.h"
		#include "SystemInfo.h"

		#include <string>

		namespace zilliz {
		namespace milvus {
		namespace server {
		@@ -32,56 +34,136 @@ class MetricsBase{
		return instance;
		}

		virtual ErrorCode Init() {};

		virtual void AddVectorsSuccessTotalIncrement(double value = 1) {};
		virtual void AddVectorsFailTotalIncrement(double value = 1) {};
		virtual void AddVectorsDurationHistogramOberve(double value) {};

		virtual void RawFileSizeHistogramObserve(double value) {};
		virtual void IndexFileSizeHistogramObserve(double value) {};
		virtual void BuildIndexDurationSecondsHistogramObserve(double value) {};

		virtual void CpuCacheUsageGaugeSet(double value) {};
		virtual void GpuCacheUsageGaugeSet() {};

		virtual void MetaAccessTotalIncrement(double value = 1) {};
		virtual void MetaAccessDurationSecondsHistogramObserve(double value) {};
		virtual void FaissDiskLoadDurationSecondsHistogramObserve(double value) {};
		virtual void FaissDiskLoadSizeBytesHistogramObserve(double value) {};
		virtual void CacheAccessTotalIncrement(double value = 1) {};
		virtual void MemTableMergeDurationSecondsHistogramObserve(double value) {};
		virtual void SearchIndexDataDurationSecondsHistogramObserve(double value) {};
		virtual void SearchRawDataDurationSecondsHistogramObserve(double value) {};
		virtual void IndexFileSizeTotalIncrement(double value = 1) {};
		virtual void RawFileSizeTotalIncrement(double value = 1) {};
		virtual void IndexFileSizeGaugeSet(double value) {};
		virtual void RawFileSizeGaugeSet(double value) {};
		virtual void FaissDiskLoadIOSpeedGaugeSet(double value) {};
		virtual void QueryResponseSummaryObserve(double value) {};
		virtual void DiskStoreIOSpeedGaugeSet(double value) {};
		virtual void DataFileSizeGaugeSet(double value) {};
		virtual void AddVectorsSuccessGaugeSet(double value) {};
		virtual void AddVectorsFailGaugeSet(double value) {};
		virtual void QueryVectorResponseSummaryObserve(double value, int count = 1) {};
		virtual void QueryVectorResponsePerSecondGaugeSet(double value) {};
		virtual void CPUUsagePercentSet() {};
		virtual void RAMUsagePercentSet() {};
		virtual void QueryResponsePerSecondGaugeSet(double value) {};
		virtual void GPUPercentGaugeSet() {};
		virtual void GPUMemoryUsageGaugeSet() {};
		virtual void AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) {};
		virtual void QueryIndexTypePerSecondSet(std::string type, double value) {};
		virtual void ConnectionGaugeIncrement() {};
		virtual void ConnectionGaugeDecrement() {};
		virtual void KeepingAliveCounterIncrement(double value = 1) {};
		virtual void OctetsSet() {};

		virtual void CPUCoreUsagePercentSet() {};
		virtual void GPUTemperature() {};
		virtual void CPUTemperature() {};
		};
		virtual ErrorCode Init() {
		}

		virtual void AddVectorsSuccessTotalIncrement(double value = 1) {
		}

		virtual void AddVectorsFailTotalIncrement(double value = 1) {
		}

		virtual void AddVectorsDurationHistogramOberve(double value) {
		}

		virtual void RawFileSizeHistogramObserve(double value) {
		}

		virtual void IndexFileSizeHistogramObserve(double value) {
		}

		virtual void BuildIndexDurationSecondsHistogramObserve(double value) {
		}

		virtual void CpuCacheUsageGaugeSet(double value) {
		}

		virtual void GpuCacheUsageGaugeSet() {
		}

		virtual void MetaAccessTotalIncrement(double value = 1) {
		}

		virtual void MetaAccessDurationSecondsHistogramObserve(double value) {
		}

		virtual void FaissDiskLoadDurationSecondsHistogramObserve(double value) {
		}

		virtual void FaissDiskLoadSizeBytesHistogramObserve(double value) {
		}

		virtual void CacheAccessTotalIncrement(double value = 1) {
		}

		virtual void MemTableMergeDurationSecondsHistogramObserve(double value) {
		}

		virtual void SearchIndexDataDurationSecondsHistogramObserve(double value) {
		}

		virtual void SearchRawDataDurationSecondsHistogramObserve(double value) {
		}

		virtual void IndexFileSizeTotalIncrement(double value = 1) {
		}

		virtual void RawFileSizeTotalIncrement(double value = 1) {
		}

		virtual void IndexFileSizeGaugeSet(double value) {
		}

		virtual void RawFileSizeGaugeSet(double value) {
		}

		virtual void FaissDiskLoadIOSpeedGaugeSet(double value) {
		}

		virtual void QueryResponseSummaryObserve(double value) {
		}

		virtual void DiskStoreIOSpeedGaugeSet(double value) {
		}

		virtual void DataFileSizeGaugeSet(double value) {
		}

		virtual void AddVectorsSuccessGaugeSet(double value) {
		}

		virtual void AddVectorsFailGaugeSet(double value) {
		}

		virtual void QueryVectorResponseSummaryObserve(double value, int count = 1) {
		}

		virtual void QueryVectorResponsePerSecondGaugeSet(double value) {
		}

		virtual void CPUUsagePercentSet() {
		}

		virtual void RAMUsagePercentSet() {
		}

		virtual void QueryResponsePerSecondGaugeSet(double value) {
		}

		virtual void GPUPercentGaugeSet() {
		}

		virtual void GPUMemoryUsageGaugeSet() {
		}

		virtual void AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) {
		}

		virtual void QueryIndexTypePerSecondSet(std::string type, double value) {
		}

		virtual void ConnectionGaugeIncrement() {
		}

		virtual void ConnectionGaugeDecrement() {
		}

		virtual void KeepingAliveCounterIncrement(double value = 1) {
		}

		virtual void OctetsSet() {
		}

		virtual void CPUCoreUsagePercentSet() {
		}

		virtual void GPUTemperature() {
		}

		virtual void CPUTemperature() {
		}
		};

		} // namespace server
		} // namespace milvus
		} // namespace zilliz

cpp/src/metrics/Metrics.cpp

+5 −4

Original line number	Diff line number	Diff line
		@@ -15,10 +15,11 @@
		// specific language governing permissions and limitations
		// under the License.

		#include "Metrics.h"
		#include "metrics/Metrics.h"
		#include "server/Config.h"
		#include "PrometheusMetrics.h"

		#include <string>

		namespace zilliz {
		namespace milvus {
		@@ -44,6 +45,6 @@ Metrics::CreateMetricsCollector() {
		}
		}

		}
		}
		}
		} // namespace server
		} // namespace milvus
		} // namespace zilliz

cpp/src/metrics/Metrics.h

+35 −38

Original line number	Diff line number	Diff line
		@@ -21,7 +21,6 @@
		#include "MetricBase.h"
		#include "db/meta/MetaTypes.h"


		namespace zilliz {
		namespace milvus {
		namespace server {
		@@ -94,7 +93,7 @@ private:
		////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		class CollectQueryMetrics : CollectMetricsBase {
		public:
		CollectQueryMetrics(size_t nq) : nq_(nq) {
		explicit CollectQueryMetrics(size_t nq) : nq_(nq) {
		}

		~CollectQueryMetrics() {
		@@ -140,7 +139,7 @@ public:
		////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		class CollectExecutionEngineMetrics : CollectMetricsBase {
		public:
		CollectExecutionEngineMetrics(double physical_size) : physical_size_(physical_size) {
		explicit CollectExecutionEngineMetrics(double physical_size) : physical_size_(physical_size) {
		}

		~CollectExecutionEngineMetrics() {
		@@ -158,13 +157,14 @@ private:
		////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		class CollectSerializeMetrics : CollectMetricsBase {
		public:
		CollectSerializeMetrics(size_t size) : size_(size) {
		explicit CollectSerializeMetrics(size_t size) : size_(size) {
		}

		~CollectSerializeMetrics() {
		auto total_time = TimeFromBegine();
		server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet((double) size_ / total_time);
		}

		private:
		size_t size_;
		};
		@@ -181,6 +181,7 @@ public:
		static_cast<int>(dimension_),
		total_time);
		}

		private:
		size_t n_;
		uint16_t dimension_;
		@@ -189,7 +190,7 @@ private:
		////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		class CollectDurationMetrics : CollectMetricsBase {
		public:
		CollectDurationMetrics(int index_type) : index_type_(index_type) {
		explicit CollectDurationMetrics(int index_type) : index_type_(index_type) {
		}

		~CollectDurationMetrics() {
		@@ -209,6 +210,7 @@ public:
		}
		}
		}

		private:
		int index_type_;
		};
		@@ -216,7 +218,7 @@ private:
		////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		class CollectSearchTaskMetrics : CollectMetricsBase {
		public:
		CollectSearchTaskMetrics(int index_type) : index_type_(index_type) {
		explicit CollectSearchTaskMetrics(int index_type) : index_type_(index_type) {
		}

		~CollectSearchTaskMetrics() {
		@@ -254,11 +256,6 @@ public:
		}
		};



		}
		}
		}



		} // namespace server
		} // namespace milvus
		} // namespace zilliz

cpp/src/metrics/PrometheusMetrics.cpp

+57 −48

Original line number	Diff line number	Diff line
		@@ -16,12 +16,14 @@
		// under the License.


		#include "metrics/PrometheusMetrics.h"
		#include "cache/GpuCacheMgr.h"
		#include "PrometheusMetrics.h"
		#include "server/Config.h"
		#include "utils/Log.h"
		#include "SystemInfo.h"

		#include <string>
		#include <utility>

		namespace zilliz {
		namespace milvus {
		@@ -53,10 +55,8 @@ PrometheusMetrics::Init() {
		}

		return SERVER_SUCCESS;

		}


		void
		PrometheusMetrics::CPUUsagePercentSet() {
		if (!startup_) return;
		@@ -75,8 +75,8 @@ void
		PrometheusMetrics::GPUPercentGaugeSet() {
		if (!startup_) return;
		int numDevice = server::SystemInfo::GetInstance().num_device();
		std::vector<unsigned long long > used_total = server::SystemInfo::GetInstance().GPUMemoryTotal();
		std::vector<unsigned long long > used_memory = server::SystemInfo::GetInstance().GPUMemoryUsed();
		std::vector<uint64_t> used_total = server::SystemInfo::GetInstance().GPUMemoryTotal();
		std::vector<uint64_t> used_memory = server::SystemInfo::GetInstance().GPUMemoryUsed();

		for (int i = 0; i < numDevice; ++i) {
		prometheus::Gauge &GPU_percent = GPU_percent_.Add({{"DeviceNum", std::to_string(i)}});
		@@ -85,55 +85,60 @@ PrometheusMetrics::GPUPercentGaugeSet() {
		}
		}

		void PrometheusMetrics::GPUMemoryUsageGaugeSet() {
		void
		PrometheusMetrics::GPUMemoryUsageGaugeSet() {
		if (!startup_) return;
		std::vector<unsigned long long> values = server::SystemInfo::GetInstance().GPUMemoryUsed();
		constexpr unsigned long long MtoB = 1024*1024;
		std::vector<uint64_t> values = server::SystemInfo::GetInstance().GPUMemoryUsed();
		constexpr uint64_t MtoB = 1024 * 1024;
		int numDevice = server::SystemInfo::GetInstance().num_device();

		for (int i = 0; i < numDevice; ++i) {
		prometheus::Gauge &GPU_memory = GPU_memory_usage_.Add({{"DeviceNum", std::to_string(i)}});
		GPU_memory.Set(values[i] / MtoB);
		}

		}
		void PrometheusMetrics::AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) {

		void
		PrometheusMetrics::AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) {
		// MB/s
		if (!startup_) return;

		long long MtoB = 1024*1024;
		long long size = num_vector * dim * 4;
		int64_t MtoB = 1024 * 1024;
		int64_t size = num_vector * dim * 4;
		add_vectors_per_second_gauge_.Set(size / time / MtoB);

		}
		void PrometheusMetrics::QueryIndexTypePerSecondSet(std::string type, double value) {

		void
		PrometheusMetrics::QueryIndexTypePerSecondSet(std::string type, double value) {
		if (!startup_) return;
		if (type == "IVF") {
		query_index_IVF_type_per_second_gauge_.Set(value);
		} else if (type == "IDMap") {
		query_index_IDMAP_type_per_second_gauge_.Set(value);
		}

		}

		void PrometheusMetrics::ConnectionGaugeIncrement() {
		void
		PrometheusMetrics::ConnectionGaugeIncrement() {
		if (!startup_) return;
		connection_gauge_.Increment();
		}

		void PrometheusMetrics::ConnectionGaugeDecrement() {
		void
		PrometheusMetrics::ConnectionGaugeDecrement() {
		if (!startup_) return;
		connection_gauge_.Decrement();
		}

		void PrometheusMetrics::OctetsSet() {
		void
		PrometheusMetrics::OctetsSet() {
		if (!startup_) return;

		// get old stats and reset them
		unsigned long long old_inoctets = SystemInfo::GetInstance().get_inoctets();
		unsigned long long old_outoctets = SystemInfo::GetInstance().get_octets();
		uint64_t old_inoctets = SystemInfo::GetInstance().get_inoctets();
		uint64_t old_outoctets = SystemInfo::GetInstance().get_octets();
		auto old_time = SystemInfo::GetInstance().get_nettime();
		std::pair<unsigned long long, unsigned long long> in_and_out_octets = SystemInfo::GetInstance().Octets();
		std::pair<uint64_t, uint64_t> in_and_out_octets = SystemInfo::GetInstance().Octets();
		SystemInfo::GetInstance().set_inoctets(in_and_out_octets.first);
		SystemInfo::GetInstance().set_outoctets(in_and_out_octets.second);
		SystemInfo::GetInstance().set_nettime();
		@@ -148,7 +153,8 @@ void PrometheusMetrics::OctetsSet() {
		outoctets_gauge_.Set((in_and_out_octets.second - old_outoctets) / total_second);
		}

		void PrometheusMetrics::CPUCoreUsagePercentSet() {
		void
		PrometheusMetrics::CPUCoreUsagePercentSet() {
		if (!startup_)
		return;

		@@ -160,11 +166,12 @@ void PrometheusMetrics::CPUCoreUsagePercentSet() {
		}
		}

		void PrometheusMetrics::GPUTemperature() {
		void
		PrometheusMetrics::GPUTemperature() {
		if (!startup_)
		return;

		std::vector<unsigned int> GPU_temperatures = server::SystemInfo::GetInstance().GPUTemperature();
		std::vector<uint64_t> GPU_temperatures = server::SystemInfo::GetInstance().GPUTemperature();

		for (int i = 0; i < GPU_temperatures.size(); ++i) {
		prometheus::Gauge &gpu_temp = GPU_temperature_.Add({{"GPU", std::to_string(i)}});
		@@ -172,7 +179,8 @@ void PrometheusMetrics::GPUTemperature() {
		}
		}

		void PrometheusMetrics::CPUTemperature() {
		void
		PrometheusMetrics::CPUTemperature() {
		if (!startup_)
		return;

		@@ -184,7 +192,8 @@ void PrometheusMetrics::CPUTemperature() {
		}
		}

		void PrometheusMetrics::GpuCacheUsageGaugeSet() {
		void
		PrometheusMetrics::GpuCacheUsageGaugeSet() {
		// std::vector<uint64_t > gpu_ids = {0};
		// for(auto i = 0; i < gpu_ids.size(); ++i) {
		// uint64_t cache_usage = cache::GpuCacheMgr::GetInstance(gpu_ids[i])->CacheUsage();
		@@ -194,6 +203,6 @@ void PrometheusMetrics::GpuCacheUsageGaugeSet() {
		// }
		}

		}
		}
		}
		} // namespace server
		} // namespace milvus
		} // namespace zilliz