fix conflicts (35f17963) · Commits · SUMMER2020 / students / proj-2018085

CHANGELOG.md

+6 −1

Original line number	Diff line number	Diff line
		@@ -8,10 +8,14 @@ Please mark all change in change log and use the ticket from JIRA.
		- \#246 - Exclude src/external folder from code coverage for jenkin ci
		- \#248 - Reside src/external in thirdparty
		- \#316 - Some files not merged after vectors added
		- \#327 - Search does not use GPU when index type is FLAT
		- \#340 - Test cases run failed on 0.6.0
		- \#374 - sdk_simple return empty result

		## Feature
		- \#12 - Pure CPU version for Milvus
		- \#77 - Support table partition
		- \#127 - Support new Index type IVFPQ
		- \#226 - Experimental shards middleware for Milvus
		- \#346 - Support build index with multiple gpu

		@@ -22,6 +26,7 @@ Please mark all change in change log and use the ticket from JIRA.
		- \#314 - add Find FAISS in CMake
		- \#310 - Add Q&A for 'protocol https not supported or disable in libcurl' issue
		- \#322 - Add option to enable / disable prometheus
		- \#358 - Add more information in build.sh and install.md

		## Task

ci/jenkins/Jenkinsfile

+218 −79

Original line number	Diff line number	Diff line
		@@ -33,22 +33,29 @@ pipeline {
		}

		stages {
		stage("Ubuntu 18.04") {
		stage("Ubuntu 18.04 x86_64") {
		environment {
		OS_NAME = "ubuntu18.04"
		CPU_ARCH = "amd64"
		}

		parallel {
		stage ("GPU Version") {
		environment {
		BINRARY_VERSION = "gpu"
		PACKAGE_VERSION = VersionNumber([
		versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-ubuntu18.04-x86_64-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}'
		versionNumberString : '${SEMVER}-gpu-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}'
		]);
		DOCKER_VERSION = "${SEMVER}-${OS_NAME}-${LOWER_BUILD_TYPE}"
		DOCKER_VERSION = "${SEMVER}-gpu-${OS_NAME}-${LOWER_BUILD_TYPE}"
		}

		stages {
		stage("Run Build") {
		agent {
		kubernetes {
		label 'build'
		label "${BINRARY_VERSION}-build"
		defaultContainer 'jnlp'
		yamlFile 'ci/jenkins/pod/milvus-build-env-pod.yaml'
		yamlFile 'ci/jenkins/pod/milvus-gpu-version-build-env-pod.yaml'
		}
		}

		@@ -86,7 +93,7 @@ pipeline {
		stage("Publish docker images") {
		agent {
		kubernetes {
		label 'publish'
		label "${BINRARY_VERSION}-publish"
		defaultContainer 'jnlp'
		yamlFile 'ci/jenkins/pod/docker-pod.yaml'
		}
		@@ -108,7 +115,7 @@ pipeline {
		stage("Deploy to Development") {
		agent {
		kubernetes {
		label 'dev-test'
		label "${BINRARY_VERSION}-dev-test"
		defaultContainer 'jnlp'
		yamlFile 'ci/jenkins/pod/testEnvironment.yaml'
		}
		@@ -162,6 +169,138 @@ pipeline {
		}
		}
		}

		stage ("CPU Version") {
		environment {
		BINRARY_VERSION = "cpu"
		PACKAGE_VERSION = VersionNumber([
		versionNumberString : '${SEMVER}-cpu-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}'
		]);
		DOCKER_VERSION = "${SEMVER}-cpu-${OS_NAME}-${LOWER_BUILD_TYPE}"
		}

		stages {
		stage("Run Build") {
		agent {
		kubernetes {
		label "${BINRARY_VERSION}-build"
		defaultContainer 'jnlp'
		yamlFile 'ci/jenkins/pod/milvus-cpu-version-build-env-pod.yaml'
		}
		}

		stages {
		stage('Build') {
		steps {
		container('milvus-build-env') {
		script {
		load "${env.WORKSPACE}/ci/jenkins/step/build.groovy"
		}
		}
		}
		}
		stage('Code Coverage') {
		steps {
		container('milvus-build-env') {
		script {
		load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy"
		}
		}
		}
		}
		stage('Upload Package') {
		steps {
		container('milvus-build-env') {
		script {
		load "${env.WORKSPACE}/ci/jenkins/step/package.groovy"
		}
		}
		}
		}
		}
		}

		stage("Publish docker images") {
		agent {
		kubernetes {
		label "${BINRARY_VERSION}-publish"
		defaultContainer 'jnlp'
		yamlFile 'ci/jenkins/pod/docker-pod.yaml'
		}
		}

		stages {
		stage('Publish') {
		steps {
		container('publish-images'){
		script {
		load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy"
		}
		}
		}
		}
		}
		}

		stage("Deploy to Development") {
		agent {
		kubernetes {
		label "${BINRARY_VERSION}-dev-test"
		defaultContainer 'jnlp'
		yamlFile 'ci/jenkins/pod/testEnvironment.yaml'
		}
		}

		stages {
		stage("Deploy to Dev") {
		steps {
		container('milvus-test-env') {
		script {
		load "${env.WORKSPACE}/ci/jenkins/step/deploySingle2Dev.groovy"
		}
		}
		}
		}

		stage("Dev Test") {
		steps {
		container('milvus-test-env') {
		script {
		boolean isNightlyTest = isTimeTriggeredBuild()
		if (isNightlyTest) {
		load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy"
		} else {
		load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy"
		}
		}
		}
		}
		}

		stage ("Cleanup Dev") {
		steps {
		container('milvus-test-env') {
		script {
		load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy"
		}
		}
		}
		}
		}
		post {
		unsuccessful {
		container('milvus-test-env') {
		script {
		load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy"
		}
		}
		}
		}
		}
		}
		}
		}
		}
		}
		}

ci/jenkins/pod/milvus-cpu-version-build-env-pod.yaml

0 → 100644

+34 −0

Original line number	Diff line number	Diff line
		apiVersion: v1
		kind: Pod
		metadata:
		name: milvus-cpu-build-env
		labels:
		app: milvus
		componet: cpu-build-env
		spec:
		containers:
		- name: milvus-build-env
		image: registry.zilliz.com/milvus/milvus-cpu-build-env:v0.6.0-ubuntu18.04
		env:
		- name: POD_IP
		valueFrom:
		fieldRef:
		fieldPath: status.podIP
		command:
		- cat
		tty: true
		resources:
		limits:
		memory: "32Gi"
		cpu: "8.0"
		requests:
		memory: "16Gi"
		cpu: "4.0"
		- name: milvus-mysql
		image: mysql:5.6
		env:
		- name: MYSQL_ROOT_PASSWORD
		value: 123456
		ports:
		- containerPort: 3306
		name: mysql

ci/jenkins/pod/milvus-build-env-pod.yaml→ci/jenkins/pod/milvus-gpu-version-build-env-pod.yaml

+3 −3

Original line number	Diff line number	Diff line
		apiVersion: v1
		kind: Pod
		metadata:
		name: milvus-build-env
		name: milvus-gpu-build-env
		labels:
		app: milvus
		componet: build-env
		componet: gpu-build-env
		spec:
		containers:
		- name: milvus-build-env
		image: registry.zilliz.com/milvus/milvus-build-env:v0.5.1-ubuntu18.04
		image: registry.zilliz.com/milvus/milvus-gpu-build-env:v0.6.0-ubuntu18.04
		env:
		- name: POD_IP
		valueFrom:

ci/jenkins/step/build.groovy

+5 −2

Original line number	Diff line number	Diff line
		timeout(time: 60, unit: 'MINUTES') {
		dir ("ci/scripts") {
		withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
		if ("${env.BINRARY_VERSION}" == "gpu") {
		sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -g -j -u -c"
		} else {
		sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -m -j -u -c"
		}
		}
		}
		}