codecov integration

                                container('milvus-build-env') {

                                    script {

                                        load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/build.groovy"

                                        load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/coverage.groovy"

                                    }

                                }

                            }

timeout(time: 60, unit: 'MINUTES') {

    dir ("core") {

        sh "git config --global user.email \"test@zilliz.com\""

        sh "git config --global user.name \"test\""

        sh "./build.sh -l"

        withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {

            sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -d /opt/milvus -j -u -c"

            sh "./coverage.sh -u root -p 123456 -t \$POD_IP"

            sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -d /opt/milvus -j -u -c"

        }

    }

}

timeout(time: 60, unit: 'MINUTES') {

    dir ("ci/jenkins/scripts") {

        sh "./coverage.sh -o /opt/milvus -u root -p 123456 -t \$POD_IP"

        // Set some env variables so codecov detection script works correctly

        sh 'bash <(curl -s https://codecov.io/bash) -f output_new.info || echo "Codecov did not collect coverage reports"'

    }

}

#!/bin/bash

SOURCE="${BASH_SOURCE[0]}"

while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink

  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"

  SOURCE="$(readlink "$SOURCE")"

  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located

done

SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"

CMAKE_BUILD_DIR="${SCRIPTS_DIR}/../../../core/cmake_build"

BUILD_TYPE="Debug"

BUILD_UNITTEST="OFF"

INSTALL_PREFIX="/opt/milvus"

BUILD_COVERAGE="OFF"

DB_PATH="/opt/milvus"

PROFILING="OFF"

USE_JFROG_CACHE="OFF"

RUN_CPPLINT="OFF"

CUSTOMIZATION="OFF" # default use ori faiss

CUDA_COMPILER=/usr/local/cuda/bin/nvcc

CUSTOMIZED_FAISS_URL="${FAISS_URL:-NONE}"

wget -q --method HEAD ${CUSTOMIZED_FAISS_URL}

if [ $? -eq 0 ]; then

  CUSTOMIZATION="ON"

else

  CUSTOMIZATION="OFF"

fi

while getopts "o:d:t:ulcgjhx" arg

do

        case $arg in

             o)

                INSTALL_PREFIX=$OPTARG

                ;;

             d)

                DB_PATH=$OPTARG

                ;;

             t)

                BUILD_TYPE=$OPTARG # BUILD_TYPE

                ;;

             u)

                echo "Build and run unittest cases" ;

                BUILD_UNITTEST="ON";

                ;;

             l)

                RUN_CPPLINT="ON"

                ;;

             c)

                BUILD_COVERAGE="ON"

                ;;

             g)

                PROFILING="ON"

                ;;

             j)

                USE_JFROG_CACHE="ON"

                ;;

             x)

                CUSTOMIZATION="OFF" # force use ori faiss

                ;;

             h) # help

                echo "

parameter:

-o: install prefix(default: /opt/milvus)

-d: db data path(default: /opt/milvus)

-t: build type(default: Debug)

-u: building unit test options(default: OFF)

-l: run cpplint, clang-format and clang-tidy(default: OFF)

-c: code coverage(default: OFF)

-g: profiling(default: OFF)

-j: use jfrog cache build directory(default: OFF)

-h: help

usage:

./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} [-u] [-l] [-r] [-c] [-g] [-j] [-h]

                "

                exit 0

                ;;

             ?)

                echo "ERROR! unknown argument"

        exit 1

        ;;

        esac

done

if [[ ! -d ${CMAKE_BUILD_DIR} ]]; then

    mkdir ${CMAKE_BUILD_DIR}

fi

cd ${CMAKE_BUILD_DIR}

# remove make cache since build.sh -l use default variables

# force update the variables each time 

make rebuild_cache

CMAKE_CMD="cmake \

-DBUILD_UNIT_TEST=${BUILD_UNITTEST} \

-DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}

-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \

-DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \

-DBUILD_COVERAGE=${BUILD_COVERAGE} \

-DMILVUS_DB_PATH=${DB_PATH} \

-DMILVUS_ENABLE_PROFILING=${PROFILING} \

-DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \

-DCUSTOMIZATION=${CUSTOMIZATION} \

-DFAISS_URL=${CUSTOMIZED_FAISS_URL} \

.."

echo ${CMAKE_CMD}

${CMAKE_CMD}

if [[ ${RUN_CPPLINT} == "ON" ]]; then

    # cpplint check

    make lint

    if [ $? -ne 0 ]; then

        echo "ERROR! cpplint check failed"

        exit 1

    fi

    echo "cpplint check passed!"

    # clang-format check

    make check-clang-format

    if [ $? -ne 0 ]; then

        echo "ERROR! clang-format check failed"

        exit 1

    fi

    echo "clang-format check passed!"

#    # clang-tidy check

#    make check-clang-tidy

#    if [ $? -ne 0 ]; then

#        echo "ERROR! clang-tidy check failed"

#        rm -f CMakeCache.txt

#        exit 1

#    fi

#    echo "clang-tidy check passed!"

else

    # compile and build

    make -j8 || exit 1

    make install || exit 1

fi

#!/bin/bash

SOURCE="${BASH_SOURCE[0]}"

while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink

  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"

  SOURCE="$(readlink "$SOURCE")"

  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located

done

SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"

INSTALL_PREFIX="/opt/milvus"

CMAKE_BUILD_DIR="${SCRIPTS_DIR}/../../../core/cmake_build"

MYSQL_USER_NAME=root

MYSQL_PASSWORD=123456

MYSQL_HOST='127.0.0.1'

MYSQL_PORT='3306'

while getopts "o:u:p:t:h" arg

do

        case $arg in

             o)

                INSTALL_PREFIX=$OPTARG

                ;;

             u)

                MYSQL_USER_NAME=$OPTARG

                ;;

             p)

                MYSQL_PASSWORD=$OPTARG

                ;;

             t)

                MYSQL_HOST=$OPTARG

                ;;

             h) # help

                echo "

parameter:

-o: milvus install prefix(default: /opt/milvus)

-u: mysql account

-p: mysql password

-t: mysql host

-h: help

usage:

./coverage.sh -o \${INSTALL_PREFIX} -u \${MYSQL_USER} -p \${MYSQL_PASSWORD} -t \${MYSQL_HOST} [-h]

                "

                exit 0

                ;;

             ?)

                echo "ERROR! unknown argument"

        exit 1

        ;;

        esac

done

export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${INSTALL_PREFIX}/lib

LCOV_CMD="lcov"

# LCOV_GEN_CMD="genhtml"

FILE_INFO_BASE="base.info"

FILE_INFO_MILVUS="server.info"

FILE_INFO_OUTPUT="output.info"

FILE_INFO_OUTPUT_NEW="output_new.info"

DIR_LCOV_OUTPUT="lcov_out"

DIR_GCNO="${CMAKE_BUILD_DIR}"

DIR_UNITTEST="${INSTALL_PREFIX}/unittest"

# delete old code coverage info files

rm -rf lcov_out

rm -f FILE_INFO_BASE FILE_INFO_MILVUS FILE_INFO_OUTPUT FILE_INFO_OUTPUT_NEW

MYSQL_DB_NAME=milvus_`date +%s%N`

function mysql_exc()

{

    cmd=$1

    mysql -h${MYSQL_HOST} -u${MYSQL_USER_NAME} -p${MYSQL_PASSWORD} -e "${cmd}"

    if [ $? -ne 0 ]; then

        echo "mysql $cmd run failed"

    fi

}

mysql_exc "CREATE DATABASE IF NOT EXISTS ${MYSQL_DB_NAME};"

mysql_exc "GRANT ALL PRIVILEGES ON ${MYSQL_DB_NAME}.* TO '${MYSQL_USER_NAME}'@'%';"

mysql_exc "FLUSH PRIVILEGES;"

mysql_exc "USE ${MYSQL_DB_NAME};"

# get baseline

${LCOV_CMD} -c -i -d ${DIR_GCNO} -o "${FILE_INFO_BASE}"

if [ $? -ne 0 ]; then

    echo "gen baseline coverage run failed"

    exit -1

fi

for test in `ls ${DIR_UNITTEST}`; do

    echo $test

    case ${test} in

        test_db)

            # set run args for test_db

            args="mysql://${MYSQL_USER_NAME}:${MYSQL_PASSWORD}@${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB_NAME}"

            ;;

        *test_*)

            args=""

            ;;

    esac

    # run unittest

    ./${DIR_UNITTEST}/${test} "${args}"

    if [ $? -ne 0 ]; then

        echo ${args}

        echo ${DIR_UNITTEST}/${test} "run failed"

    fi

done

mysql_exc "DROP DATABASE IF EXISTS ${MYSQL_DB_NAME};"

# gen code coverage

${LCOV_CMD} -d ${DIR_GCNO} -o "${FILE_INFO_MILVUS}" -c

# merge coverage

${LCOV_CMD} -a ${FILE_INFO_BASE} -a ${FILE_INFO_MILVUS} -o "${FILE_INFO_OUTPUT}"

# remove third party from tracefiles

${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \

    "/usr/*" \

    "*/boost/*" \

    "*/cmake_build/*_ep-prefix/*" \

    "src/index/cmake_build*" \

    "src/index/thirdparty*" \

    "src/grpc*"\

    "src/metrics/MetricBase.h"\

    "src/server/Server.cpp"\

    "src/server/DBWrapper.cpp"\

    "src/server/grpc_impl/GrpcServer.cpp"\

    "src/utils/easylogging++.h"\

    "src/utils/easylogging++.cc"\

# gen html report

# ${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/