Commit e35d4621 authored by Jin Hai's avatar Jin Hai Committed by GitHub
Browse files

Merge pull request #173 from JinHai-CN/0.5.1 

Merge from internal

Former-commit-id: d5cfcf03040a92a5abb24878f9762205819989cb
parents aa5eea39 48420dca

ci/jenkinsfile/dev_test.groovy

+2 −2
Original line number Diff line number Diff line
@@ -3,7 +3,7 @@ timeout(time: 30, unit: 'MINUTES') { 
        dir ("${PROJECT_NAME}_test") {

            checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])

            sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'

            sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local"

            sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local --internal=true"

        }

        // mysql database backend test

        load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy"
@@ -19,7 +19,7 @@ timeout(time: 30, unit: 'MINUTES') { 
            }

        }

        dir ("${PROJECT_NAME}_test") {

            sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local"

            sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local --internal=true"

        }

    } catch (exc) {

        echo 'Milvus Test Failed !'

ci/jenkinsfile/dev_test_all.groovy

+2 −2
Original line number Diff line number Diff line
@@ -3,7 +3,7 @@ timeout(time: 60, unit: 'MINUTES') { 
        dir ("${PROJECT_NAME}_test") {

            checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])

            sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'

            sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local"

            sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local --internal=true"

        }



        // mysql database backend test
@@ -20,7 +20,7 @@ timeout(time: 60, unit: 'MINUTES') { 
            }

        }

        dir ("${PROJECT_NAME}_test") {

            sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local"

            sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local --internal=true"

        }

    } catch (exc) {

        echo 'Milvus Test Failed !'

core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp

+4 −3
Original line number Diff line number Diff line
@@ -221,9 +221,10 @@ IVF::search_impl(int64_t n, const float* data, int64_t k, float* distances, int6 
    faiss::ivflib::search_with_parameters(index_.get(), n, (float*)data, k, distances, labels, params.get());

    stdclock::time_point after = stdclock::now();

    double search_cost = (std::chrono::duration<double, std::micro>(after - before)).count();

    KNOWHERE_LOG_DEBUG << "IVF search cost: " << search_cost

                       << ", quantization cost: " << faiss::indexIVF_stats.quantization_time

                       << ", data search cost: " << faiss::indexIVF_stats.search_time;

    KNOWHERE_LOG_DEBUG << "K=" << k << " NQ=" << n << " NL=" << faiss::indexIVF_stats.nlist

                       << " ND=" << faiss::indexIVF_stats.ndis << " NH=" << faiss::indexIVF_stats.nheap_updates

                       << " Q=" << faiss::indexIVF_stats.quantization_time

                       << " S=" << faiss::indexIVF_stats.search_time;

    faiss::indexIVF_stats.quantization_time = 0;

    faiss::indexIVF_stats.search_time = 0;

}

core/src/scheduler/job/SearchJob.h

+5 −0
Original line number Diff line number Diff line
@@ -90,6 +90,11 @@ class SearchJob : public Job { 
        return index_files_;

    }



    std::mutex&

    mutex() {

        return mutex_;

    }



 private:

    uint64_t topk_ = 0;

    uint64_t nq_ = 0;

core/src/scheduler/task/SearchTask.cpp

+5 −2
Original line number Diff line number Diff line
@@ -219,8 +219,11 @@ XSearchTask::Execute() { 


            // step 3: pick up topk result

            auto spec_k = index_engine_->Count() < topk ? index_engine_->Count() : topk;

            {

                std::unique_lock<std::mutex> lock(search_job->mutex());

                XSearchTask::MergeTopkToResultSet(output_ids, output_distance, spec_k, nq, topk, metric_l2,

                                                  search_job->GetResult());

            }



            span = rc.RecordSection(hdr + ", reduce topk");

            //            search_job->AccumReduceCost(span);