Unverified Commit 8034123a authored by Jin Hai's avatar Jin Hai Committed by GitHub
Browse files

Merge pull request #1562 from yhmo/master 

rename C++ sdk
parents 929a976b 0c09eb9e

sdk/examples/binary_vector/src/ClientTest.cpp

+54 −53
Original line number Diff line number Diff line
@@ -22,52 +22,53 @@ 


namespace {



const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str();

const char* COLLECTION_NAME = milvus_sdk::Utils::GenCollectionName().c_str();



constexpr int64_t TABLE_DIMENSION = 512;

constexpr int64_t TABLE_INDEX_FILE_SIZE = 128;

constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::TANIMOTO;

constexpr int64_t BATCH_ROW_COUNT = 100000;

constexpr int64_t COLLECTION_DIMENSION = 512;

constexpr int64_t COLLECTION_INDEX_FILE_SIZE = 128;

constexpr milvus::MetricType COLLECTION_METRIC_TYPE = milvus::MetricType::TANIMOTO;

constexpr int64_t BATCH_ENTITY_COUNT = 100000;

constexpr int64_t NQ = 5;

constexpr int64_t TOP_K = 10;

constexpr int64_t NPROBE = 32;

constexpr int64_t SEARCH_TARGET = 5000;  // change this value, result is different, ensure less than BATCH_ROW_COUNT

constexpr int64_t ADD_VECTOR_LOOP = 20;

constexpr int64_t SEARCH_TARGET = 5000;  // change this value, result is different, ensure less than BATCH_ENTITY_COUNT

constexpr int64_t ADD_ENTITY_LOOP = 20;

constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFFLAT;



milvus::TableSchema

BuildTableSchema() {

    milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE};

    return tb_schema;

milvus::CollectionParam

BuildCollectionParam() {

    milvus::CollectionParam

        collection_param = {COLLECTION_NAME, COLLECTION_DIMENSION, COLLECTION_INDEX_FILE_SIZE, COLLECTION_METRIC_TYPE};

    return collection_param;

}



milvus::IndexParam

BuildIndexParam() {

    JSON json_params = {{"nlist", 1024}};

    milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, json_params.dump()};

    milvus::IndexParam index_param = {COLLECTION_NAME, INDEX_TYPE, json_params.dump()};

    return index_param;

}



void

BuildBinaryVectors(int64_t from, int64_t to, std::vector<milvus::RowRecord>& vector_record_array,

                   std::vector<int64_t>& record_ids, int64_t dimension) {

BuildBinaryVectors(int64_t from, int64_t to, std::vector<milvus::Entity>& entity_array,

                   std::vector<int64_t>& entity_ids, int64_t dimension) {

    if (to <= from) {

        return;

    }



    vector_record_array.clear();

    record_ids.clear();

    entity_array.clear();

    entity_ids.clear();



    int64_t dim_byte = dimension/8;

    for (int64_t k = from; k < to; k++) {

        milvus::RowRecord record;

        record.binary_data.resize(dim_byte);

        milvus::Entity entity;

        entity.binary_data.resize(dim_byte);

        for (int64_t i = 0; i < dim_byte; i++) {

            record.binary_data[i] = (uint8_t)lrand48();

            entity.binary_data[i] = (uint8_t)lrand48();

        }



        vector_record_array.emplace_back(record);

        record_ids.push_back(k);

        entity_array.emplace_back(entity);

        entity_ids.push_back(k);

    }

}
@@ -84,54 +85,54 @@ ClientTest::Test(const std::string& address, const std::string& port) { 
        std::cout << "Connect function call status: " << stat.message() << std::endl;

    }



    {  // create table

        milvus::TableSchema tb_schema = BuildTableSchema();

        stat = conn->CreateTable(tb_schema);

        std::cout << "CreateTable function call status: " << stat.message() << std::endl;

        milvus_sdk::Utils::PrintTableSchema(tb_schema);

    {  // create collection

        milvus::CollectionParam collection_param = BuildCollectionParam();

        stat = conn->CreateCollection(collection_param);

        std::cout << "CreateCollection function call status: " << stat.message() << std::endl;

        milvus_sdk::Utils::PrintCollectionParam(collection_param);



        bool has_table = conn->HasTable(tb_schema.table_name);

        if (has_table) {

            std::cout << "Table is created" << std::endl;

        bool has_collection = conn->HasCollection(collection_param.collection_name);

        if (has_collection) {

            std::cout << "Collection is created" << std::endl;

        }

    }



    std::vector<std::pair<int64_t, milvus::RowRecord>> search_record_array;

    std::vector<std::pair<int64_t, milvus::Entity>> search_entity_array;

    {  // insert vectors

        for (int i = 0; i < ADD_VECTOR_LOOP; i++) {

            std::vector<milvus::RowRecord> record_array;

            std::vector<int64_t> record_ids;

            int64_t begin_index = i * BATCH_ROW_COUNT;

        for (int i = 0; i < ADD_ENTITY_LOOP; i++) {

            std::vector<milvus::Entity> entity_array;

            std::vector<int64_t> entity_ids;

            int64_t begin_index = i * BATCH_ENTITY_COUNT;

            {  // generate vectors

                milvus_sdk::TimeRecorder rc("Build vectors No." + std::to_string(i));

                milvus_sdk::TimeRecorder rc("Build entities No." + std::to_string(i));

                BuildBinaryVectors(begin_index,

                                   begin_index + BATCH_ROW_COUNT,

                                   record_array,

                                   record_ids,

                                   TABLE_DIMENSION);

                                   begin_index + BATCH_ENTITY_COUNT,

                                   entity_array,

                                   entity_ids,

                                   COLLECTION_DIMENSION);

            }



            if (search_record_array.size() < NQ) {

                search_record_array.push_back(std::make_pair(record_ids[SEARCH_TARGET], record_array[SEARCH_TARGET]));

            if (search_entity_array.size() < NQ) {

                search_entity_array.push_back(std::make_pair(entity_ids[SEARCH_TARGET], entity_array[SEARCH_TARGET]));

            }



            std::string title = "Insert " + std::to_string(record_array.size()) + " vectors No." + std::to_string(i);

            std::string title = "Insert " + std::to_string(entity_array.size()) + " entities No." + std::to_string(i);

            milvus_sdk::TimeRecorder rc(title);

            stat = conn->Insert(TABLE_NAME, "", record_array, record_ids);

            std::cout << "InsertVector function call status: " << stat.message() << std::endl;

            std::cout << "Returned id array count: " << record_ids.size() << std::endl;

            stat = conn->Insert(COLLECTION_NAME, "", entity_array, entity_ids);

            std::cout << "Insert function call status: " << stat.message() << std::endl;

            std::cout << "Returned id array count: " << entity_ids.size() << std::endl;

        }

    }



    {  // flush buffer

        stat = conn->FlushTable(TABLE_NAME);

        std::cout << "FlushTable function call status: " << stat.message() << std::endl;

        stat = conn->FlushCollection(COLLECTION_NAME);

        std::cout << "FlushCollection function call status: " << stat.message() << std::endl;

    }



    {  // search vectors

        std::vector<std::string> partition_tags;

        milvus::TopKQueryResult topk_query_result;

        milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array,

        milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array,

                                    topk_query_result);

    }
@@ -144,7 +145,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { 
        std::cout << "CreateIndex function call status: " << stat.message() << std::endl;



        milvus::IndexParam index2;

        stat = conn->DescribeIndex(TABLE_NAME, index2);

        stat = conn->DescribeIndex(COLLECTION_NAME, index2);

        std::cout << "DescribeIndex function call status: " << stat.message() << std::endl;

        milvus_sdk::Utils::PrintIndexParam(index2);

    }
@@ -152,13 +153,13 @@ ClientTest::Test(const std::string& address, const std::string& port) { 
    {  // search vectors

        std::vector<std::string> partition_tags;

        milvus::TopKQueryResult topk_query_result;

        milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array,

        milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array,

                                    topk_query_result);

    }



    {  // drop table

        stat = conn->DropTable(TABLE_NAME);

        std::cout << "DropTable function call status: " << stat.message() << std::endl;

    {  // drop collection

        stat = conn->DropCollection(COLLECTION_NAME);

        std::cout << "DropCollection function call status: " << stat.message() << std::endl;

    }



    milvus::Connection::Destroy(conn);

sdk/examples/partition/src/ClientTest.cpp

+74 −79
Original line number Diff line number Diff line
@@ -22,12 +22,12 @@ 


namespace {



const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str();

const char* COLLECTION_NAME = milvus_sdk::Utils::GenCollectionName().c_str();



constexpr int64_t TABLE_DIMENSION = 512;

constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024;

constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2;

constexpr int64_t BATCH_ROW_COUNT = 10000;

constexpr int64_t COLLECTION_DIMENSION = 512;

constexpr int64_t COLLECTION_INDEX_FILE_SIZE = 1024;

constexpr milvus::MetricType COLLECTION_METRIC_TYPE = milvus::MetricType::L2;

constexpr int64_t BATCH_ENTITY_COUNT = 10000;

constexpr int64_t NQ = 5;

constexpr int64_t TOP_K = 10;

constexpr int64_t NPROBE = 32;
@@ -36,27 +36,45 @@ constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; 
constexpr int32_t PARTITION_COUNT = 5;

constexpr int32_t TARGET_PARTITION = 3;



milvus::TableSchema

BuildTableSchema() {

    milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE};

    return tb_schema;

milvus::CollectionParam

BuildCollectionParam() {

    milvus::CollectionParam

        collection_param = {COLLECTION_NAME, COLLECTION_DIMENSION, COLLECTION_INDEX_FILE_SIZE, COLLECTION_METRIC_TYPE};

    return collection_param;

}



milvus::PartitionParam

BuildPartitionParam(int32_t index) {

    std::string tag = std::to_string(index);

    std::string partition_name = std::string(TABLE_NAME) + "_" + tag;

    milvus::PartitionParam partition_param = {TABLE_NAME, tag};

    std::string partition_name = std::string(COLLECTION_NAME) + "_" + tag;

    milvus::PartitionParam partition_param = {COLLECTION_NAME, tag};

    return partition_param;

}



milvus::IndexParam

BuildIndexParam() {

    JSON json_params = {{"nlist", 16384}};

    milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, json_params.dump()};

    milvus::IndexParam index_param = {COLLECTION_NAME, INDEX_TYPE, json_params.dump()};

    return index_param;

}



void

CountCollection(std::shared_ptr<milvus::Connection>& conn) {

    int64_t entity_count = 0;

    auto stat = conn->CountCollection(COLLECTION_NAME, entity_count);

    std::cout << COLLECTION_NAME << "(" << entity_count << " entities)" << std::endl;

}



void

ShowCollectionInfo(std::shared_ptr<milvus::Connection>& conn) {

    CountCollection(conn);



    milvus::CollectionInfo collection_info;

    auto stat = conn->ShowCollectionInfo(COLLECTION_NAME, collection_info);

    milvus_sdk::Utils::PrintCollectionInfo(collection_info);

    std::cout << "ShowCollectionInfo function call status: " << stat.message() << std::endl;

}



}  // namespace



void
@@ -70,11 +88,11 @@ ClientTest::Test(const std::string& address, const std::string& port) { 
        std::cout << "Connect function call status: " << stat.message() << std::endl;

    }



    {  // create table

        milvus::TableSchema tb_schema = BuildTableSchema();

        stat = conn->CreateTable(tb_schema);

        std::cout << "CreateTable function call status: " << stat.message() << std::endl;

        milvus_sdk::Utils::PrintTableSchema(tb_schema);

    {  // create collection

        milvus::CollectionParam tb_schema = BuildCollectionParam();

        stat = conn->CreateCollection(tb_schema);

        std::cout << "CreateCollection function call status: " << stat.message() << std::endl;

        milvus_sdk::Utils::PrintCollectionParam(tb_schema);

    }



    {  // create partition
@@ -87,7 +105,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { 


        // show partitions

        milvus::PartitionTagList partition_array;

        stat = conn->ShowPartitions(TABLE_NAME, partition_array);

        stat = conn->ShowPartitions(COLLECTION_NAME, partition_array);



        std::cout << partition_array.size() << " partitions created:" << std::endl;

        for (auto& partition_tag : partition_array) {
@@ -96,64 +114,56 @@ ClientTest::Test(const std::string& address, const std::string& port) { 
    }



    {  // insert vectors

        milvus_sdk::TimeRecorder rc("All vectors");

        milvus_sdk::TimeRecorder rc("All entities");

        for (int i = 0; i < PARTITION_COUNT * 5; i++) {

            std::vector<milvus::RowRecord> record_array;

            std::vector<int64_t> record_ids;

            int64_t begin_index = i * BATCH_ROW_COUNT;

            std::vector<milvus::Entity> entity_array;

            std::vector<int64_t> entity_ids;

            int64_t begin_index = i * BATCH_ENTITY_COUNT;

            {  // generate vectors

                milvus_sdk::TimeRecorder rc("Build vectors No." + std::to_string(i));

                milvus_sdk::Utils::BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array, record_ids,

                                                TABLE_DIMENSION);

                milvus_sdk::TimeRecorder rc("Build entities No." + std::to_string(i));

                milvus_sdk::Utils::BuildEntities(begin_index,

                                                 begin_index + BATCH_ENTITY_COUNT,

                                                 entity_array,

                                                 entity_ids,

                                                 COLLECTION_DIMENSION);

            }



            std::string title = "Insert " + std::to_string(record_array.size()) + " vectors No." + std::to_string(i);

            std::string title = "Insert " + std::to_string(entity_array.size()) + " entities No." + std::to_string(i);

            milvus_sdk::TimeRecorder rc(title);

            stat = conn->Insert(TABLE_NAME, std::to_string(i % PARTITION_COUNT), record_array, record_ids);

            stat = conn->Insert(COLLECTION_NAME, std::to_string(i % PARTITION_COUNT), entity_array, entity_ids);

        }

    }



    {  // flush buffer

        stat = conn->FlushTable(TABLE_NAME);

        std::cout << "FlushTable function call status: " << stat.message() << std::endl;

        stat = conn->FlushCollection(COLLECTION_NAME);

        std::cout << "FlushCollection function call status: " << stat.message() << std::endl;

    }



    {  // table row count

        int64_t row_count = 0;

        stat = conn->CountTable(TABLE_NAME, row_count);

        std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl;

    }

    ShowCollectionInfo(conn);



    {  // get table information

        milvus::TableInfo table_info;

        stat = conn->ShowTableInfo(TABLE_NAME, table_info);

        milvus_sdk::Utils::PrintTableInfo(table_info);

        std::cout << "ShowTableInfo function call status: " << stat.message() << std::endl;

    }



    std::vector<std::pair<int64_t, milvus::RowRecord>> search_record_array;

    std::vector<std::pair<int64_t, milvus::Entity>> search_entity_array;

    {  // build search vectors

        std::vector<milvus::RowRecord> record_array;

        std::vector<int64_t> record_ids;

        int64_t index = TARGET_PARTITION * BATCH_ROW_COUNT + SEARCH_TARGET;

        milvus_sdk::Utils::BuildVectors(index, index + 1, record_array, record_ids, TABLE_DIMENSION);

        search_record_array.push_back(std::make_pair(record_ids[0], record_array[0]));

        std::vector<milvus::Entity> entity_array;

        std::vector<int64_t> entity_ids;

        int64_t index = TARGET_PARTITION * BATCH_ENTITY_COUNT + SEARCH_TARGET;

        milvus_sdk::Utils::BuildEntities(index, index + 1, entity_array, entity_ids, COLLECTION_DIMENSION);

        search_entity_array.push_back(std::make_pair(entity_ids[0], entity_array[0]));

    }



    {  // search vectors

        std::cout << "Search in correct partition" << std::endl;

        std::vector<std::string> partition_tags = {std::to_string(TARGET_PARTITION)};

        milvus::TopKQueryResult topk_query_result;

        milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array,

        milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array,

                                    topk_query_result);

        std::cout << "Search in wrong partition" << std::endl;

        partition_tags = {"0"};

        milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array,

        milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array,

                                    topk_query_result);



        std::cout << "Search by regex matched partition tag" << std::endl;

        partition_tags = {"\\d"};

        milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array,

        milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array,

                                    topk_query_result);

    }
@@ -166,57 +176,42 @@ ClientTest::Test(const std::string& address, const std::string& port) { 
        std::cout << "CreateIndex function call status: " << stat.message() << std::endl;



        milvus::IndexParam index2;

        stat = conn->DescribeIndex(TABLE_NAME, index2);

        stat = conn->DescribeIndex(COLLECTION_NAME, index2);

        std::cout << "DescribeIndex function call status: " << stat.message() << std::endl;

        milvus_sdk::Utils::PrintIndexParam(index2);

    }



    {  // table row count

        int64_t row_count = 0;

        stat = conn->CountTable(TABLE_NAME, row_count);

        std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl;

    }



    {  // get table information

        milvus::TableInfo table_info;

        stat = conn->ShowTableInfo(TABLE_NAME, table_info);

        milvus_sdk::Utils::PrintTableInfo(table_info);

        std::cout << "ShowTableInfo function call status: " << stat.message() << std::endl;

    }

    ShowCollectionInfo(conn);



    {  // drop partition

        milvus::PartitionParam param1 = {TABLE_NAME, std::to_string(TARGET_PARTITION)};

        milvus::PartitionParam param1 = {COLLECTION_NAME, std::to_string(TARGET_PARTITION)};

        milvus_sdk::Utils::PrintPartitionParam(param1);

        stat = conn->DropPartition(param1);

        std::cout << "DropPartition function call status: " << stat.message() << std::endl;

    }



    {  // table row count

        int64_t row_count = 0;

        stat = conn->CountTable(TABLE_NAME, row_count);

        std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl;

    }

    CountCollection(conn);



    {  // search vectors

        std::cout << "Search in whole table" << std::endl;

    {  // search vectors, will get search error since we delete a partition

        std::cout << "Search in whole collection after delete one partition" << std::endl;

        std::vector<std::string> partition_tags;

        milvus::TopKQueryResult topk_query_result;

        milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array,

        milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array,

                                    topk_query_result);

    }



    {  // drop index

        stat = conn->DropIndex(TABLE_NAME);

        stat = conn->DropIndex(COLLECTION_NAME);

        std::cout << "DropIndex function call status: " << stat.message() << std::endl;



        int64_t row_count = 0;

        stat = conn->CountTable(TABLE_NAME, row_count);

        std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl;

        int64_t entity_count = 0;

        stat = conn->CountCollection(COLLECTION_NAME, entity_count);

        std::cout << COLLECTION_NAME << "(" << entity_count << " entities)" << std::endl;

    }



    {  // drop table

        stat = conn->DropTable(TABLE_NAME);

        std::cout << "DropTable function call status: " << stat.message() << std::endl;

    {  // drop collection

        stat = conn->DropCollection(COLLECTION_NAME);

        std::cout << "DropCollection function call status: " << stat.message() << std::endl;

    }



    milvus::Connection::Destroy(conn);

sdk/examples/simple/src/ClientTest.cpp

+104 −95

File changed.

Preview size limit exceeded, changes collapsed.

sdk/examples/simple/src/ClientTest.h

+12 −12
Original line number Diff line number Diff line
@@ -34,49 +34,49 @@ class ClientTest { 
    ShowSdkVersion();



    void

    ShowTables(std::vector<std::string>&);

    ShowCollections(std::vector<std::string>&);



    void

    CreateTable(const std::string&, int64_t, milvus::MetricType);

    CreateCollection(const std::string&, int64_t, milvus::MetricType);



    void

    DescribeTable(const std::string&);

    DescribeCollection(const std::string&);



    void

    InsertVectors(const std::string&, int64_t);

    InsertEntities(const std::string&, int64_t);



    void

    BuildSearchVectors(int64_t, int64_t);

    BuildSearchEntities(int64_t, int64_t);



    void

    Flush(const std::string&);



    void

    ShowTableInfo(const std::string&);

    ShowCollectionInfo(const std::string&);



    void

    GetVectorById(const std::string&, int64_t);

    GetEntityById(const std::string&, int64_t);



    void

    SearchVectors(const std::string&, int64_t, int64_t);

    SearchEntities(const std::string&, int64_t, int64_t);



    void

    CreateIndex(const std::string&, milvus::IndexType, int64_t);



    void

    PreloadTable(const std::string&);

    PreloadCollection(const std::string&);



    void

    DeleteByIds(const std::string&, const std::vector<int64_t>&);

    DeleteByIds(const std::string&, const std::vector<int64_t>& id_array);



    void

    DropIndex(const std::string&);



    void

    DropTable(const std::string&);

    DropCollection(const std::string&);



 private:

    std::shared_ptr<milvus::Connection> conn_;

    std::vector<std::pair<int64_t, milvus::RowRecord>> search_record_array_;

    std::vector<std::pair<int64_t, milvus::Entity>> search_entity_array_;

    std::vector<int64_t> search_id_array_;

};

sdk/examples/utils/Utils.cpp

+42 −37

File changed.

Preview size limit exceeded, changes collapsed.