Commit 3d40a4b6 authored by jinhai's avatar jinhai
Browse files

Merge branch 'upgrade_sptag' into 'branch-0.5.0' 

MS-655 upgrade sptag remove tbb

See merge request megasearch/milvus!721

Former-commit-id: 6242392315e82599efada5893306b7a954115174
parents f7edb868 c4f704aa

core/CHANGELOG.md

+1 −0
Original line number Diff line number Diff line
@@ -42,6 +42,7 @@ Please mark all change in change log and use the ticket from JIRA. 
- MS-614 - Preload table at startup

- MS-626 - Refactor DataObj to support cache any type data

- MS-648 - Improve unittest

- MS-655 - Upgrade SPTAG



## New Feature

- MS-627 - Integrate new index: IVFSQHybrid

core/src/CMakeLists.txt

+0 −2
Original line number Diff line number Diff line
@@ -187,8 +187,6 @@ target_link_libraries(milvus_server 
install(TARGETS milvus_server DESTINATION bin)



install(FILES

        ${CMAKE_SOURCE_DIR}/src/index/thirdparty/tbb/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}

        ${CMAKE_SOURCE_DIR}/src/index/thirdparty/tbb/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}.2

        ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}

        ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3

        ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4

core/src/index/knowhere/CMakeLists.txt

+0 −14
Original line number Diff line number Diff line 
set(TBB_DIR ${CORE_SOURCE_DIR}/thirdparty/tbb)

set(TBB_LIBRARIES ${TBB_DIR}/libtbb.so)

include_directories(${TBB_DIR}/include)



include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})

link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib64)
@@ -60,7 +56,6 @@ set(index_srcs 


set(depend_libs

        SPTAGLibStatic

        ${TBB_LIBRARIES}

        faiss

        openblas

        lapack
@@ -107,13 +102,6 @@ INSTALL(FILES ${OPENBLAS_REAL_STATIC_LIB} 
        DESTINATION lib

        )



INSTALL(FILES ${CORE_SOURCE_DIR}/thirdparty/tbb/libtbb.so.2

        DESTINATION lib

        )

INSTALL(FILES ${CORE_SOURCE_DIR}/thirdparty/tbb/libtbb.so

        DESTINATION lib

        )



set(CORE_INCLUDE_DIRS

        ${CORE_SOURCE_DIR}/knowhere

        ${CORE_SOURCE_DIR}/thirdparty
@@ -122,7 +110,6 @@ set(CORE_INCLUDE_DIRS 
        ${FAISS_INCLUDE_DIR}

        ${OPENBLAS_INCLUDE_DIR}

        ${LAPACK_INCLUDE_DIR}

        ${CORE_SOURCE_DIR}/thirdparty/tbb/include

        )



set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE)
@@ -132,7 +119,6 @@ set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE) 
#        ${ARROW_INCLUDE_DIR}/arrow

#        ${FAISS_PREFIX}/include/faiss

#        ${OPENBLAS_INCLUDE_DIR}/

#        ${CORE_SOURCE_DIR}/thirdparty/tbb/include/tbb

#        DESTINATION

#        include)

#

core/src/index/knowhere/knowhere/index/vector_index/IndexKDT.cpp

+33 −28
Original line number Diff line number Diff line
@@ -36,42 +36,47 @@ BinarySet 
CPUKDTRNG::Serialize() {

    std::vector<void*> index_blobs;

    std::vector<int64_t> index_len;

    index_ptr_->SaveIndexToMemory(index_blobs, index_len);



    // TODO(zirui): dev

//    index_ptr_->SaveIndexToMemory(index_blobs, index_len);

    BinarySet binary_set;



    auto sample = std::make_shared<uint8_t>();

    sample.reset(static_cast<uint8_t*>(index_blobs[0]));

    auto tree = std::make_shared<uint8_t>();

    tree.reset(static_cast<uint8_t*>(index_blobs[1]));

    auto graph = std::make_shared<uint8_t>();

    graph.reset(static_cast<uint8_t*>(index_blobs[2]));

    auto metadata = std::make_shared<uint8_t>();

    metadata.reset(static_cast<uint8_t*>(index_blobs[3]));



    binary_set.Append("samples", sample, index_len[0]);

    binary_set.Append("tree", tree, index_len[1]);

    binary_set.Append("graph", graph, index_len[2]);

    binary_set.Append("metadata", metadata, index_len[3]);

//

//    auto sample = std::make_shared<uint8_t>();

//    sample.reset(static_cast<uint8_t*>(index_blobs[0]));

//    auto tree = std::make_shared<uint8_t>();

//    tree.reset(static_cast<uint8_t*>(index_blobs[1]));

//    auto graph = std::make_shared<uint8_t>();

//    graph.reset(static_cast<uint8_t*>(index_blobs[2]));

//    auto metadata = std::make_shared<uint8_t>();

//    metadata.reset(static_cast<uint8_t*>(index_blobs[3]));

//

//    binary_set.Append("samples", sample, index_len[0]);

//    binary_set.Append("tree", tree, index_len[1]);

//    binary_set.Append("graph", graph, index_len[2]);

//    binary_set.Append("metadata", metadata, index_len[3]);

    return binary_set;

}



void

CPUKDTRNG::Load(const BinarySet& binary_set) {

    std::vector<void*> index_blobs;



    auto samples = binary_set.GetByName("samples");

    index_blobs.push_back(samples->data.get());



    auto tree = binary_set.GetByName("tree");

    index_blobs.push_back(tree->data.get());



    auto graph = binary_set.GetByName("graph");

    index_blobs.push_back(graph->data.get());

    // TODO(zirui): dev



    auto metadata = binary_set.GetByName("metadata");

    index_blobs.push_back(metadata->data.get());



    index_ptr_->LoadIndexFromMemory(index_blobs);

//    std::vector<void*> index_blobs;

//

//    auto samples = binary_set.GetByName("samples");

//    index_blobs.push_back(samples->data.get());

//

//    auto tree = binary_set.GetByName("tree");

//    index_blobs.push_back(tree->data.get());

//

//    auto graph = binary_set.GetByName("graph");

//    index_blobs.push_back(graph->data.get());

//

//    auto metadata = binary_set.GetByName("metadata");

//    index_blobs.push_back(metadata->data.get());

//

//    index_ptr_->LoadIndexFromMemory(index_blobs);

}



// PreprocessorPtr

core/src/index/thirdparty/SPTAG/.gitignore

+0 −2
Original line number Diff line number Diff line
@@ -89,5 +89,3 @@ dkms.conf 
/Wrappers/inc/AnnClient.java

/AnnService.users - Copy.props

/.vs
 
Release/

Debug/