Commit 3a87f16b authored by Nicky's avatar Nicky
Browse files

Merge remote-tracking branch 'official/0.6.0' into issue-548

parents d721e385 9b389264

CHANGELOG.md

+2 −0
Original line number Diff line number Diff line
@@ -43,6 +43,7 @@ Please mark all change in change log and use the ticket from JIRA. 
- \#599 - Build index log is incorrect

- \#602 - Optimizer specify wrong gpu_id

- \#606 - No log generated during building index with CPU

- \#631 - FAISS isn't compiled with O3 option



## Feature

- \#12 - Pure CPU version for Milvus
@@ -77,6 +78,7 @@ Please mark all change in change log and use the ticket from JIRA. 
- \#470 - Small raw files should not be build index

- \#584 - Intergrate internal FAISS

- \#611 - Remove MILVUS_CPU_VERSION

- \#634 - FAISS GPU version is compiled with O0



## Task

core/src/index/cmake/ThirdPartyPackagesCore.cmake

+1 −1
Original line number Diff line number Diff line
@@ -708,7 +708,7 @@ macro(build_faiss) 
    set(FAISS_CONFIGURE_ARGS

            "--prefix=${FAISS_PREFIX}"

            "CFLAGS=${EP_C_FLAGS}"

            "CXXFLAGS=${EP_CXX_FLAGS} -mavx2 -mf16c"

            "CXXFLAGS=${EP_CXX_FLAGS} -mavx2 -mf16c -O3"

            --without-python)



    if (FAISS_WITH_MKL)

core/src/index/thirdparty/faiss/makefile.inc.in

+1 −1
Original line number Diff line number Diff line
@@ -17,7 +17,7 @@ NVCC = @NVCC@ 
CUDA_ROOT    = @CUDA_PREFIX@

CUDA_ARCH    = @CUDA_ARCH@

NVCCFLAGS    = -I $(CUDA_ROOT)/targets/x86_64-linux/include/ \

-O0 -g \

-O3 \

-Xcompiler -fPIC \

-Xcudafe --diag_suppress=unrecognized_attribute \

$(CUDA_ARCH) \

tests/milvus_python_test/test_add_vectors.py

+2 −0
Original line number Diff line number Diff line
@@ -31,6 +31,8 @@ class TestAddBase: 
        if "internal" not in args:

            if request.param["index_type"] == IndexType.IVF_SQ8H:

                pytest.skip("sq8h not support in open source")

            if request.param["index_type"] == IndexType.IVF_PQ:

                pytest.skip("Skip PQ Temporary")

        return request.param



    def test_add_vector_create_table(self, connect, table):

tests/milvus_python_test/test_index.py

+154 −141
Original line number Diff line number Diff line
@@ -14,7 +14,7 @@ from utils import * 


nb = 10000

dim = 128

index_file_size = 10

index_file_size = 20

vectors = gen_vectors(nb, dim)

vectors = sklearn.preprocessing.normalize(vectors, axis=1, norm='l2')

vectors = vectors.tolist()
@@ -63,6 +63,18 @@ class TestIndexBase: 
        status = connect.create_index(table, index_params)

        assert status.OK()



    @pytest.mark.timeout(BUILD_TIMEOUT)

    def test_create_index_no_vectors(self, connect, table, get_simple_index_params):

        '''

        target: test create index interface

        method: create table and add vectors in it, create index

        expected: return code equals to 0, and search success

        '''

        index_params = get_simple_index_params

        logging.getLogger().info(index_params)

        status = connect.create_index(table, index_params)

        assert status.OK()



    @pytest.mark.timeout(BUILD_TIMEOUT)

    def test_create_index_partition(self, connect, table, get_simple_index_params):

        '''
@@ -72,6 +84,8 @@ class TestIndexBase: 
        '''

        partition_name = gen_unique_str()

        index_params = get_simple_index_params

        if index_params["index_type"] == IndexType.IVF_PQ:

            pytest.skip("Skip some PQ cases")

        logging.getLogger().info(index_params)

        status = connect.create_partition(table, partition_name, tag)

        status, ids = connect.add_vectors(table, vectors, partition_tag=tag)
@@ -242,6 +256,8 @@ class TestIndexBase: 
        expected: return code equals to 0

        '''

        index_param = get_simple_index_params

        if index_param["index_type"] == IndexType.IVF_PQ:

            pytest.skip("Skip some PQ cases")

        status = connect.create_index(table, index_param)

        status, ids = connect.add_vectors(table, vectors)

        assert status.OK()
@@ -255,6 +271,8 @@ class TestIndexBase: 
        '''

        status, ids = connect.add_vectors(table, vectors)

        index_param = get_simple_index_params

        if index_param["index_type"] == IndexType.IVF_PQ:

            pytest.skip("Skip some PQ cases")

        status = connect.create_index(table, index_param)

        status = connect.create_index(table, index_param)

        assert status.OK()
@@ -291,15 +309,15 @@ class TestIndexBase: 
    ******************************************************************

    """



    def test_describe_index(self, connect, table, get_simple_index_params):

    def test_describe_index(self, connect, table, get_index_params):

        '''

        target: test describe index interface

        method: create table and add vectors in it, create index, call describe index

        expected: return code 0, and index instructure

        '''

        index_params = get_simple_index_params

        index_params = get_index_params

        logging.getLogger().info(index_params)

        status, ids = connect.add_vectors(table, vectors)

        # status, ids = connect.add_vectors(table, vectors)

        status = connect.create_index(table, index_params)

        status, result = connect.describe_index(table)

        logging.getLogger().info(result)
@@ -325,6 +343,8 @@ class TestIndexBase: 
                     'metric_type': MetricType.L2}

            connect.create_table(param)

            index_params = get_simple_index_params

            if index_params["index_type"] == IndexType.IVF_PQ:

                pytest.skip("Skip some PQ cases")

            logging.getLogger().info(index_params)

            status, ids = connect.add_vectors(table_name=table_name, records=vectors)

            status = connect.create_index(table_name, index_params)
@@ -405,7 +425,7 @@ class TestIndexBase: 
        expected: return code 0, and default index param

        '''

        index_param = get_simple_index_params

        status, ids = connect.add_vectors(table, vectors)

        # status, ids = connect.add_vectors(table, vectors)

        status = connect.create_index(table, index_param)

        assert status.OK()

        status, result = connect.describe_index(table)
@@ -425,7 +445,7 @@ class TestIndexBase: 
        expected: return code 0

        '''

        index_param = get_simple_index_params

        status, ids = connect.add_vectors(table, vectors)

        # status, ids = connect.add_vectors(table, vectors)

        status = connect.create_index(table, index_param)

        assert status.OK()

        status, result = connect.describe_index(table)
@@ -494,10 +514,9 @@ class TestIndexBase: 
        expected: return code 0

        '''

        index_params = get_simple_index_params

        status, ids = connect.add_vectors(table, vectors)

        # status, ids = connect.add_vectors(table, vectors)

        for i in range(2):

            status = connect.create_index(table, index_params)



            assert status.OK()

            status, result = connect.describe_index(table)

            logging.getLogger().info(result)
@@ -517,7 +536,7 @@ class TestIndexBase: 
        '''

        nlist = 16384

        index_params = [{"index_type": IndexType.IVFLAT, "nlist": nlist}, {"index_type": IndexType.IVF_SQ8, "nlist": nlist}]

        status, ids = connect.add_vectors(table, vectors)

        # status, ids = connect.add_vectors(table, vectors)

        for i in range(2):

            status = connect.create_index(table, index_params[i])

            assert status.OK()
@@ -570,9 +589,6 @@ class TestIndexIP: 
        logging.getLogger().info(index_params)

        status, ids = connect.add_vectors(ip_table, vectors)

        status = connect.create_index(ip_table, index_params)

        if index_params["index_type"] == IndexType.IVF_PQ:

            assert not status.OK()

        else:

        assert status.OK()



    @pytest.mark.timeout(BUILD_TIMEOUT)
@@ -584,13 +600,12 @@ class TestIndexIP: 
        '''

        partition_name = gen_unique_str()

        index_params = get_simple_index_params

        if index_params["index_type"] == IndexType.IVF_PQ:

            pytest.skip("Skip some PQ cases")

        logging.getLogger().info(index_params)

        status = connect.create_partition(ip_table, partition_name, tag)

        status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag)

        status = connect.create_index(partition_name, index_params)

        if index_params["index_type"] == IndexType.IVF_PQ:

            assert not status.OK()

        else:

        assert status.OK()



    @pytest.mark.level(2)
@@ -616,10 +631,6 @@ class TestIndexIP: 
        logging.getLogger().info(index_params)

        status, ids = connect.add_vectors(ip_table, vectors)

        status = connect.create_index(ip_table, index_params)

        if index_params["index_type"] == IndexType.IVF_PQ:

            assert not status.OK()

        else:

            assert status.OK()

        logging.getLogger().info(connect.describe_index(ip_table))

        query_vecs = [vectors[0], vectors[1], vectors[2]]

        top_k = 5
@@ -734,6 +745,8 @@ class TestIndexIP: 
        expected: return code equals to 0

        '''

        index_param = get_simple_index_params

        if index_param["index_type"] == IndexType.IVF_PQ:

            pytest.skip("Skip some PQ cases")

        status = connect.create_index(ip_table, index_param)

        status, ids = connect.add_vectors(ip_table, vectors)

        assert status.OK()
@@ -792,7 +805,7 @@ class TestIndexIP: 
        '''

        index_params = get_simple_index_params

        logging.getLogger().info(index_params)

        status, ids = connect.add_vectors(ip_table, vectors)

        # status, ids = connect.add_vectors(ip_table, vectors[:5000])

        status = connect.create_index(ip_table, index_params)

        status, result = connect.describe_index(ip_table)

        logging.getLogger().info(result)
@@ -808,6 +821,8 @@ class TestIndexIP: 
        '''

        partition_name = gen_unique_str()

        index_params = get_simple_index_params

        if index_params["index_type"] == IndexType.IVF_PQ:

            pytest.skip("Skip some PQ cases")

        logging.getLogger().info(index_params)

        status = connect.create_partition(ip_table, partition_name, tag)

        status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag)
@@ -831,6 +846,8 @@ class TestIndexIP: 
        '''

        partition_name = gen_unique_str()

        index_params = get_simple_index_params

        if index_params["index_type"] == IndexType.IVF_PQ:

            pytest.skip("Skip some PQ cases")

        logging.getLogger().info(index_params)

        status = connect.create_partition(ip_table, partition_name, tag)

        status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag)
@@ -856,6 +873,8 @@ class TestIndexIP: 
        new_partition_name = gen_unique_str()

        new_tag = "new_tag"

        index_params = get_simple_index_params

        if index_params["index_type"] == IndexType.IVF_PQ:

            pytest.skip("Skip some PQ cases")

        logging.getLogger().info(index_params)

        status = connect.create_partition(ip_table, partition_name, tag)

        status = connect.create_partition(ip_table, new_partition_name, new_tag)
@@ -892,6 +911,8 @@ class TestIndexIP: 
                     'metric_type': MetricType.IP}

            connect.create_table(param)

            index_params = get_simple_index_params

            if index_params["index_type"] == IndexType.IVF_PQ:

                pytest.skip("Skip some PQ cases")

            logging.getLogger().info(index_params)

            status, ids = connect.add_vectors(table_name=table_name, records=vectors)

            status = connect.create_index(table_name, index_params)
@@ -944,14 +965,14 @@ class TestIndexIP: 
    ******************************************************************

    """



    def test_drop_index(self, connect, ip_table, get_index_params):

    def test_drop_index(self, connect, ip_table, get_simple_index_params):

        '''

        target: test drop index interface

        method: create table and add vectors in it, create index, call drop index

        expected: return code 0, and default index param

        '''

        index_params = get_index_params

        status, ids = connect.add_vectors(ip_table, vectors)

        index_params = get_simple_index_params

        # status, ids = connect.add_vectors(ip_table, vectors)

        status = connect.create_index(ip_table, index_params)

        if index_params["index_type"] == IndexType.IVF_PQ:

            assert not status.OK()
@@ -975,12 +996,11 @@ class TestIndexIP: 
        '''

        partition_name = gen_unique_str()

        index_params = get_simple_index_params

        if index_params["index_type"] == IndexType.IVF_PQ:

            pytest.skip("Skip some PQ cases")

        status = connect.create_partition(ip_table, partition_name, tag)

        status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag)

        status = connect.create_index(ip_table, index_params)

        if index_params["index_type"] == IndexType.IVF_PQ:

            assert not status.OK()

        else:

        assert status.OK()

        status, result = connect.describe_index(ip_table)

        logging.getLogger().info(result)
@@ -1000,12 +1020,11 @@ class TestIndexIP: 
        '''

        partition_name = gen_unique_str()

        index_params = get_simple_index_params

        if index_params["index_type"] == IndexType.IVF_PQ:

            pytest.skip("Skip some PQ cases")

        status = connect.create_partition(ip_table, partition_name, tag)

        status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag)

        status = connect.create_index(partition_name, index_params)

        if index_params["index_type"] == IndexType.IVF_PQ:

            assert not status.OK()

        else:

        assert status.OK()

        status = connect.drop_index(ip_table)

        assert status.OK()
@@ -1028,12 +1047,11 @@ class TestIndexIP: 
        '''

        partition_name = gen_unique_str()

        index_params = get_simple_index_params

        if index_params["index_type"] == IndexType.IVF_PQ:

            pytest.skip("Skip some PQ cases")

        status = connect.create_partition(ip_table, partition_name, tag)

        status, ids = connect.add_vectors(ip_table, vectors, partition_tag=tag)

        status = connect.create_index(partition_name, index_params)

        if index_params["index_type"] == IndexType.IVF_PQ:

            assert not status.OK()

        else:

        assert status.OK()

        status = connect.drop_index(partition_name)

        assert status.OK()
@@ -1058,13 +1076,12 @@ class TestIndexIP: 
        new_partition_name = gen_unique_str()

        new_tag = "new_tag"

        index_params = get_simple_index_params

        if index_params["index_type"] == IndexType.IVF_PQ:

            pytest.skip("Skip some PQ cases")

        status = connect.create_partition(ip_table, partition_name, tag)

        status = connect.create_partition(ip_table, new_partition_name, new_tag)

        status, ids = connect.add_vectors(ip_table, vectors)

        status = connect.create_index(ip_table, index_params)

        if index_params["index_type"] == IndexType.IVF_PQ:

            assert not status.OK()

        else:

        assert status.OK()

        status = connect.drop_index(new_partition_name)

        assert status.OK()
@@ -1091,11 +1108,8 @@ class TestIndexIP: 
        expected: return code 0

        '''

        index_params = get_simple_index_params

        status, ids = connect.add_vectors(ip_table, vectors)

        # status, ids = connect.add_vectors(ip_table, vectors)

        status = connect.create_index(ip_table, index_params)

        if index_params["index_type"] == IndexType.IVF_PQ:

            assert not status.OK()

        else:

        assert status.OK()

        status, result = connect.describe_index(ip_table)

        logging.getLogger().info(result)
@@ -1145,12 +1159,11 @@ class TestIndexIP: 
        expected: return code 0

        '''

        index_params = get_simple_index_params

        if index_params["index_type"] == IndexType.IVF_PQ:

            pytest.skip("Skip some PQ cases")

        status, ids = connect.add_vectors(ip_table, vectors)

        for i in range(2):

            status = connect.create_index(ip_table, index_params)

            if index_params["index_type"] == IndexType.IVF_PQ:

                assert not status.OK()

            else:

            assert status.OK()

            status, result = connect.describe_index(ip_table)

            logging.getLogger().info(result)
@@ -1200,7 +1213,7 @@ class TestIndexTableInvalid(object): 
    def get_table_name(self, request):

        yield request.param



    @pytest.mark.level(2)

    @pytest.mark.level(1)

    def test_create_index_with_invalid_tablename(self, connect, get_table_name):

        table_name = get_table_name

        nlist = 16384
@@ -1208,13 +1221,13 @@ class TestIndexTableInvalid(object): 
        status = connect.create_index(table_name, index_param)

        assert not status.OK()



    @pytest.mark.level(2)

    @pytest.mark.level(1)

    def test_describe_index_with_invalid_tablename(self, connect, get_table_name):

        table_name = get_table_name

        status, result = connect.describe_index(table_name)

        assert not status.OK()   



    @pytest.mark.level(2)

    @pytest.mark.level(1)

    def test_drop_index_with_invalid_tablename(self, connect, get_table_name):

        table_name = get_table_name

        status = connect.drop_index(table_name)
@@ -1232,13 +1245,13 @@ class TestCreateIndexParamsInvalid(object): 
    def get_index_params(self, request):

        yield request.param



    @pytest.mark.level(2)

    @pytest.mark.level(1)

    def test_create_index_with_invalid_index_params(self, connect, table, get_index_params):

        index_params = get_index_params

        index_type = index_params["index_type"]

        nlist = index_params["nlist"]

        logging.getLogger().info(index_params)

        status, ids = connect.add_vectors(table, vectors)

        # status, ids = connect.add_vectors(table, vectors)

        if (not index_type) or (not nlist) or (not isinstance(index_type, IndexType)) or (not isinstance(nlist, int)):

            with pytest.raises(Exception) as e:

                status = connect.create_index(table, index_params)