test: run integration tests split

Signed-off-by: Joan Fontanals Martinez <joan.martinez@jina.ai>

.github/workflows/ci.yml

@@ -62,7 +62,7 @@ jobs:
           echo "::set-output name=matrix::$(bash scripts/get-all-test-paths.sh unit 1)"
       - id: set-matrix-integration
         run: |
-          echo "::set-output name=matrix::$(bash scripts/get-all-test-paths.sh integration_local 1)"
+          echo "::set-output name=matrix::$(bash scripts/get-all-test-paths.sh integration 1)"
     outputs:
       matrix-unit: ${{ steps.set-matrix-unit.outputs.matrix }}
       matrix-integration: ${{ steps.set-matrix-integration.outputs.matrix }}
@@ -88,14 +88,17 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install wheel
           pip install pytest
+          pip install pytest-repeat
+          pip install pytest-timeout
+          pip install flaky
           pip install .
           #pip install -U docarray[hnswlib]>=0.33.0
-          pip install git+https://github.com/jina-ai/jina.git@fix-handle-list-float
+          pip install git+https://github.com/jina-ai/jina.git@help-investigate
           pip install git+https://github.com/docarray/docarray.git@main
       - name: Test
         id: test
         run: |
-          pytest -v -s ${{ matrix.test-path }}
+          pytest -v -s --force-flaky --min-passes 1 --max-runs 5 ${{ matrix.test-path }}
         timeout-minutes: 30
 
   integration-tests:
@@ -119,14 +122,17 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install wheel
           pip install pytest
+          pip install pytest-repeat
+          pip install pytest-timeout
+          pip install flaky
           pip install .
           #pip install -U docarray[hnswlib]>=0.33.0
-          pip install git+https://github.com/jina-ai/jina.git@fix-handle-list-float
+          pip install git+https://github.com/jina-ai/jina.git@help-investigate
           pip install git+https://github.com/docarray/docarray.git@main
       - name: Test
         id: test
         run: |
-          pytest -v -s ${{ matrix.test-path }}
+          pytest -v -s --force-flaky --min-passes 1 --max-runs 5 ${{ matrix.test-path }}
         timeout-minutes: 30
 
   # just for blocking the merge until all parallel integration-tests are successful

scripts/get-all-test-paths.sh

@@ -7,16 +7,13 @@ DEFAULT_BATCH_SIZE=5
 BATCH_SIZE="${2:-$DEFAULT_BATCH_SIZE}"
 
 declare -a unit_tests=($(find tests/unit -name "test_*.py"))
-declare -a integration_tests_local=($(find tests/integration/local -name "test_*.py"))
+declare -a integration_tests=($(find tests/integration -name "test_*.py"))
-declare -a integration_tests_jcloud=($(find tests/integration/jcloud -name "test_*.py"))
 declare -a all_tests=("${unit_tests[@]}" "${integration_tests[@]}")
 
 if [ "$TEST_SUITE" == "unit" ]; then
     dest="$(echo "${unit_tests[@]}" | xargs -n$BATCH_SIZE)"
-elif [[ "$TEST_SUITE" == "integration_local" ]]; then
+elif [[ "$TEST_SUITE" == "integration" ]]; then
-    dest="$(echo "${integration_tests_local[@]}" | xargs -n$BATCH_SIZE)"
+    dest="$(echo "${integration_tests[@]}" | xargs -n$BATCH_SIZE)"
-elif [[ "$TEST_SUITE" == "integration_jcloud" ]]; then
-    dest="$(echo "${integration_tests_jcloud[@]}" | xargs -n$BATCH_SIZE)"
 else
     dest="$(echo "${all_tests[@]}" | xargs -n$BATCH_SIZE)"
 fi

setup.py

@@ -47,6 +47,9 @@ setup(
         'test': [
             'pytest',
             'pytest-asyncio',
+            'pytest-repeat',
+            'flaky',
+            'pytest-timeout'
         ],
     },
     install_requires=requirements,

tests/integration/test_hnswlib_vectordb_serve.py

@@ -1,3 +1,4 @@
+import multiprocessing
 import pytest
 import random
 import time
@@ -22,6 +23,7 @@ def docs_to_index():
          for _ in range(2000)])
 
 
+@pytest.mark.timeout(180)
 @pytest.mark.parametrize('shards', [1, 2])
 @pytest.mark.parametrize('replicas', [1, 3])
 @pytest.mark.parametrize('protocol', ['grpc', 'http', 'websocket'])
@@ -42,6 +44,7 @@ def test_hnswlib_vectordb_batch(docs_to_index, replicas, shards, protocol, tmpdi
             assert res.scores[0] < 0.001  # some precision issues, should be 0.0
 
 
+@pytest.mark.timeout(270)
 @pytest.mark.parametrize('limit', [1, 10, 2000, 2500])
 @pytest.mark.parametrize('shards', [1, 2])
 @pytest.mark.parametrize('replicas', [1, 3])
@@ -61,6 +64,7 @@ def test_hnswlib_vectordb_single_query(docs_to_index, limit, replicas, shards, p
         assert resp.scores[0] < 0.001  # some precision issues, should be 0.0
 
 
+@pytest.mark.timeout(180)
 @pytest.mark.parametrize('shards', [1, 2])
 @pytest.mark.parametrize('replicas', [1, 3])
 @pytest.mark.parametrize('protocol', ['grpc', 'http', 'websocket'])
@@ -90,6 +94,7 @@ def test_hnswlib_vectordb_delete(docs_to_index, replicas, shards, protocol, tmpd
         assert resp.text != resp.matches[0].text
 
 
+@pytest.mark.timeout(180)
 @pytest.mark.parametrize('shards', [1, 2])
 @pytest.mark.parametrize('replicas', [1, 3])
 @pytest.mark.parametrize('protocol', ['grpc', 'http', 'websocket'])
@@ -118,6 +123,7 @@ def test_hnswlib_vectordb_udpate_text(docs_to_index, replicas, shards, protocol,
         assert resp.matches[0].text == resp.text + '_changed'
 
 
+@pytest.mark.timeout(180)
 @pytest.mark.parametrize('shards', [1, 2])
 @pytest.mark.parametrize('replicas', [1, 3])
 @pytest.mark.parametrize('protocol', ['grpc', 'http', 'websocket'])

tests/integration/test_inmemory_vectordb_serve.py

@@ -1,3 +1,4 @@
+import multiprocessing
 import pytest
 import random
 import string
@@ -22,6 +23,7 @@ def docs_to_index():
          for _ in range(2000)])
 
 
+@pytest.mark.timeout(180)
 @pytest.mark.parametrize('shards', [1, 2])
 @pytest.mark.parametrize('replicas', [1, 3])
 @pytest.mark.parametrize('protocol', ['grpc', 'http', 'websocket'])
@@ -42,6 +44,7 @@ def test_inmemory_vectordb_batch(docs_to_index, replicas, shards, protocol, tmpd
             assert res.scores[0] > 0.99  # some precision issues, should be 1.0
 
 
+@pytest.mark.timeout(270)
 @pytest.mark.parametrize('limit', [1, 10, 2000, 2500])
 @pytest.mark.parametrize('shards', [1, 2])
 @pytest.mark.parametrize('replicas', [1, 3])
@@ -61,6 +64,7 @@ def test_inmemory_vectordb_single_query(docs_to_index, limit, replicas, shards,
         assert resp.scores[0] > 0.99  # some precision issues, should be 1.0
 
 
+@pytest.mark.timeout(180)
 @pytest.mark.parametrize('shards', [1, 2])
 @pytest.mark.parametrize('replicas', [1, 3])
 @pytest.mark.parametrize('protocol', ['grpc', 'http', 'websocket'])
@@ -90,6 +94,7 @@ def test_inmemory_vectordb_delete(docs_to_index, replicas, shards, protocol, tmp
         assert resp.text != resp.matches[0].text
 
 
+@pytest.mark.timeout(180)
 @pytest.mark.parametrize('shards', [1, 2])
 @pytest.mark.parametrize('replicas', [1, 3])
 @pytest.mark.parametrize('protocol', ['grpc', 'http', 'websocket'])
@@ -118,6 +123,7 @@ def test_inmemory_vectordb_udpate_text(docs_to_index, replicas, shards, protocol
         assert resp.matches[0].text == resp.text + '_changed'
 
 
+@pytest.mark.timeout(180)
 @pytest.mark.parametrize('shards', [1, 2])
 @pytest.mark.parametrize('replicas', [1, 3])
 @pytest.mark.parametrize('protocol', ['grpc', 'http', 'websocket'])