diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f70b1b9..67237a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,7 @@ jobs: echo "::set-output name=matrix::$(bash scripts/get-all-test-paths.sh unit 1)" - id: set-matrix-integration run: | - echo "::set-output name=matrix::$(bash scripts/get-all-test-paths.sh integration_local 1)" + echo "::set-output name=matrix::$(bash scripts/get-all-test-paths.sh integration 1)" outputs: matrix-unit: ${{ steps.set-matrix-unit.outputs.matrix }} matrix-integration: ${{ steps.set-matrix-integration.outputs.matrix }} @@ -88,6 +88,9 @@ jobs: python -m pip install --upgrade pip python -m pip install wheel pip install pytest + pip install pytest-repeat + pip install pytest-timeout + pip install flaky pip install . #pip install -U docarray[hnswlib]>=0.33.0 pip install git+https://github.com/jina-ai/jina.git@jraft-import-prints @@ -95,7 +98,7 @@ jobs: - name: Test id: test run: | - pytest -v -s ${{ matrix.test-path }} + pytest -v -s --force-flaky --min-passes 1 --max-runs 5 ${{ matrix.test-path }} timeout-minutes: 30 integration-tests: @@ -119,6 +122,9 @@ jobs: python -m pip install --upgrade pip python -m pip install wheel pip install pytest + pip install pytest-repeat + pip install pytest-timeout + pip install flaky pip install . #pip install -U docarray[hnswlib]>=0.33.0 pip install git+https://github.com/jina-ai/jina.git@jraft-import-prints @@ -126,7 +132,7 @@ jobs: - name: Test id: test run: | - pytest -v -s ${{ matrix.test-path }} + pytest -v -s --force-flaky --min-passes 1 --max-runs 5 ${{ matrix.test-path }} timeout-minutes: 30 # just for blocking the merge until all parallel integration-tests are successful diff --git a/scripts/get-all-test-paths.sh b/scripts/get-all-test-paths.sh index 5ce9378..e3bbe35 100644 --- a/scripts/get-all-test-paths.sh +++ b/scripts/get-all-test-paths.sh @@ -7,16 +7,13 @@ DEFAULT_BATCH_SIZE=5 BATCH_SIZE="${2:-$DEFAULT_BATCH_SIZE}" declare -a unit_tests=($(find tests/unit -name "test_*.py")) -declare -a integration_tests_local=($(find tests/integration/local -name "test_*.py")) -declare -a integration_tests_jcloud=($(find tests/integration/jcloud -name "test_*.py")) +declare -a integration_tests=($(find tests/integration -name "test_*.py")) declare -a all_tests=("${unit_tests[@]}" "${integration_tests[@]}") if [ "$TEST_SUITE" == "unit" ]; then dest="$(echo "${unit_tests[@]}" | xargs -n$BATCH_SIZE)" -elif [[ "$TEST_SUITE" == "integration_local" ]]; then - dest="$(echo "${integration_tests_local[@]}" | xargs -n$BATCH_SIZE)" -elif [[ "$TEST_SUITE" == "integration_jcloud" ]]; then - dest="$(echo "${integration_tests_jcloud[@]}" | xargs -n$BATCH_SIZE)" +elif [[ "$TEST_SUITE" == "integration" ]]; then + dest="$(echo "${integration_tests[@]}" | xargs -n$BATCH_SIZE)" else dest="$(echo "${all_tests[@]}" | xargs -n$BATCH_SIZE)" fi diff --git a/setup.py b/setup.py index 4ce82ea..3da8d7c 100644 --- a/setup.py +++ b/setup.py @@ -47,6 +47,9 @@ setup( 'test': [ 'pytest', 'pytest-asyncio', + 'pytest-repeat', + 'flaky', + 'pytest-timeout' ], }, install_requires=requirements, diff --git a/tests/integration/test_hnswlib_vectordb_serve.py b/tests/integration/test_hnswlib_vectordb_serve.py index a4c84e6..1728b0b 100644 --- a/tests/integration/test_hnswlib_vectordb_serve.py +++ b/tests/integration/test_hnswlib_vectordb_serve.py @@ -1,3 +1,4 @@ +import multiprocessing import pytest import random import time @@ -22,6 +23,7 @@ def docs_to_index(): for _ in range(2000)]) +@pytest.mark.timeout(180) @pytest.mark.parametrize('shards', [1, 2]) @pytest.mark.parametrize('replicas', [1, 3]) @pytest.mark.parametrize('protocol', ['grpc', 'http', 'websocket']) @@ -42,6 +44,7 @@ def test_hnswlib_vectordb_batch(docs_to_index, replicas, shards, protocol, tmpdi assert res.scores[0] < 0.001 # some precision issues, should be 0.0 +@pytest.mark.timeout(270) @pytest.mark.parametrize('limit', [1, 10, 2000, 2500]) @pytest.mark.parametrize('shards', [1, 2]) @pytest.mark.parametrize('replicas', [1, 3]) @@ -61,6 +64,7 @@ def test_hnswlib_vectordb_single_query(docs_to_index, limit, replicas, shards, p assert resp.scores[0] < 0.001 # some precision issues, should be 0.0 +@pytest.mark.timeout(180) @pytest.mark.parametrize('shards', [1, 2]) @pytest.mark.parametrize('replicas', [1, 3]) @pytest.mark.parametrize('protocol', ['grpc', 'http', 'websocket']) @@ -90,6 +94,7 @@ def test_hnswlib_vectordb_delete(docs_to_index, replicas, shards, protocol, tmpd assert resp.text != resp.matches[0].text +@pytest.mark.timeout(180) @pytest.mark.parametrize('shards', [1, 2]) @pytest.mark.parametrize('replicas', [1, 3]) @pytest.mark.parametrize('protocol', ['grpc', 'http', 'websocket']) @@ -118,6 +123,7 @@ def test_hnswlib_vectordb_udpate_text(docs_to_index, replicas, shards, protocol, assert resp.matches[0].text == resp.text + '_changed' +@pytest.mark.timeout(180) @pytest.mark.parametrize('shards', [1, 2]) @pytest.mark.parametrize('replicas', [1, 3]) @pytest.mark.parametrize('protocol', ['grpc', 'http', 'websocket']) diff --git a/tests/integration/test_inmemory_vectordb_serve.py b/tests/integration/test_inmemory_vectordb_serve.py index ac42ffc..a150c3f 100644 --- a/tests/integration/test_inmemory_vectordb_serve.py +++ b/tests/integration/test_inmemory_vectordb_serve.py @@ -1,3 +1,4 @@ +import multiprocessing import pytest import random import string @@ -22,6 +23,7 @@ def docs_to_index(): for _ in range(2000)]) +@pytest.mark.timeout(180) @pytest.mark.parametrize('shards', [1, 2]) @pytest.mark.parametrize('replicas', [1, 3]) @pytest.mark.parametrize('protocol', ['grpc', 'http', 'websocket']) @@ -42,6 +44,7 @@ def test_inmemory_vectordb_batch(docs_to_index, replicas, shards, protocol, tmpd assert res.scores[0] > 0.99 # some precision issues, should be 1.0 +@pytest.mark.timeout(270) @pytest.mark.parametrize('limit', [1, 10, 2000, 2500]) @pytest.mark.parametrize('shards', [1, 2]) @pytest.mark.parametrize('replicas', [1, 3]) @@ -61,6 +64,7 @@ def test_inmemory_vectordb_single_query(docs_to_index, limit, replicas, shards, assert resp.scores[0] > 0.99 # some precision issues, should be 1.0 +@pytest.mark.timeout(180) @pytest.mark.parametrize('shards', [1, 2]) @pytest.mark.parametrize('replicas', [1, 3]) @pytest.mark.parametrize('protocol', ['grpc', 'http', 'websocket']) @@ -90,6 +94,7 @@ def test_inmemory_vectordb_delete(docs_to_index, replicas, shards, protocol, tmp assert resp.text != resp.matches[0].text +@pytest.mark.timeout(180) @pytest.mark.parametrize('shards', [1, 2]) @pytest.mark.parametrize('replicas', [1, 3]) @pytest.mark.parametrize('protocol', ['grpc', 'http', 'websocket']) @@ -118,6 +123,7 @@ def test_inmemory_vectordb_udpate_text(docs_to_index, replicas, shards, protocol assert resp.matches[0].text == resp.text + '_changed' +@pytest.mark.timeout(180) @pytest.mark.parametrize('shards', [1, 2]) @pytest.mark.parametrize('replicas', [1, 3]) @pytest.mark.parametrize('protocol', ['grpc', 'http', 'websocket'])