Speed up test pipeline (#831)

* Convert test pipeline into matrix across tox envs * Reorder tox envs * If each worker runs a diff tox env, no need to maximise space * Rename base env to mlserver * Only install tox in test pipeline * Fix relative path * Revert back to single dev.txt reqs * Remove unneeded extra reqs * Move around dev deps installation to make pip's life easier * Fix HF tests * Add types-requests to main dev deps * Fix all-runtime tests * Convert all tests folders into Python packages and change Pytest's importmode * Change test imports to relative * Use python -m pytest to call pytest * Format
SeldonIO · Nov 9, 2022 · 4377f21 · 4377f21
1 parent 9d95eb7
commit 4377f21
Show file tree

Hide file tree

Showing 20 changed files with 86 additions and 74 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -29,22 +29,25 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.8", "3.9", "3.10"]
+        tox-environment:
+          - mlserver
+          - sklearn
+          - xgboost
+          - lightgbm
+          - mlflow
+          - huggingface
+          - alibi-explain
+          - alibi-detect
+          - all-runtimes
     steps:
-      - name: Maximize build space
-        uses: easimon/maximize-build-space@master
-        with:
-          remove-dotnet: "true"
-          remove-haskell: "true"
-          remove-android: "true"
-          root-reserve-mb: "20480"
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
-          make install-dev
+          pip install -r requirements/dev.txt
       - name: Test
         run: |
-          make test
+          tox -e ${{ matrix.tox-environment }}
diff --git a/requirements/dev.txt b/requirements/dev.txt
@@ -32,6 +32,7 @@ mypy-protobuf==3.1.0
 types-protobuf==3.20.4.1
 types-orjson==3.6.2
 types-aiofiles==22.1.0
+types-requests==2.28.11.2
 black==22.10.0
 
 # Pushing to PyPi

diff --git a/runtimes/alibi-detect/tests/__init__.py b/runtimes/alibi-detect/tests/__init__.py
diff --git a/runtimes/alibi-explain/requirements/dev.txt b/runtimes/alibi-explain/requirements/dev.txt
diff --git a/runtimes/alibi-explain/setup.py b/runtimes/alibi-explain/setup.py
@@ -38,7 +38,7 @@ def _load_description() -> str:
         "alibi[shap, tensorflow]",
         "orjson",
         # numba 0.55.0 requires: numpy <1.22 (for Shap / alibi)
-        "numpy<1.22",
+        "numpy<=1.22",
     ],
     long_description=_load_description(),
     long_description_content_type="text/markdown",

diff --git a/runtimes/alibi-explain/tests/__init__.py b/runtimes/alibi-explain/tests/__init__.py
diff --git a/runtimes/alibi-explain/tests/conftest.py b/runtimes/alibi-explain/tests/conftest.py
@@ -27,8 +27,8 @@
 from mlserver_alibi_explain.common import AlibiExplainSettings
 from mlserver_alibi_explain.runtime import AlibiExplainRuntime, AlibiExplainRuntimeBase
 
-from helpers.tf_model import get_tf_mnist_model_uri
-from helpers.run_async import run_async_as_sync
+from .helpers.tf_model import get_tf_mnist_model_uri, TFMNISTModel
+from .helpers.run_async import run_async_as_sync
 
 TESTS_PATH = Path(os.path.dirname(__file__))
 _ANCHOR_IMAGE_DIR = TESTS_PATH / ".data" / "mnist_anchor_image"
@@ -55,7 +55,7 @@ def event_loop():
 def custom_runtime_tf_settings() -> ModelSettings:
     return ModelSettings(
         name="custom_tf_mnist_model",
-        implementation="helpers.tf_model.TFMNISTModel",
+        implementation=TFMNISTModel,
     )
 
 

diff --git a/runtimes/alibi-explain/tests/test_alibi_runtime_base.py b/runtimes/alibi-explain/tests/test_alibi_runtime_base.py
@@ -28,7 +28,7 @@
 from mlserver_alibi_explain.runtime import AlibiExplainRuntime, AlibiExplainRuntimeBase
 from mlserver_alibi_explain.errors import InvalidExplanationShape
 
-from helpers.run_async import run_async_as_sync
+from .helpers.run_async import run_async_as_sync
 
 """
 Smoke tests for runtimes

diff --git a/runtimes/alibi-explain/tests/test_black_box.py b/runtimes/alibi-explain/tests/test_black_box.py
@@ -9,7 +9,6 @@
 from alibi.saving import load_explainer
 from numpy.testing import assert_allclose
 
-from helpers.tf_model import get_tf_mnist_model_uri
 from mlserver import MLModel
 from mlserver.codecs import NumpyCodec, StringCodec
 from mlserver.types import (
@@ -28,6 +27,8 @@
     _DEFAULT_INPUT_NAME,
 )
 
+from .helpers.tf_model import get_tf_mnist_model_uri
+
 TESTS_PATH = Path(os.path.dirname(__file__))
 _DEFAULT_ID_NAME = "dummy_id"
 

diff --git a/runtimes/alibi-explain/tests/test_integrated_gradients.py b/runtimes/alibi-explain/tests/test_integrated_gradients.py
@@ -7,11 +7,12 @@
 from alibi.explainers import IntegratedGradients
 from numpy.testing import assert_array_almost_equal
 
-from helpers.tf_model import get_tf_mnist_model_uri
 from mlserver.codecs import NumpyCodec
 from mlserver.types import InferenceRequest, Parameters, RequestInput
 from mlserver_alibi_explain.common import convert_from_bytes
 
+from .helpers.tf_model import get_tf_mnist_model_uri
+
 
 @pytest.fixture()
 def payload() -> InferenceRequest:

diff --git a/runtimes/huggingface/setup.py b/runtimes/huggingface/setup.py
@@ -36,6 +36,9 @@ def _load_description() -> str:
     install_requires=[
         "mlserver",
         "optimum[onnxruntime]>=1.2.3",
+        # Pin `transformers`, otherwise we risk falling into this issue:
+        # https://github.com/huggingface/optimum/issues/344
+        "transformers<=4.21.1",
     ],
     long_description=_load_description(),
     long_description_content_type="text/markdown",

diff --git a/runtimes/huggingface/tests/__init__.py b/runtimes/huggingface/tests/__init__.py
diff --git a/runtimes/huggingface/tests/test_codecs.py b/runtimes/huggingface/tests/test_codecs.py
@@ -17,10 +17,13 @@
             InferenceRequest(
                 inputs=[
                     RequestInput(
-                        name="foo", datatype="BYTES", data=["bar1", "bar2"], shape=[2]
+                        name="foo",
+                        datatype="BYTES",
+                        data=["bar1", "bar2"],
+                        shape=[2, 1],
                     ),
                     RequestInput(
-                        name="foo2", datatype="BYTES", data=["var1"], shape=[1]
+                        name="foo2", datatype="BYTES", data=["var1"], shape=[1, 1]
                     ),
                 ]
             ),
@@ -46,7 +49,7 @@ def test_decode_request(inference_request, expected):
                         name="foo",
                         datatype="BYTES",
                         data=[b"bar1", b"bar2"],
-                        shape=[2],
+                        shape=[2, 1],
                         parameters=Parameters(
                             content_type=MultiStringRequestCodec.InputCodec.ContentType
                         ),
@@ -55,7 +58,7 @@ def test_decode_request(inference_request, expected):
                         name="foo2",
                         datatype="BYTES",
                         data=[b"var1"],
-                        shape=[1],
+                        shape=[1, 1],
                         parameters=Parameters(
                             content_type=MultiStringRequestCodec.InputCodec.ContentType
                         ),
@@ -73,7 +76,7 @@ def test_decode_request(inference_request, expected):
                         name="foo",
                         datatype="BYTES",
                         data=["bar1", "bar2"],
-                        shape=[2],
+                        shape=[2, 1],
                         parameters=Parameters(
                             content_type=MultiStringRequestCodec.InputCodec.ContentType
                         ),
@@ -82,7 +85,7 @@ def test_decode_request(inference_request, expected):
                         name="foo2",
                         datatype="BYTES",
                         data=["var1"],
-                        shape=[1],
+                        shape=[1, 1],
                         parameters=Parameters(
                             content_type=MultiStringRequestCodec.InputCodec.ContentType
                         ),
@@ -108,10 +111,13 @@ def test_encode_request(payload, use_bytes, expected):
                 model_name="my-model",
                 outputs=[
                     ResponseOutput(
-                        name="foo", datatype="BYTES", data=["bar1", "bar2"], shape=[2]
+                        name="foo",
+                        datatype="BYTES",
+                        data=["bar1", "bar2"],
+                        shape=[2, 1],
                     ),
                     ResponseOutput(
-                        name="foo2", datatype="BYTES", data=["var1"], shape=[1]
+                        name="foo2", datatype="BYTES", data=["var1"], shape=[1, 1]
                     ),
                 ],
             ),
@@ -138,7 +144,7 @@ def test_decode_response(inference_response, expected):
                         name="foo",
                         datatype="BYTES",
                         data=[b"bar1", b"bar2"],
-                        shape=[2],
+                        shape=[2, 1],
                         parameters=Parameters(
                             content_type=MultiStringRequestCodec.InputCodec.ContentType
                         ),
@@ -147,7 +153,7 @@ def test_decode_response(inference_response, expected):
                         name="foo2",
                         datatype="BYTES",
                         data=[b"var1"],
-                        shape=[1],
+                        shape=[1, 1],
                         parameters=Parameters(
                             content_type=MultiStringRequestCodec.InputCodec.ContentType
                         ),
@@ -165,7 +171,7 @@ def test_decode_response(inference_response, expected):
                         name="foo",
                         datatype="BYTES",
                         data=["bar1", "bar2"],
-                        shape=[2],
+                        shape=[2, 1],
                         parameters=Parameters(
                             content_type=MultiStringRequestCodec.InputCodec.ContentType
                         ),
@@ -174,7 +180,7 @@ def test_decode_response(inference_response, expected):
                         name="foo2",
                         datatype="BYTES",
                         data=["var1"],
-                        shape=[1],
+                        shape=[1, 1],
                         parameters=Parameters(
                             content_type=MultiStringRequestCodec.InputCodec.ContentType
                         ),

diff --git a/runtimes/lightgbm/tests/__init__.py b/runtimes/lightgbm/tests/__init__.py
diff --git a/runtimes/mlflow/requirements/dev.txt b/runtimes/mlflow/requirements/dev.txt
@@ -1,11 +1,9 @@
-scikit-learn
 torch==1.13.0
 pytorch-lightning==1.8.0.post1
 # Pin torchmetrics to a version older than 0.5.0 to avoid this issue:
 # https://github.com/PyTorchLightning/pytorch-lightning/issues/10233
 torchmetrics==0.10.2
 torchvision==0.14.0
-httpx==0.23.0
 
 # Force local tests to use MLflow 2.x
 mlflow >= 2.0.0rc0
diff --git a/runtimes/mlflow/tests/__init__.py b/runtimes/mlflow/tests/__init__.py
diff --git a/runtimes/mlflow/tests/conftest.py b/runtimes/mlflow/tests/conftest.py
@@ -15,7 +15,7 @@
 
 from mlserver_mlflow import MLflowRuntime
 
-from torch_fixtures import MNISTDataModule, LightningMNISTClassifier
+from .torch_fixtures import MNISTDataModule, LightningMNISTClassifier
 
 TESTS_PATH = os.path.dirname(__file__)
 TESTDATA_PATH = os.path.join(TESTS_PATH, "testdata")

diff --git a/runtimes/sklearn/tests/__init__.py b/runtimes/sklearn/tests/__init__.py
diff --git a/runtimes/xgboost/tests/__init__.py b/runtimes/xgboost/tests/__init__.py