Merge pull request #137 from tarilabs/tarilabs-20241018-sync

periodic sync upstream KF to midstream ODH

.github/workflows/build-image-pr.yml

@@ -1,5 +1,6 @@
 name: Test container image build and deployment
 on:
+  workflow_dispatch:
   pull_request:
     paths-ignore:
       - "LICENSE*"
@@ -9,7 +10,7 @@ on:
       - ".github/ISSUE_TEMPLATE/**"
       - ".github/dependabot.yml"
       - "docs/**"
-      - "clients/python/docs/**"
+      - "clients/python/**"
 env:
   IMG_ORG: opendatahub
   IMG_REPO: model-registry
@@ -30,8 +31,8 @@ jobs:
       - name: Build Image
         shell: bash
         env:
-          VERSION: ${{ steps.tags.outputs.tag }}
-        run: ./scripts/build_deploy.sh
+          IMG_VERSION: ${{ steps.tags.outputs.tag }}
+        run: make image/build
       - name: Start Kind Cluster
         uses: helm/[email protected]
         with:

.github/workflows/python-tests.yml

@@ -3,6 +3,7 @@ on:
   push:
     branches:
       - "main"
+  workflow_dispatch:
   pull_request:
     paths-ignore:
       - "LICENSE*"
@@ -12,26 +13,19 @@ on:
       - ".github/ISSUE_TEMPLATE/**"
       - ".github/dependabot.yml"
       - "docs/**"
+
 jobs:
-  tests:
-    name: ${{ matrix.session }} ${{ matrix.python }}
+  lint:
+    name: ${{ matrix.session }}
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         python: ["3.12"]
-        session: [lint, tests, mypy, docs-build]
-        include:
-          - python: "3.9"
-            session: tests
-          - python: "3.10"
-            session: tests
-          - python: "3.11"
-            session: tests
+        session: [lint, mypy]
     env:
       NOXSESSION: ${{ matrix.session }}
       FORCE_COLOR: "1"
-      PRE_COMMIT_COLOR: "always"
     steps:
       - name: Check out the repository
         uses: actions/checkout@v4
@@ -61,34 +55,152 @@ jobs:
           pipx install --pip-args=--constraint=${{ github.workspace }}/.github/workflows/constraints.txt nox
           pipx inject --pip-args=--constraint=${{ github.workspace }}/.github/workflows/constraints.txt nox nox-poetry
           nox --version
-      - name: Run Nox
+      - name: Nox lint
         working-directory: clients/python
         run: |
-          if [[ ${{ matrix.session }} == "tests" ]]; then
-            make build-mr
-            nox --python=${{ matrix.python }} -- --cov-report=xml
-            poetry build
-          elif [[ ${{ matrix.session }} == "mypy" ]]; then
+          if [[ ${{ matrix.session }} == "mypy" ]]; then
             nox --python=${{ matrix.python }} ||\
               echo "::error title='mypy failure'::Check the logs for more details"
           else
             nox --python=${{ matrix.python }}
           fi
-      - name: Upload coverage report
+
+  test:
+    name: Test against Py ${{ matrix.python }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ["3.12", "3.11", "3.10", "3.9"]
+    env:
+      FORCE_COLOR: "1"
+      IMG_ORG: opendatahub
+      IMG_REPO: model-registry
+      MR_NAMESPACE: kubeflow # used for kind testing, align to /manifest
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Upgrade pip
+        run: |
+          pip install --constraint=.github/workflows/constraints.txt pip
+          pip --version
+      - name: Upgrade pip in virtual environments
+        shell: python
+        run: |
+          import os
+          import pip
+
+          with open(os.environ["GITHUB_ENV"], mode="a") as io:
+              print(f"VIRTUALENV_PIP={pip.__version__}", file=io)
+      - name: Install Poetry
+        # use absolute path as recommended with: https://github.com/pypa/pipx/issues/1331
+        run: |
+          pipx install --pip-args=--constraint=${{ github.workspace }}/.github/workflows/constraints.txt poetry
+          poetry --version
+      - name: Install Nox
+        run: |
+          pipx install --pip-args=--constraint=${{ github.workspace }}/.github/workflows/constraints.txt nox
+          pipx inject --pip-args=--constraint=${{ github.workspace }}/.github/workflows/constraints.txt nox nox-poetry
+          nox --version
+      - name: Nox test
+        working-directory: clients/python
+        run: |
+          nox --python=${{ matrix.python }} --session=tests
+      - name: Generate Tag
+        shell: bash
+        id: tags
+        run: |
+          commit_sha=${{ github.event.after }}
+          tag=main-${commit_sha:0:7}
+          echo "tag=${tag}" >> $GITHUB_OUTPUT
+      - name: Build Image
+        shell: bash
+        env:
+          IMG_VERSION: ${{ steps.tags.outputs.tag }}
+        run: make image/build
+      - name: Start Kind Cluster
+        uses: helm/[email protected]
+        with:
+          node_image: "kindest/node:v1.27.11"
+          cluster_name: chart-testing-py-${{ matrix.python }}
+      - name: Load Local Registry Test Image
+        env:
+          IMG: "quay.io/${{ env.IMG_ORG }}/${{ env.IMG_REPO }}:${{ steps.tags.outputs.tag }}"
+        run: |
+          kind load docker-image -n chart-testing-py-${{ matrix.python }} ${IMG}
+      - name: Deploy Model Registry using manifests
+        env:
+          IMG: "quay.io/${{ env.IMG_ORG }}/${{ env.IMG_REPO }}:${{ steps.tags.outputs.tag }}"
+        run: ./scripts/deploy_on_kind.sh
+      - name: Nox test end-to-end
+        working-directory: clients/python
+        run: |
+          kubectl port-forward -n ${{ env.MR_NAMESPACE }} service/model-registry-service 8080:8080 &
+          sleep 2
+          nox --python=${{ matrix.python }} --session=e2e -- --cov-report=xml
+      - name: Upload coverage report # we upload coverage stats for py once, regardless if previous step failed
         uses: codecov/[email protected]
-        if: always() && matrix.session == 'tests'
+        if: always() && matrix.python == '3.12'
         with:
           files: coverage.xml
           fail_ci_if_error: true
           token: ${{ secrets.CODECOV_TOKEN }}
+
+  docs-build:
+    name: ${{ matrix.session }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ["3.12"]
+        session: [docs-build]
+    env:
+      NOXSESSION: ${{ matrix.session }}
+      FORCE_COLOR: "1"
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Upgrade pip
+        run: |
+          pip install --constraint=.github/workflows/constraints.txt pip
+          pip --version
+      - name: Upgrade pip in virtual environments
+        shell: python
+        run: |
+          import os
+          import pip
+
+          with open(os.environ["GITHUB_ENV"], mode="a") as io:
+              print(f"VIRTUALENV_PIP={pip.__version__}", file=io)
+      - name: Install Poetry
+        # use absolute path as recommended with: https://github.com/pypa/pipx/issues/1331
+        run: |
+          pipx install --pip-args=--constraint=${{ github.workspace }}/.github/workflows/constraints.txt poetry
+          poetry --version
+      - name: Install Nox
+        run: |
+          pipx install --pip-args=--constraint=${{ github.workspace }}/.github/workflows/constraints.txt nox
+          pipx inject --pip-args=--constraint=${{ github.workspace }}/.github/workflows/constraints.txt nox nox-poetry
+          nox --version
+      - name: Run Nox
+        working-directory: clients/python
+        run: |
+          nox --python=${{ matrix.python }}
+          poetry build
       - name: Upload dist
-        if: matrix.session == 'tests' && matrix.python == '3.12'
         uses: actions/upload-artifact@v4
         with:
           name: py-dist
           path: clients/python/dist
       - name: Upload documentation
-        if: matrix.session == 'docs-build'
         uses: actions/upload-artifact@v4
         with:
           name: py-docs

clients/python/Makefile

@@ -14,12 +14,13 @@ install:
 clean:
 	rm -rf src/mr_openapi
 
-.PHONY: build-mr
-build-mr:
-	cd ../../ && IMG_REGISTRY=${IMG_REGISTRY} IMG_VERSION=${IMG_VERSION} make image/build
+.PHONY: deploy-latest-mr
+deploy-latest-mr:
+	cd ../../ && IMG_VERSION=${IMG_VERSION} make image/build && LOCAL=1 ./scripts/deploy_on_kind.sh
+	kubectl port-forward -n kubeflow services/model-registry-service 8080:8080 &
 
 .PHONY: test-e2e
-test-e2e: build-mr
+test-e2e: deploy-latest-mr
 	poetry run pytest --e2e -s
 
 .PHONY: test

clients/python/noxfile.py

@@ -41,8 +41,8 @@ def lint(session: Session) -> None:
 @session(python=python_versions)
 def mypy(session: Session) -> None:
     """Type check using mypy."""
-    session.install(".")
     session.install(
+        ".",
         "mypy",
         "types-python-dateutil",
     )
@@ -53,21 +53,31 @@ def mypy(session: Session) -> None:
 @session(python=python_versions)
 def tests(session: Session) -> None:
     """Run the test suite."""
-    session.install(".")
     session.install(
-        "coverage[toml]",
+        ".",
+        "requests",
+        "pytest",
+        "pytest-asyncio",
+    )
+    session.run(
+        "pytest",
+        *session.posargs,
+    )
+
+
+@session(name="e2e", python=python_versions)
+def e2e_tests(session: Session) -> None:
+    """Run the test suite."""
+    session.install(
+        ".",
+        "requests",
         "pytest",
         "pytest-asyncio",
-        "nest-asyncio",
+        "coverage[toml]",
         "pytest-cov",
-        "pygments",
         "huggingface-hub",
     )
     try:
-        session.run(
-            "pytest",
-            *session.posargs,
-        )
         session.run(
             "pytest",
             "--e2e",
@@ -101,8 +111,12 @@ def docs_build(session: Session) -> None:
     if not session.posargs and "FORCE_COLOR" in os.environ:
         args.insert(0, "--color")
 
-    session.install(".")
-    session.install("sphinx", "myst-parser[linkify]", "furo")
+    session.install(
+        ".",
+        "sphinx",
+        "myst-parser[linkify]",
+        "furo",
+    )
 
     build_dir = Path("docs", "_build")
     if build_dir.exists():
@@ -115,8 +129,13 @@ def docs_build(session: Session) -> None:
 def docs(session: Session) -> None:
     """Build and serve the documentation with live reloading on file changes."""
     args = session.posargs or ["--open-browser", "docs", "docs/_build"]
-    session.install(".")
-    session.install("sphinx", "myst-parser[linkify]", "furo", "sphinx-autobuild")
+    session.install(
+        ".",
+        "sphinx",
+        "myst-parser[linkify]",
+        "furo",
+        "sphinx-autobuild",
+    )
 
     build_dir = Path("docs", "_build")
     if build_dir.exists():