draft: PoC for refactor CI epic

.github/workflows/integrate.yaml

@@ -66,20 +66,42 @@ jobs:
  - run: python3 -m pip install tox
  - run: tox -e ${{ matrix.charm }}-unit
 
+ build:
+ name: Build charm
+ uses: canonical/data-platform-workflows/.github/workflows/[email protected]
+ strategy:
+ fail-fast: false
+ matrix:
+ charm:
+ - kfp-api
+ - kfp-metadata-writer
+ - kfp-persistence
+ - kfp-profile-controller
+ - kfp-schedwf
+ - kfp-ui
+ - kfp-viewer
+ - kfp-viz
+ with:
+ cache: ${{ github.event_name == 'pull_request' }}
+ charmcraft-snap-channel: 3.x/edge
+ path-to-charm-directory: ./charms/${{ matrix.charm }}
+
+
  integration:
  name: Integration tests (microk8s)
  runs-on: ubuntu-20.04
+ needs: [build]
  strategy:
  fail-fast: false
  matrix:
  charm:
+ - kfp-api
+ - kfp-metadata-writer
  - kfp-persistence
  - kfp-profile-controller
- - kfp-api
  - kfp-schedwf
- - kfp-viewer
  - kfp-ui
- - kfp-metadata-writer
+ - kfp-viewer
  - kfp-viz
  steps:
  # Ideally we'd use self-hosted runners, but this effort is still not stable
@@ -90,7 +112,7 @@ jobs:
  - name: Maximise GH runner space
  uses: jlumbroso/[email protected]
 
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
 
  - name: Setup operator environment
  uses: charmed-kubernetes/actions-operator@main
@@ -101,12 +123,20 @@ jobs:
  # Pinned to 3.x/stable due to https://github.com/canonical/charmcraft/issues/1845
  charmcraft-channel: 3.x/stable
 
+ - name: Download packed charm(s)
+ id: download-charms
+ timeout-minutes: 5
+ uses: actions/download-artifact@v4
+ with:
+ pattern: packed-charm-cache-true-.-charms-${{ matrix.charm }}-*
+ merge-multiple: true
+
  - name: Integration tests
  run: |
  # Requires the model to be called kubeflow due to
  # https://github.com/canonical/kfp-operators/issues/389
  juju add-model kubeflow
- sg snap_microk8s -c "tox -e ${{ matrix.charm }}-integration -- --model kubeflow"
+ sg snap_microk8s -c "tox -vve ${{ matrix.charm }}-integration-using-packed-charms -- --model kubeflow"
 
  - name: Collect charm debug artifacts
  uses: canonical/kubeflow-ci/actions/dump-charm-debug-artifacts@main
@@ -144,11 +174,19 @@ jobs:
  charmcraft-channel: 3.x/stable
  microk8s-addons: "dns hostpath-storage rbac metallb:10.64.140.43-10.64.140.49"
 
+ - name: Download packed charm(s)
+ id: download-charms
+ timeout-minutes: 5
+ uses: actions/download-artifact@v4
+ with:
+ pattern: packed-charm-cache-true-.-*
+ merge-multiple: true
+
  - name: Run test
  run: |
  # Requires the model to be called kubeflow due to kfp-viewer
  juju add-model kubeflow
- sg snap_microk8s -c "tox -e bundle-integration-${{ matrix.sdk }} -- --model kubeflow --bundle=./tests/integration/bundles/kfp_latest_edge.yaml.j2"
+ sg snap_microk8s -c "tox -e bundle-integration-${{ matrix.sdk }}-using-packed-charms -- --model kubeflow --bundle=./tests/integration/bundles/kfp_latest_edge.yaml.j2"
 
  - name: Get all
  run: kubectl get all -A

.github/workflows/publish.yaml

@@ -41,14 +41,12 @@ jobs:
  run: bash .github/workflows/get-charm-paths.sh
 
 
- publish-charm:
- name: Publish Charm
+ define-channel:
+ name: Define destination channel
  runs-on: ubuntu-20.04
  needs: get-charm-paths
- strategy:
- fail-fast: false
- matrix:
- charm-path: ${{ fromJson(needs.get-charm-paths.outputs.charm_paths_list) }}
+ outputs:
+ destination-channel: ${{ steps.parse-inputs.outputs.destination_channel }}
 
  steps:
  - name: Checkout
@@ -72,24 +70,19 @@ jobs:
  echo "setting output of destination_channel=$destination_channel"
  echo "::set-output name=destination_channel::$destination_channel"
 
- # tag_prefix
- # if charm_path = ./ --> tag_prefix = '' (null)
- # if charm_path != ./some-charm (eg: a charm in a ./charms dir) --> tag_prefix = 'some-charm'
- if [ ${{ matrix.charm-path }} == './' ]; then
- tag_prefix=''
- else
- tag_prefix=$(basename ${{ matrix.charm-path }} )
- fi
- echo "setting output of tag_prefix=$tag_prefix"
- echo "::set-output name=tag_prefix::$tag_prefix"
-
- - name: Upload charm to charmhub
- uses: canonical/charming-actions/[email protected]
- with:
- credentials: ${{ secrets.CHARMCRAFT_CREDENTIALS }}
- github-token: ${{ secrets.GITHUB_TOKEN }}
- charm-path: ${{ matrix.charm-path }}
- channel: ${{ steps.parse-inputs.outputs.destination_channel }}
- tag-prefix: ${{ steps.parse-inputs.outputs.tag_prefix }}
- # Pinned to 3.x/stable due to https://github.com/canonical/charmcraft/issues/1845
- charmcraft-channel: 3.x/stable
+ release:
+ name: Release charm
+ uses: canonical/data-platform-workflows/.github/workflows/[email protected]
+ needs: [define-channel, get-charm-paths]
+ strategy:
+ fail-fast: false
+ matrix:
+ charm-path: ${{ fromJson(needs.get-charm-paths.outputs.charm_paths_list) }}
+ with:
+ channel: ${{ needs.define-channel.outputs.destination-channel }}
+ artifact-prefix: packed-charm-cache-true-.-charms-${{ matrix.charm-path }}
+ create-github-release: ${{ github.event_name == 'push' }}
+ secrets:
+ charmhub-token: ${{ secrets.CHARMCRAFT_CREDENTIALS }}
+ permissions:
+ contents: write # Needed to create GitHub release

charms/kfp-api/tests/integration/test_charm.py

@@ -2,6 +2,7 @@
 # See LICENSE file for licensing details.
 
 import logging
+import os
 from pathlib import Path
 
 import pytest
@@ -15,6 +16,7 @@
  get_alert_rules,
 )
 from pytest_operator.plugin import OpsTest
+from utils import get_packed_charms
 
 logger = logging.getLogger(__name__)
 
@@ -39,14 +41,24 @@
 MYSQL_TRUST = True
 
 
+@pytest.fixture
+def use_packed_charms() -> str:
+ """Return environment variable `USE_PACKED_CHARMS`. If it's not found, return `false`."""
+ return os.environ.get("USE_PACKED_CHARMS", "false").replace('"', "")
+
+
 class TestCharm:
  """Integration test charm"""
 
  @pytest.mark.abort_on_fail
- async def test_build_and_deploy(self, ops_test: OpsTest):
+ async def test_build_and_deploy(self, ops_test: OpsTest, use_packed_charms):
  """Deploy kfp-api with required charms and relations."""
- built_charm_path = await ops_test.build_charm("./")
- logger.info(f"Built charm {built_charm_path}")
+ charm_path = "."
+ if use_packed_charms.lower() == "true":
+ built_charm_path = await get_packed_charms(charm_path)
+ else:
+ built_charm_path = await ops_test.build_charm(charm_path)
+ logger.info(f"Built charm {built_charm_path}")
 
  image_path = METADATA["resources"]["oci-image"]["upstream-source"]
  resources = {"oci-image": image_path}

charms/kfp-api/tests/integration/utils.py

@@ -0,0 +1,43 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+"""Utils functions for integration tests. Should be moved to chisme."""
+
+import os
+import pathlib
+import subprocess
+import typing
+
+
+async def get_packed_charms(
+ charm_path: typing.Union[str, os.PathLike], bases_index: int = None
+) -> pathlib.Path:
+ """Simplified version of https://github.com/canonical/data-platform-workflows/blob/06f252ea079edfd055cee236ede28c237467f9b0/python/pytest_plugins/pytest_operator_cache/pytest_operator_cache/_plugin.py#L22.""" # noqa: E501
+ charm_path = pathlib.Path(charm_path)
+ # namespace-node-affinity_ubuntu-20.04-amd64.charm
+ # <metadata-name>_<base>-<architecture>.charm
+ architecture = subprocess.run(
+ ["dpkg", "--print-architecture"],
+ capture_output=True,
+ check=True,
+ encoding="utf-8",
+ ).stdout.strip()
+ assert architecture in ("amd64", "arm64")
+ packed_charms = list(charm_path.glob(f"*-{architecture}.charm"))
+ if len(packed_charms) == 1:
+ # python-libjuju's model.deploy(), juju deploy, and juju bundle files expect local charms
+ # to begin with `./` or `/` to distinguish them from Charmhub charms.
+ # Therefore, we need to return an absolute path—a relative `pathlib.Path` does not start
+ # with `./` when cast to a str.
+ # (python-libjuju model.deploy() expects a str but will cast any input to a str as a
+ # workaround for pytest-operator's non-compliant `build_charm` return type of
+ # `pathlib.Path`.)
+ return packed_charms[0].resolve(strict=True)
+ elif len(packed_charms) > 1:
+ message = f"More than one matching .charm file found at {charm_path=} for {architecture=}: {packed_charms}." # noqa: E501
+ if bases_index is None:
+ message += " Specify `bases_index`"
+ raise ValueError(message)
+ else:
+ raise ValueError(
+ f"Unable to find .charm file for {architecture=} and {bases_index=} at {charm_path=}"
+ )

charms/kfp-api/tox.ini

@@ -7,7 +7,7 @@ max-line-length = 100
 [tox]
 skipsdist = True
 skip_missing_interpreters = True
-envlist = fmt, lint, unit, integration
+envlist = fmt, lint, unit, integration{-using-packed-charms,}
 
 [vars]
 all_path = {[vars]src_path} {[vars]tst_path}
@@ -73,8 +73,10 @@ deps =
  -r requirements-unit.txt
 description = Run unit tests
 
-[testenv:integration]
+[testenv:integration{-using-packed-charms,}]
 commands = pytest -vv --tb native --asyncio-mode=auto {[vars]tst_path}integration --log-cli-level=INFO -s {posargs}
+setenv = 
+ using-packed-charms: USE_PACKED_CHARMS = "true"
 deps =
  -r requirements-integration.txt
 description = Run integration tests

charms/kfp-metadata-writer/tests/integration/test_charm.py

@@ -2,6 +2,7 @@
 # See LICENSE file for licensing details.
 
 import logging
+import os
 from pathlib import Path
 
 import pytest
@@ -12,6 +13,7 @@
  deploy_and_assert_grafana_agent,
 )
 from pytest_operator.plugin import OpsTest
+from utils import get_packed_charms
 
 METADATA = yaml.safe_load(Path("./metadata.yaml").read_text())
 CHARM_ROOT = "."
@@ -25,10 +27,20 @@
 log = logging.getLogger(__name__)
 
 
+@pytest.fixture
+def use_packed_charms() -> str:
+ """Return environment variable `USE_PACKED_CHARMS`. If it's not found, return `false`."""
+ return os.environ.get("USE_PACKED_CHARMS", "false").replace('"', "")
+
+
 @pytest.mark.abort_on_fail
-async def test_build_and_deploy_with_relations(ops_test: OpsTest):
- built_charm_path = await ops_test.build_charm(CHARM_ROOT)
- log.info(f"Built charm {built_charm_path}")
+async def test_build_and_deploy_with_relations(ops_test: OpsTest, use_packed_charms):
+ charm_path = "."
+ if use_packed_charms.lower() == "true":
+ built_charm_path = await get_packed_charms(charm_path)
+ else:
+ built_charm_path = await ops_test.build_charm(charm_path)
+ log.info(f"Built charm {built_charm_path}")
 
  image_path = METADATA["resources"]["oci-image"]["upstream-source"]
  resources = {"oci-image": image_path}

charms/kfp-metadata-writer/tests/integration/utils.py

@@ -0,0 +1,43 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+"""Utils functions for integration tests. Should be moved to chisme."""
+
+import os
+import pathlib
+import subprocess
+import typing
+
+
+async def get_packed_charms(
+ charm_path: typing.Union[str, os.PathLike], bases_index: int = None
+) -> pathlib.Path:
+ """Simplified version of https://github.com/canonical/data-platform-workflows/blob/06f252ea079edfd055cee236ede28c237467f9b0/python/pytest_plugins/pytest_operator_cache/pytest_operator_cache/_plugin.py#L22.""" # noqa: E501
+ charm_path = pathlib.Path(charm_path)
+ # namespace-node-affinity_ubuntu-20.04-amd64.charm
+ # <metadata-name>_<base>-<architecture>.charm
+ architecture = subprocess.run(
+ ["dpkg", "--print-architecture"],
+ capture_output=True,
+ check=True,
+ encoding="utf-8",
+ ).stdout.strip()
+ assert architecture in ("amd64", "arm64")
+ packed_charms = list(charm_path.glob(f"*-{architecture}.charm"))
+ if len(packed_charms) == 1:
+ # python-libjuju's model.deploy(), juju deploy, and juju bundle files expect local charms
+ # to begin with `./` or `/` to distinguish them from Charmhub charms.
+ # Therefore, we need to return an absolute path—a relative `pathlib.Path` does not start
+ # with `./` when cast to a str.
+ # (python-libjuju model.deploy() expects a str but will cast any input to a str as a
+ # workaround for pytest-operator's non-compliant `build_charm` return type of
+ # `pathlib.Path`.)
+ return packed_charms[0].resolve(strict=True)
+ elif len(packed_charms) > 1:
+ message = f"More than one matching .charm file found at {charm_path=} for {architecture=}: {packed_charms}." # noqa: E501
+ if bases_index is None:
+ message += " Specify `bases_index`"
+ raise ValueError(message)
+ else:
+ raise ValueError(
+ f"Unable to find .charm file for {architecture=} and {bases_index=} at {charm_path=}"
+ )

charms/kfp-metadata-writer/tox.ini

@@ -7,7 +7,7 @@ max-line-length = 100
 [tox]
 skipsdist = True
 skip_missing_interpreters = True
-envlist = fmt, lint, unit, integration
+envlist = fmt, lint, unit, integration{-using-packed-charms,}
 
 [vars]
 all_path = {[vars]src_path} {[vars]tst_path}
@@ -73,8 +73,10 @@ deps =
  -r requirements-unit.txt
 description = Run unit tests
 
-[testenv:integration]
+[testenv:integration{-using-packed-charms,}]
 commands = pytest -v --tb native --asyncio-mode=auto {[vars]tst_path}integration --log-cli-level=INFO -s {posargs}
+setenv = 
+ using-packed-charms: USE_PACKED_CHARMS = "true"
 deps =
  -r requirements-integration.txt
 description = Run integration tests