From d02fb2594dea427bb7c6791f28fd6dfe277f939e Mon Sep 17 00:00:00 2001 From: wallyworld Date: Mon, 25 Nov 2024 09:52:36 +1000 Subject: [PATCH] fix: ensure obsolete tests are not run on 4.0 Add support for masking out tests from 3.x that are not applicable to 4.0. --- go.mod | 12 +- go.sum | 27 ++ jobs/ci-run/integration/gen/test-actions.yml | 12 + .../integration/gen/test-authorized_keys.yml | 48 +++ jobs/ci-run/integration/gen/test-backup.yml | 42 +- jobs/ci-run/integration/gen/test-branches.yml | 370 ++++++++++++++++++ .../integration/gen/test-cloud_azure.yml | 3 + .../integration/gen/test-controller.yml | 6 + .../integration/gen/test-controllercharm.yml | 6 + jobs/ci-run/integration/gen/test-coslite.yml | 3 + .../ci-run/integration/gen/test-dashboard.yml | 6 + jobs/ci-run/integration/gen/test-firewall.yml | 3 + jobs/ci-run/integration/gen/test-kubeflow.yml | 3 + jobs/ci-run/integration/gen/test-metrics.yml | 116 ++++++ .../integration/gen/test-secrets_iaas.yml | 15 + .../integration/gen/test-secrets_k8s.yml | 12 + jobs/ci-run/integration/gen/test-sidecar.yml | 9 + .../ci-run/integration/gen/test-smoke_k8s.yml | 3 + jobs/ci-run/integration/gen/test-unit.yml | 171 ++++++++ .../integration/gen/test-upgrade_series.yml | 85 ++++ jobs/ci-run/integration/integrationtests.yml | 4 + tools/gen-wire-tests/README.md | 17 +- tools/gen-wire-tests/juju.config | 9 +- tools/gen-wire-tests/main.go | 335 +++++++++++----- 24 files changed, 1199 insertions(+), 118 deletions(-) create mode 100644 jobs/ci-run/integration/gen/test-branches.yml create mode 100644 jobs/ci-run/integration/gen/test-metrics.yml diff --git a/go.mod b/go.mod index 26b9d84..93f2543 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,18 @@ module github.com/juju/juju-qa-jenkins -go 1.20 +go 1.23 + +toolchain go1.23.3 require ( + github.com/schollz/progressbar/v3 v3.17.1 gopkg.in/yaml.v2 v2.3.0 mvdan.cc/sh/v3 v3.6.0 ) + +require ( + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/rivo/uniseg v0.4.7 // indirect + golang.org/x/sys v0.27.0 // indirect + golang.org/x/term v0.26.0 // indirect +) diff --git a/go.sum b/go.sum index cd5dd59..eb539a1 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,38 @@ +github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= +github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/schollz/progressbar/v3 v3.17.1 h1:bI1MTaoQO+v5kzklBjYNRQLoVpe0zbyRZNK6DFkVC5U= +github.com/schollz/progressbar/v3 v3.17.1/go.mod h1:RzqpnsPQNjUyIgdglUjRLgD7sVnxN1wpmBMV+UiEbL4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU= +golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= mvdan.cc/sh/v3 v3.6.0 h1:gtva4EXJ0dFNvl5bHjcUEvws+KRcDslT8VKheTYkbGU= mvdan.cc/sh/v3 v3.6.0/go.mod h1:U4mhtBLZ32iWhif5/lD+ygy1zrgaQhUu+XFy7C8+TTA= diff --git a/jobs/ci-run/integration/gen/test-actions.yml b/jobs/ci-run/integration/gen/test-actions.yml index a12584c..13eecc9 100644 --- a/jobs/ci-run/integration/gen/test-actions.yml +++ b/jobs/ci-run/integration/gen/test-actions.yml @@ -107,6 +107,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -186,6 +189,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -265,6 +271,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -340,6 +349,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" diff --git a/jobs/ci-run/integration/gen/test-authorized_keys.yml b/jobs/ci-run/integration/gen/test-authorized_keys.yml index d13c234..a2d7cb3 100644 --- a/jobs/ci-run/integration/gen/test-authorized_keys.yml +++ b/jobs/ci-run/integration/gen/test-authorized_keys.yml @@ -131,6 +131,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -210,6 +213,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -289,6 +295,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -364,6 +373,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -443,6 +455,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -522,6 +537,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -601,6 +619,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -676,6 +697,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -755,6 +779,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -834,6 +861,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -913,6 +943,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -988,6 +1021,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -1067,6 +1103,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -1146,6 +1185,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -1225,6 +1267,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -1300,6 +1345,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" diff --git a/jobs/ci-run/integration/gen/test-backup.yml b/jobs/ci-run/integration/gen/test-backup.yml index 4b40d37..7ea31bd 100644 --- a/jobs/ci-run/integration/gen/test-backup.yml +++ b/jobs/ci-run/integration/gen/test-backup.yml @@ -102,11 +102,22 @@ - select-oci-registry - wait-for-cloud-init - prepare-integration-test - - run-integration-test: - test_name: 'backup' - setup_steps: '' - task_name: '' - skip_tasks: '' + - conditional-step: + # Do not run on regexp version match. + # Accounts for tests which do not exist + # in later Juju versions. + condition-kind: not + condition-operand: + condition-kind: regex-match + regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" + label: "${{JUJU_VERSION}}" + on-evaluation-failure: "dont-run" + steps: + - run-integration-test: + test_name: 'backup' + setup_steps: '' + task_name: '' + skip_tasks: '' publishers: - integration-artifacts @@ -171,10 +182,21 @@ - select-oci-registry - wait-for-cloud-init - prepare-integration-test - - run-integration-test: - test_name: 'backup' - setup_steps: '' - task_name: '' - skip_tasks: '' + - conditional-step: + # Do not run on regexp version match. + # Accounts for tests which do not exist + # in later Juju versions. + condition-kind: not + condition-operand: + condition-kind: regex-match + regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" + label: "${{JUJU_VERSION}}" + on-evaluation-failure: "dont-run" + steps: + - run-integration-test: + test_name: 'backup' + setup_steps: '' + task_name: '' + skip_tasks: '' publishers: - integration-artifacts diff --git a/jobs/ci-run/integration/gen/test-branches.yml b/jobs/ci-run/integration/gen/test-branches.yml new file mode 100644 index 0000000..477a31d --- /dev/null +++ b/jobs/ci-run/integration/gen/test-branches.yml @@ -0,0 +1,370 @@ + + +# Code generated by gen-wire-tests. DO NOT EDIT. +- job: + name: 'test-branches-multijob' + project-type: 'multijob' + description: |- + Test branches Suite + condition: SUCCESSFUL + node: noop-parent-jobs + concurrent: true + wrappers: + - ansicolor + - workspace-cleanup + - timestamps + parameters: + - string: + default: '' + description: 'Enable sub job to be run individually.' + name: SHORT_GIT_COMMIT + - string: + default: '' + description: 'Build arch used to download the build tar.gz.' + name: BUILD_ARCH + - string: + default: '' + description: 'Ubuntu series to use when bootstrapping Juju' + name: BOOTSTRAP_SERIES + builders: + - get-build-details + - set-test-description + - multijob: + name: 'IntegrationTests-branches' + projects: + - name: 'test-branches-test-active-branch-output-aws' + current-parameters: true + - name: 'test-branches-test-active-branch-output-lxd' + current-parameters: true + - name: 'test-branches-test-branch-aws' + current-parameters: true + - name: 'test-branches-test-branch-lxd' + current-parameters: true + +- job: + name: test-branches-test-active-branch-output-aws + node: ephemeral-noble-small-amd64 + concurrent: true + description: |- + Test test_active_branch_output in branches suite on aws + parameters: + - validating-string: + name: SHORT_GIT_COMMIT + description: 'Enable sub job to be run individually.' + regex: ^\S{{7}}$ + msg: Enter a valid 7 char git sha + - choice: + default: 'amd64' + description: 'Build arch used to download the build tar.gz.' + name: BUILD_ARCH + choices: + - amd64 + - arm64 + - s390x + - ppc64el + - choice: + default: '' + description: 'Arch used to boostrap controller.' + name: BOOTSTRAP_ARCH + choices: + - amd64 + - arm64 + - s390x + - ppc64el + - choice: + default: '' + description: 'Arch used for hosted models.' + name: MODEL_ARCH + choices: + - amd64 + - arm64 + - s390x + - ppc64el + - string: + default: 'aws' + description: 'Cloud to use when bootstrapping Juju' + name: BOOTSTRAP_CLOUD + - string: + default: 'ec2' + description: 'Provider to use when bootstrapping Juju' + name: BOOTSTRAP_PROVIDER + - string: + default: 'us-east-1' + description: 'Cloud Region to use when bootstrapping Juju' + name: BOOTSTRAP_REGION + - string: + default: '' + description: 'Ubuntu series to use when bootstrapping Juju' + name: BOOTSTRAP_SERIES + wrappers: + - default-integration-test-wrapper + - timeout: + timeout: 30 + fail: true + type: absolute + builders: + - select-oci-registry + - wait-for-cloud-init + - prepare-integration-test + - conditional-step: + # Do not run on regexp version match. + # Accounts for tests which do not exist + # in later Juju versions. + condition-kind: not + condition-operand: + condition-kind: regex-match + regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" + label: "${{JUJU_VERSION}}" + on-evaluation-failure: "dont-run" + steps: + - run-integration-test: + test_name: 'branches' + setup_steps: '' + task_name: 'test_active_branch_output' + skip_tasks: 'test_branch' + publishers: + - integration-artifacts + +- job: + name: test-branches-test-active-branch-output-lxd + node: ephemeral-noble-8c-32g-amd64 + concurrent: true + description: |- + Test test_active_branch_output in branches suite on lxd + parameters: + - validating-string: + name: SHORT_GIT_COMMIT + description: 'Enable sub job to be run individually.' + regex: ^\S{{7}}$ + msg: Enter a valid 7 char git sha + - choice: + default: 'amd64' + description: 'Build arch used to download the build tar.gz.' + name: BUILD_ARCH + choices: + - amd64 + - arm64 + - s390x + - ppc64el + - choice: + default: '' + description: 'Arch used to boostrap controller.' + name: BOOTSTRAP_ARCH + choices: + - amd64 + - arm64 + - s390x + - ppc64el + - choice: + default: '' + description: 'Arch used for hosted models.' + name: MODEL_ARCH + choices: + - amd64 + - arm64 + - s390x + - ppc64el + - string: + default: 'localhost' + description: 'Cloud to use when bootstrapping Juju' + name: BOOTSTRAP_CLOUD + - string: + default: 'lxd' + description: 'Provider to use when bootstrapping Juju' + name: BOOTSTRAP_PROVIDER + - string: + default: '' + description: 'Ubuntu series to use when bootstrapping Juju' + name: BOOTSTRAP_SERIES + wrappers: + - default-integration-test-wrapper + - timeout: + timeout: 30 + fail: true + type: absolute + builders: + - select-oci-registry + - wait-for-cloud-init + - prepare-integration-test + - conditional-step: + # Do not run on regexp version match. + # Accounts for tests which do not exist + # in later Juju versions. + condition-kind: not + condition-operand: + condition-kind: regex-match + regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" + label: "${{JUJU_VERSION}}" + on-evaluation-failure: "dont-run" + steps: + - run-integration-test: + test_name: 'branches' + setup_steps: '' + task_name: 'test_active_branch_output' + skip_tasks: 'test_branch' + publishers: + - integration-artifacts + +- job: + name: test-branches-test-branch-aws + node: ephemeral-noble-small-amd64 + concurrent: true + description: |- + Test test_branch in branches suite on aws + parameters: + - validating-string: + name: SHORT_GIT_COMMIT + description: 'Enable sub job to be run individually.' + regex: ^\S{{7}}$ + msg: Enter a valid 7 char git sha + - choice: + default: 'amd64' + description: 'Build arch used to download the build tar.gz.' + name: BUILD_ARCH + choices: + - amd64 + - arm64 + - s390x + - ppc64el + - choice: + default: '' + description: 'Arch used to boostrap controller.' + name: BOOTSTRAP_ARCH + choices: + - amd64 + - arm64 + - s390x + - ppc64el + - choice: + default: '' + description: 'Arch used for hosted models.' + name: MODEL_ARCH + choices: + - amd64 + - arm64 + - s390x + - ppc64el + - string: + default: 'aws' + description: 'Cloud to use when bootstrapping Juju' + name: BOOTSTRAP_CLOUD + - string: + default: 'ec2' + description: 'Provider to use when bootstrapping Juju' + name: BOOTSTRAP_PROVIDER + - string: + default: 'us-east-1' + description: 'Cloud Region to use when bootstrapping Juju' + name: BOOTSTRAP_REGION + - string: + default: '' + description: 'Ubuntu series to use when bootstrapping Juju' + name: BOOTSTRAP_SERIES + wrappers: + - default-integration-test-wrapper + - timeout: + timeout: 30 + fail: true + type: absolute + builders: + - select-oci-registry + - wait-for-cloud-init + - prepare-integration-test + - conditional-step: + # Do not run on regexp version match. + # Accounts for tests which do not exist + # in later Juju versions. + condition-kind: not + condition-operand: + condition-kind: regex-match + regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" + label: "${{JUJU_VERSION}}" + on-evaluation-failure: "dont-run" + steps: + - run-integration-test: + test_name: 'branches' + setup_steps: '' + task_name: 'test_branch' + skip_tasks: 'test_active_branch_output' + publishers: + - integration-artifacts + +- job: + name: test-branches-test-branch-lxd + node: ephemeral-noble-8c-32g-amd64 + concurrent: true + description: |- + Test test_branch in branches suite on lxd + parameters: + - validating-string: + name: SHORT_GIT_COMMIT + description: 'Enable sub job to be run individually.' + regex: ^\S{{7}}$ + msg: Enter a valid 7 char git sha + - choice: + default: 'amd64' + description: 'Build arch used to download the build tar.gz.' + name: BUILD_ARCH + choices: + - amd64 + - arm64 + - s390x + - ppc64el + - choice: + default: '' + description: 'Arch used to boostrap controller.' + name: BOOTSTRAP_ARCH + choices: + - amd64 + - arm64 + - s390x + - ppc64el + - choice: + default: '' + description: 'Arch used for hosted models.' + name: MODEL_ARCH + choices: + - amd64 + - arm64 + - s390x + - ppc64el + - string: + default: 'localhost' + description: 'Cloud to use when bootstrapping Juju' + name: BOOTSTRAP_CLOUD + - string: + default: 'lxd' + description: 'Provider to use when bootstrapping Juju' + name: BOOTSTRAP_PROVIDER + - string: + default: '' + description: 'Ubuntu series to use when bootstrapping Juju' + name: BOOTSTRAP_SERIES + wrappers: + - default-integration-test-wrapper + - timeout: + timeout: 30 + fail: true + type: absolute + builders: + - select-oci-registry + - wait-for-cloud-init + - prepare-integration-test + - conditional-step: + # Do not run on regexp version match. + # Accounts for tests which do not exist + # in later Juju versions. + condition-kind: not + condition-operand: + condition-kind: regex-match + regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" + label: "${{JUJU_VERSION}}" + on-evaluation-failure: "dont-run" + steps: + - run-integration-test: + test_name: 'branches' + setup_steps: '' + task_name: 'test_branch' + skip_tasks: 'test_active_branch_output' + publishers: + - integration-artifacts diff --git a/jobs/ci-run/integration/gen/test-cloud_azure.yml b/jobs/ci-run/integration/gen/test-cloud_azure.yml index 2b95b31..c14a4b0 100644 --- a/jobs/ci-run/integration/gen/test-cloud_azure.yml +++ b/jobs/ci-run/integration/gen/test-cloud_azure.yml @@ -101,6 +101,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[4-9].*|^3\\.([6-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" diff --git a/jobs/ci-run/integration/gen/test-controller.yml b/jobs/ci-run/integration/gen/test-controller.yml index ab453a5..28dc5fd 100644 --- a/jobs/ci-run/integration/gen/test-controller.yml +++ b/jobs/ci-run/integration/gen/test-controller.yml @@ -541,6 +541,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[4-9].*|^3\\.([2-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -616,6 +619,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[4-9].*|^3\\.([2-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" diff --git a/jobs/ci-run/integration/gen/test-controllercharm.yml b/jobs/ci-run/integration/gen/test-controllercharm.yml index 5062be6..a6875bd 100644 --- a/jobs/ci-run/integration/gen/test-controllercharm.yml +++ b/jobs/ci-run/integration/gen/test-controllercharm.yml @@ -99,6 +99,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[4-9].*|^3\\.([3-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -174,6 +177,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[4-9].*|^3\\.([3-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" diff --git a/jobs/ci-run/integration/gen/test-coslite.yml b/jobs/ci-run/integration/gen/test-coslite.yml index 869de5f..62b6d57 100644 --- a/jobs/ci-run/integration/gen/test-coslite.yml +++ b/jobs/ci-run/integration/gen/test-coslite.yml @@ -97,6 +97,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[4-9].*|^3\\.([1-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" diff --git a/jobs/ci-run/integration/gen/test-dashboard.yml b/jobs/ci-run/integration/gen/test-dashboard.yml index 84d5d36..3feb06c 100644 --- a/jobs/ci-run/integration/gen/test-dashboard.yml +++ b/jobs/ci-run/integration/gen/test-dashboard.yml @@ -103,6 +103,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[4-9].*|^3\\.([1-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -178,6 +181,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[4-9].*|^3\\.([1-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" diff --git a/jobs/ci-run/integration/gen/test-firewall.yml b/jobs/ci-run/integration/gen/test-firewall.yml index ecabcfc..96074a4 100644 --- a/jobs/ci-run/integration/gen/test-firewall.yml +++ b/jobs/ci-run/integration/gen/test-firewall.yml @@ -251,6 +251,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[4-9].*|^3\\.([2-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" diff --git a/jobs/ci-run/integration/gen/test-kubeflow.yml b/jobs/ci-run/integration/gen/test-kubeflow.yml index 797b227..d88e571 100644 --- a/jobs/ci-run/integration/gen/test-kubeflow.yml +++ b/jobs/ci-run/integration/gen/test-kubeflow.yml @@ -97,6 +97,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[4-9].*|^3\\.([5-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" diff --git a/jobs/ci-run/integration/gen/test-metrics.yml b/jobs/ci-run/integration/gen/test-metrics.yml new file mode 100644 index 0000000..cbd1aeb --- /dev/null +++ b/jobs/ci-run/integration/gen/test-metrics.yml @@ -0,0 +1,116 @@ + + +# Code generated by gen-wire-tests. DO NOT EDIT. +- job: + name: 'test-metrics-multijob' + project-type: 'multijob' + description: |- + Test metrics Suite + condition: SUCCESSFUL + node: noop-parent-jobs + concurrent: true + wrappers: + - ansicolor + - workspace-cleanup + - timestamps + parameters: + - string: + default: '' + description: 'Enable sub job to be run individually.' + name: SHORT_GIT_COMMIT + - string: + default: '' + description: 'Build arch used to download the build tar.gz.' + name: BUILD_ARCH + - string: + default: '' + description: 'Ubuntu series to use when bootstrapping Juju' + name: BOOTSTRAP_SERIES + builders: + - get-build-details + - set-test-description + - multijob: + name: 'IntegrationTests-metrics' + projects: + - name: 'test-metrics-test-smoke-test-lxd' + current-parameters: true + +- job: + name: test-metrics-test-smoke-test-lxd + node: ephemeral-noble-8c-32g-amd64 + concurrent: true + description: |- + Test metrics suite on lxd + parameters: + - validating-string: + name: SHORT_GIT_COMMIT + description: 'Enable sub job to be run individually.' + regex: ^\S{{7}}$ + msg: Enter a valid 7 char git sha + - choice: + default: 'amd64' + description: 'Build arch used to download the build tar.gz.' + name: BUILD_ARCH + choices: + - amd64 + - arm64 + - s390x + - ppc64el + - choice: + default: '' + description: 'Arch used to boostrap controller.' + name: BOOTSTRAP_ARCH + choices: + - amd64 + - arm64 + - s390x + - ppc64el + - choice: + default: '' + description: 'Arch used for hosted models.' + name: MODEL_ARCH + choices: + - amd64 + - arm64 + - s390x + - ppc64el + - string: + default: 'localhost' + description: 'Cloud to use when bootstrapping Juju' + name: BOOTSTRAP_CLOUD + - string: + default: 'lxd' + description: 'Provider to use when bootstrapping Juju' + name: BOOTSTRAP_PROVIDER + - string: + default: '' + description: 'Ubuntu series to use when bootstrapping Juju' + name: BOOTSTRAP_SERIES + wrappers: + - default-integration-test-wrapper + - timeout: + timeout: 30 + fail: true + type: absolute + builders: + - select-oci-registry + - wait-for-cloud-init + - prepare-integration-test + - conditional-step: + # Do not run on regexp version match. + # Accounts for tests which do not exist + # in later Juju versions. + condition-kind: not + condition-operand: + condition-kind: regex-match + regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" + label: "${{JUJU_VERSION}}" + on-evaluation-failure: "dont-run" + steps: + - run-integration-test: + test_name: 'metrics' + setup_steps: '' + task_name: '' + skip_tasks: '' + publishers: + - integration-artifacts diff --git a/jobs/ci-run/integration/gen/test-secrets_iaas.yml b/jobs/ci-run/integration/gen/test-secrets_iaas.yml index debd9f6..3665c34 100644 --- a/jobs/ci-run/integration/gen/test-secrets_iaas.yml +++ b/jobs/ci-run/integration/gen/test-secrets_iaas.yml @@ -105,6 +105,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[4-9].*|^3\\.([2-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -180,6 +183,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[4-9].*|^3\\.([1-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -255,6 +261,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[4-9].*|^3\\.([1-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -330,6 +339,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[4-9].*|^3\\.([1-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -405,6 +417,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[4-9].*|^3\\.([3-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" diff --git a/jobs/ci-run/integration/gen/test-secrets_k8s.yml b/jobs/ci-run/integration/gen/test-secrets_k8s.yml index 0e7eff6..f439735 100644 --- a/jobs/ci-run/integration/gen/test-secrets_k8s.yml +++ b/jobs/ci-run/integration/gen/test-secrets_k8s.yml @@ -103,6 +103,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[4-9].*|^3\\.([2-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -178,6 +181,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[4-9].*|^3\\.([1-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -253,6 +259,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[4-9].*|^3\\.([3-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -328,6 +337,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[4-9].*|^3\\.([3-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" diff --git a/jobs/ci-run/integration/gen/test-sidecar.yml b/jobs/ci-run/integration/gen/test-sidecar.yml index c4ca9b7..a891bc5 100644 --- a/jobs/ci-run/integration/gen/test-sidecar.yml +++ b/jobs/ci-run/integration/gen/test-sidecar.yml @@ -243,6 +243,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[4-9].*|^3\\.([6-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -318,6 +321,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[4-9].*|^3\\.([4-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" @@ -393,6 +399,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[4-9].*|^3\\.([6-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" diff --git a/jobs/ci-run/integration/gen/test-smoke_k8s.yml b/jobs/ci-run/integration/gen/test-smoke_k8s.yml index 562e508..8e6e0e4 100644 --- a/jobs/ci-run/integration/gen/test-smoke_k8s.yml +++ b/jobs/ci-run/integration/gen/test-smoke_k8s.yml @@ -97,6 +97,9 @@ - wait-for-cloud-init - prepare-integration-test - conditional-step: + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" label: "${{JUJU_VERSION}}" diff --git a/jobs/ci-run/integration/gen/test-unit.yml b/jobs/ci-run/integration/gen/test-unit.yml index 6425184..d06b53b 100644 --- a/jobs/ci-run/integration/gen/test-unit.yml +++ b/jobs/ci-run/integration/gen/test-unit.yml @@ -29,3 +29,174 @@ builders: - get-build-details - set-test-description + - multijob: + name: 'IntegrationTests-unit' + projects: + - name: 'test-unit-test-unit-series-aws' + current-parameters: true + - name: 'test-unit-test-unit-series-lxd' + current-parameters: true + +- job: + name: test-unit-test-unit-series-aws + node: ephemeral-noble-small-amd64 + concurrent: true + description: |- + Test unit suite on aws + parameters: + - validating-string: + name: SHORT_GIT_COMMIT + description: 'Enable sub job to be run individually.' + regex: ^\S{{7}}$ + msg: Enter a valid 7 char git sha + - choice: + default: 'amd64' + description: 'Build arch used to download the build tar.gz.' + name: BUILD_ARCH + choices: + - amd64 + - arm64 + - s390x + - ppc64el + - choice: + default: '' + description: 'Arch used to boostrap controller.' + name: BOOTSTRAP_ARCH + choices: + - amd64 + - arm64 + - s390x + - ppc64el + - choice: + default: '' + description: 'Arch used for hosted models.' + name: MODEL_ARCH + choices: + - amd64 + - arm64 + - s390x + - ppc64el + - string: + default: 'aws' + description: 'Cloud to use when bootstrapping Juju' + name: BOOTSTRAP_CLOUD + - string: + default: 'ec2' + description: 'Provider to use when bootstrapping Juju' + name: BOOTSTRAP_PROVIDER + - string: + default: 'us-east-1' + description: 'Cloud Region to use when bootstrapping Juju' + name: BOOTSTRAP_REGION + - string: + default: '' + description: 'Ubuntu series to use when bootstrapping Juju' + name: BOOTSTRAP_SERIES + wrappers: + - default-integration-test-wrapper + - timeout: + timeout: 30 + fail: true + type: absolute + builders: + - select-oci-registry + - wait-for-cloud-init + - prepare-integration-test + - conditional-step: + # Do not run on regexp version match. + # Accounts for tests which do not exist + # in later Juju versions. + condition-kind: not + condition-operand: + condition-kind: regex-match + regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" + label: "${{JUJU_VERSION}}" + on-evaluation-failure: "dont-run" + steps: + - run-integration-test: + test_name: 'unit' + setup_steps: '' + task_name: '' + skip_tasks: '' + publishers: + - integration-artifacts + +- job: + name: test-unit-test-unit-series-lxd + node: ephemeral-noble-8c-32g-amd64 + concurrent: true + description: |- + Test unit suite on lxd + parameters: + - validating-string: + name: SHORT_GIT_COMMIT + description: 'Enable sub job to be run individually.' + regex: ^\S{{7}}$ + msg: Enter a valid 7 char git sha + - choice: + default: 'amd64' + description: 'Build arch used to download the build tar.gz.' + name: BUILD_ARCH + choices: + - amd64 + - arm64 + - s390x + - ppc64el + - choice: + default: '' + description: 'Arch used to boostrap controller.' + name: BOOTSTRAP_ARCH + choices: + - amd64 + - arm64 + - s390x + - ppc64el + - choice: + default: '' + description: 'Arch used for hosted models.' + name: MODEL_ARCH + choices: + - amd64 + - arm64 + - s390x + - ppc64el + - string: + default: 'localhost' + description: 'Cloud to use when bootstrapping Juju' + name: BOOTSTRAP_CLOUD + - string: + default: 'lxd' + description: 'Provider to use when bootstrapping Juju' + name: BOOTSTRAP_PROVIDER + - string: + default: '' + description: 'Ubuntu series to use when bootstrapping Juju' + name: BOOTSTRAP_SERIES + wrappers: + - default-integration-test-wrapper + - timeout: + timeout: 30 + fail: true + type: absolute + builders: + - select-oci-registry + - wait-for-cloud-init + - prepare-integration-test + - conditional-step: + # Do not run on regexp version match. + # Accounts for tests which do not exist + # in later Juju versions. + condition-kind: not + condition-operand: + condition-kind: regex-match + regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" + label: "${{JUJU_VERSION}}" + on-evaluation-failure: "dont-run" + steps: + - run-integration-test: + test_name: 'unit' + setup_steps: '' + task_name: '' + skip_tasks: '' + publishers: + - integration-artifacts diff --git a/jobs/ci-run/integration/gen/test-upgrade_series.yml b/jobs/ci-run/integration/gen/test-upgrade_series.yml index 10ad77e..a1a59d0 100644 --- a/jobs/ci-run/integration/gen/test-upgrade_series.yml +++ b/jobs/ci-run/integration/gen/test-upgrade_series.yml @@ -29,3 +29,88 @@ builders: - get-build-details - set-test-description + - multijob: + name: 'IntegrationTests-upgrade_series' + projects: + - name: 'test-upgrade_series-test-upgrade-series-relation-lxd' + current-parameters: true + +- job: + name: test-upgrade_series-test-upgrade-series-relation-lxd + node: ephemeral-noble-8c-32g-amd64 + concurrent: true + description: |- + Test upgrade_series suite on lxd + parameters: + - validating-string: + name: SHORT_GIT_COMMIT + description: 'Enable sub job to be run individually.' + regex: ^\S{{7}}$ + msg: Enter a valid 7 char git sha + - choice: + default: 'amd64' + description: 'Build arch used to download the build tar.gz.' + name: BUILD_ARCH + choices: + - amd64 + - arm64 + - s390x + - ppc64el + - choice: + default: '' + description: 'Arch used to boostrap controller.' + name: BOOTSTRAP_ARCH + choices: + - amd64 + - arm64 + - s390x + - ppc64el + - choice: + default: '' + description: 'Arch used for hosted models.' + name: MODEL_ARCH + choices: + - amd64 + - arm64 + - s390x + - ppc64el + - string: + default: 'localhost' + description: 'Cloud to use when bootstrapping Juju' + name: BOOTSTRAP_CLOUD + - string: + default: 'lxd' + description: 'Provider to use when bootstrapping Juju' + name: BOOTSTRAP_PROVIDER + - string: + default: '' + description: 'Ubuntu series to use when bootstrapping Juju' + name: BOOTSTRAP_SERIES + wrappers: + - default-integration-test-wrapper + - timeout: + timeout: 30 + fail: true + type: absolute + builders: + - select-oci-registry + - wait-for-cloud-init + - prepare-integration-test + - conditional-step: + # Do not run on regexp version match. + # Accounts for tests which do not exist + # in later Juju versions. + condition-kind: not + condition-operand: + condition-kind: regex-match + regex: "^[5-9].*|^4\\.([0-9]|\\d{{2,}})(\\.|-).*" + label: "${{JUJU_VERSION}}" + on-evaluation-failure: "dont-run" + steps: + - run-integration-test: + test_name: 'upgrade_series' + setup_steps: '' + task_name: '' + skip_tasks: '' + publishers: + - integration-artifacts diff --git a/jobs/ci-run/integration/integrationtests.yml b/jobs/ci-run/integration/integrationtests.yml index cf28611..4ea96a8 100644 --- a/jobs/ci-run/integration/integrationtests.yml +++ b/jobs/ci-run/integration/integrationtests.yml @@ -138,6 +138,10 @@ current-parameters: true predefined-parameters: |- BUILD_ARCH=amd64 + - name: 'test-metrics-multijob' + current-parameters: true + predefined-parameters: |- + BUILD_ARCH=amd64 - name: 'test-model-multijob' current-parameters: true predefined-parameters: |- diff --git a/tools/gen-wire-tests/README.md b/tools/gen-wire-tests/README.md index 68b74f7..0cc5ec7 100644 --- a/tools/gen-wire-tests/README.md +++ b/tools/gen-wire-tests/README.md @@ -1,12 +1,15 @@ # gen-wire-tests The `gen-wire-tests` Go tool is used to generate the Jenkins integration tests -in `jobs/ci-run/integration/gen`. It requires two arguments: -- The path to the Juju `tests/suites` directory +in `jobs/ci-run/integration/gen`. It requires one argument: - The directory where the resulting yaml files will be written You also need to provide a configuration file (usually `juju.config`) via stdin. +The tool reads the test suites from the Juju repo on github. To avoid API rate +limiting for unauthenticated requests, create an access token and set the env +var GH_TOKEN before running the tool. This is optional but recommended. + We suggest running it as follows: ```bash @@ -18,8 +21,8 @@ To run by hand, or the hard way: ```bash cd tools/gen-wire-tests -cat juju.config | go run main.go /tests/suites \ - ../../jobs/ci-run/integration/gen +cat juju.config | go run main.go \ + ../../jobs/ci-run/integration/gen ``` or @@ -27,11 +30,9 @@ or ```bash cd tools/gen-wire-tests go build main.go -cat juju.config | ./main /tests/suites \ - ../../jobs/ci-run/integration/gen +cat juju.config | ./main \ + ../../jobs/ci-run/integration/gen ``` -where `` is the path to the Juju source tree on your local machine. - If you are adding new test suites, you will also need to add a new job to `jobs/ci-run/integration/integrationtests.yml`. diff --git a/tools/gen-wire-tests/juju.config b/tools/gen-wire-tests/juju.config index c324307..d825b0d 100644 --- a/tools/gen-wire-tests/juju.config +++ b/tools/gen-wire-tests/juju.config @@ -1,4 +1,7 @@ folders: + removed: + deploy-test_deploy_os: + 4.0 introduced: cloud_azure-test_managed_identity: 3.6 @@ -36,7 +39,7 @@ folders: 3.6 test_rootless: 3.6 - authorized_keys-*: + authorized_keys: 4.0 test_actions_params: 4.0 @@ -100,6 +103,7 @@ folders: - secrets_k8s - smoke_k8s - kubeflow + - metrics skip-google: - agents - appdata @@ -141,6 +145,7 @@ folders: - model_multi - kubeflow - model_multi + - metrics skip-azure: - agents - appdata @@ -181,6 +186,7 @@ folders: - secrets_k8s - kubeflow - model_multi + - metrics skip-microk8s: - actions - agents @@ -218,6 +224,7 @@ folders: - user - secrets_iaas - model_multi + - metrics skip-subtasks: - test_pebble_change_updated - test_upgrade_simplestream_previous diff --git a/tools/gen-wire-tests/main.go b/tools/gen-wire-tests/main.go index 98b7705..0861fd9 100644 --- a/tools/gen-wire-tests/main.go +++ b/tools/gen-wire-tests/main.go @@ -2,18 +2,18 @@ package main import ( "bufio" + "encoding/json" "fmt" "io" - "io/ioutil" "log" + "net/http" "os" - "os/exec" "path/filepath" - "regexp" "sort" "strings" "text/template" + "github.com/schollz/progressbar/v3" "gopkg.in/yaml.v2" "mvdan.cc/sh/v3/syntax" ) @@ -33,6 +33,7 @@ type Config struct { CrossCloud []string `yaml:"cross-cloud"` Timeout map[string]map[string]int `yaml:"timeout"` Introduced map[string]string `yaml:"introduced"` + Removed map[string]string `yaml:"removed"` } } @@ -77,28 +78,28 @@ var minVersionRegex = map[string]string{ "4.0": "^[5-9].*|^4\\\\.([0-9]|\\\\d{{2,}})(\\\\.|-).*", } +// Override tests if testing on personal branches. +const ( + v4BranchName = "main" + v3BranchName = "3.6" + repoOrg = "juju" +) + // Gen-wire-tests will generate the integration test files for the juju // integration tests. This will help prevent wire up mistakes or any missing // test suite tests. // // It expects two arguments to be passed in: -// - inputDir: the juju test suite location // - outputDir: the location of the new jenkins config files // // Additionally it expects a config file passed in via stdin, this allows the // configuration of the gen-wire-tests. In reality it allows the skipping of // folders that are custom and don't follow the generic setup. func main() { - if len(os.Args) < 3 { + if len(os.Args) < 1 { log.Fatal("expected directory argument only.") } - inputDir := os.Args[1] - outputDir := os.Args[2] - trackingBranch := os.Args[3] - - if !isTrackingBranch(inputDir, trackingBranch) { - log.Fatalf("not on the tracking branch %q, or not update to date with HEAD", trackingBranch) - } + outputDir := os.Args[1] if outDir, err := os.Open(outputDir); os.IsNotExist(err) { if err := os.MkdirAll(outputDir, os.ModePerm); err != nil { @@ -133,15 +134,125 @@ func main() { log.Fatal("config parse error: ", err) } - dirs, err := ioutil.ReadDir(inputDir) + v4fourSuiteInfo := fetchSuitesFromRepo(v4BranchName) + + // Create progress bar ASAP with known info. + pb := progressbar.NewOptions(2*len(v4fourSuiteInfo), + progressbar.OptionUseANSICodes(false), + progressbar.OptionSetTheme(progressbar.ThemeASCII), + progressbar.OptionShowElapsedTimeOnFinish(), + progressbar.OptionSetWidth(50), + progressbar.OptionSetDescription("Reading test suites..."), + progressbar.OptionSetTheme(progressbar.Theme{ + Saucer: "=", + SaucerHead: ">", + SaucerPadding: " ", + BarStart: "[", + BarEnd: "]", + })) + + v3SuiteInfo := fetchSuitesFromRepo(v3BranchName) + + v4Tests := fetchTestsFromRepo(pb, config, v4fourSuiteInfo) + v3Tests := fetchTestsFromRepo(pb, config, v3SuiteInfo) + + // Finish the progress bar. + _ = pb.Exit() + fmt.Println() + + funcMap := map[string]interface{}{ + "ensureHyphen": func(s string) string { + return strings.ReplaceAll(s, "_", "-") + }, + "contains": func(arr []string, s string) bool { + for _, v := range arr { + if s == v { + return true + } + } + return false + }, + } + t := template.Must(template.New("integration").Funcs(funcMap).Parse(Template)) + + allTests := make(map[string]Task) + for suiteName, task := range v3Tests { + allTests[suiteName] = task + if _, ok := v4Tests[suiteName]; !ok { + config.Folders.Removed[suiteName] = "4.0" + } + } + for suiteName, task := range v4Tests { + allTests[suiteName] = task + } + + for suiteName, task := range allTests { + writeJobDefinitions(t, config, outputDir, task, suiteName) + } +} + +type ghObject struct { + Name string `json:"name"` + Path string `json:"path"` + Type string `json:"type"` + URL string `json:"url"` + DownloadURL string `json:"download_url"` +} + +func fetchGithubObjects(url string) ([]ghObject, error) { + client := http.DefaultClient + req, _ := http.NewRequest("GET", url, nil) + + req.Header.Add("Accept", "application/vnd.github+json") + token := os.Getenv("GH_TOKEN") + if token != "" { + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + } + resp, err := client.Do(req) if err != nil { - log.Fatalf("unable to get listing dir %q with error %v", inputDir, err) + log.Fatalf("unable to fetch URL %q: %v", url, err) } + if resp.StatusCode != 200 { + log.Fatalf("unable to fetch URL %q; status code %d", url, resp.StatusCode) + } + defer resp.Body.Close() + + data, err := io.ReadAll(resp.Body) + if err != nil { + log.Fatalf("unable to get URL %q content: %v", url, err) + } + var ghObjects []ghObject + err = json.Unmarshal(data, &ghObjects) + if err != nil { + log.Fatalf("unable to unmarshal URL %q content: %v", url, err) + } + return ghObjects, nil +} +func fetchSuitesFromRepo(branch string) []ghObject { + url := fmt.Sprintf("https://api.github.com/repos/%s/juju/contents/tests/suites?ref=%s", repoOrg, branch) + ghObjects, err := fetchGithubObjects(url) + if err != nil { + log.Fatal("unable to fetch GitHub objects: ", err) + } + + var ghDirectories []ghObject + for _, o := range ghObjects { + if o.Type != "dir" { + continue + } + ghDirectories = append(ghDirectories, o) + } + return ghDirectories +} + +func fetchTestsFromRepo(pb *progressbar.ProgressBar, config Config, suiteInfo []ghObject) map[string]Task { var suiteNames []string testSuites := make(map[string]Task) - for _, dir := range dirs { - suiteName := dir.Name() + for _, dir := range suiteInfo { + suiteName := dir.Name + pb.Add(1) + if contains(config.Folders.Skip, suiteName) { continue } @@ -150,7 +261,7 @@ func main() { excluded := []string{} if !contains(config.Folders.PreventSplit, suiteName) { taskNames = []string{} - subTaskNames := parseTaskNames(inputDir, dir) + subTaskNames := parseTaskNames(dir) for _, subTask := range subTaskNames { if !contains(config.Folders.SkipSubTasks, subTask) { taskNames = append(taskNames, subTask) @@ -186,49 +297,7 @@ func main() { Timeout: config.Folders.Timeout[suiteName], } } - - funcMap := map[string]interface{}{ - "ensureHyphen": func(s string) string { - return strings.ReplaceAll(s, "_", "-") - }, - "contains": func(arr []string, s string) bool { - for _, v := range arr { - if s == v { - return true - } - } - return false - }, - } - t := template.Must(template.New("integration").Funcs(funcMap).Parse(Template)) - - for _, name := range suiteNames { - task := testSuites[name] - writeJobDefinitions(t, config, outputDir, task, name) - } -} - -func isTrackingBranch(inputDir, trackingBranch string) bool { - cmd := exec.Command("git", "-C", inputDir, "branch", "--show-current") - result, err := cmd.Output() - if err != nil { - log.Fatalf("unable to get current branch: %v", err) - } - if strings.TrimSpace(string(result)) != trackingBranch { - return false - } - - cmd = exec.Command("git", "-C", inputDir, "fetch", "upstream") - if err := cmd.Run(); err != nil { - log.Fatalf("unable to fetch: %v", err) - } - - cmd = exec.Command("git", "-C", inputDir, "diff", "--quiet", fmt.Sprintf("upstream/%s", trackingBranch)) - if err := cmd.Run(); err != nil { - log.Fatalf("unable to diff: %v", err) - } - - return true + return testSuites } func writeJobDefinitions( @@ -274,6 +343,7 @@ func writeJobDefinitions( } minVersions := make(map[string]string) + maxVersions := make(map[string]string) for _, task := range task.SubTasks { if introduced, ok := config.Folders.Introduced[task]; ok { minVersions[task] = minVersionRegex[introduced] @@ -281,9 +351,15 @@ func writeJobDefinitions( if introduced, ok := config.Folders.Introduced[suiteName+"-"+task]; ok { minVersions[suiteName+"-"+task] = minVersionRegex[introduced] } - if introduced, ok := config.Folders.Introduced[suiteName+"-*"]; ok { + if introduced, ok := config.Folders.Introduced[suiteName]; ok { minVersions[suiteName+"-"+task] = minVersionRegex[introduced] } + if removed, ok := config.Folders.Removed[suiteName+"-"+task]; ok { + maxVersions[suiteName+"-"+task] = minVersionRegex[removed] + } + if removed, ok := config.Folders.Removed[suiteName]; ok { + maxVersions[suiteName+"-"+task] = minVersionRegex[removed] + } } if err := t.Execute(f, struct { @@ -296,6 +372,7 @@ func writeJobDefinitions( CrossCloud map[string]bool Timeout map[string]int MinVersions map[string]string + MaxVersions map[string]string }{ SuiteName: suiteName, Clouds: task.Clouds, @@ -306,39 +383,48 @@ func writeJobDefinitions( CrossCloud: crossCloud, Timeout: task.Timeout, MinVersions: minVersions, + MaxVersions: maxVersions, }); err != nil { log.Fatalf("unable to execute template %q with error %v", suiteName, err) } f.Sync() } -func parseTaskNames(rootDir string, dir os.FileInfo) []string { +func parseTaskNames(dir ghObject) []string { tasks := make(map[string]int) - leaf := filepath.Join(rootDir, dir.Name()) - filepath.Walk(leaf, func(s string, d os.FileInfo, e error) error { - if e != nil { - return e - } - matched, err := regexp.Match("^"+leaf+"/\\w+.sh$", []byte(s)) - if err != nil { - return err + ghObjects, err := fetchGithubObjects(dir.URL) + if err != nil { + log.Fatalf("unable to fetch test suite %q files: %v", dir.Name, err) + } + + for _, f := range ghObjects { + if !strings.HasSuffix(f.Name, ".sh") { + continue } - if !matched { - return nil + + req, _ := http.NewRequest("GET", f.DownloadURL, nil) + token := os.Getenv("GH_TOKEN") + if token == "" { + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) } - // Now we've got a match, read the file. - file, err := os.Open(filepath.Join(leaf, d.Name())) + resp, err := http.DefaultClient.Do(req) if err != nil { - return err + log.Fatalf("unable to get test file %q context: %v", f.Name, err) + } + if resp.StatusCode != 200 { + _ = resp.Body.Close() + log.Fatalf("unable to get test file %q context; status code: %d", f.Name, resp.StatusCode) } parser := syntax.NewParser(syntax.Variant(syntax.LangBash)) - prog, err := parser.Parse(file, d.Name()) + prog, err := parser.Parse(resp.Body, "task.sh") if err != nil { - return err + _ = resp.Body.Close() + log.Fatalf("unable to parse test suite task content with error %v", err) } + _ = resp.Body.Close() syntax.Walk(prog, func(node syntax.Node) bool { switch t := node.(type) { case *syntax.FuncDecl: @@ -383,12 +469,7 @@ func parseTaskNames(rootDir string, dir os.FileInfo) []string { } return true }) - if err != nil { - return err - } - - return nil - }) + } subtasks := make([]string, 0, len(tasks)) for name, count := range tasks { @@ -460,8 +541,8 @@ const Template = ` {{- range $cloud := $node.Clouds -}} {{- $task_name := "" -}} {{- $test_name := (printf "%s-%s" $.SuiteName $cloud.Name) -}} - {{- $task_name = index $node.TaskNames $k -}} - {{- $full_task_name := (printf "test-%s-%s-%s" $.SuiteName (ensureHyphen $task_name) $cloud.Name) -}} + {{- $task_name = index $node.TaskNames $k -}} + {{- $full_task_name := (printf "test-%s-%s-%s" $.SuiteName (ensureHyphen $task_name) $cloud.Name) -}} {{- $builder := "run-integration-test" -}} {{- $run_on := "ephemeral-noble-small-amd64" -}} @@ -545,39 +626,97 @@ const Template = ` - select-oci-registry - wait-for-cloud-init - prepare-integration-test -{{- if or (index $.MinVersions $task_name) (index $.MinVersions (printf "%s-%s" $.SuiteName $task_name)) }} - {{- $cond := index $.MinVersions $task_name -}} - {{- if eq $cond "" }} - {{- $cond = index $.MinVersions (printf "%s-%s" $.SuiteName $task_name) -}} - {{- end }} +{{- $minRegexp := index $.MinVersions $task_name -}} +{{- if eq $minRegexp "" }} + {{- $minRegexp = index $.MinVersions (printf "%s-%s" $.SuiteName $task_name) -}} +{{- end }} +{{- $excludeRegexp := index $.MaxVersions (printf "%s-%s" $.SuiteName $task_name) -}} +{{- if or (ne $minRegexp "") (ne $excludeRegexp "") }} - conditional-step: + {{- if and (ne $minRegexp "") (ne $excludeRegexp "") }} + condition-kind: and + condition-operands: + # Do not run on regexp version match. + # Accounts for tests which do not exist + # in later Juju versions. + - condition-kind: not + condition-operand: + condition-kind: regex-match + regex: "{{ $excludeRegexp }}" + label: "{{ "${{JUJU_VERSION}}" }}" + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. + - condition-kind: regex-match + regex: "{{ $minRegexp }}" + label: "{{ "${{JUJU_VERSION}}" }}" + on-evaluation-failure: "dont-run" + steps: + - {{$builder}}: + test_name: '{{$.SuiteName}}' + setup_steps: '' + {{- if gt (len $node.SkipTasks) 1 }} + task_name: '{{$task_name}}' + skip_tasks: '{{$skip_tasks}}' + {{- else }} + task_name: '' + skip_tasks: '{{$node.ExcludedTasks}}' + {{- end}} + {{- else }} + {{- if ne $excludeRegexp "" }} + # Do not run on regexp version match. + # Accounts for tests which do not exist + # in later Juju versions. + condition-kind: not + condition-operand: + condition-kind: regex-match + regex: "{{ $excludeRegexp }}" + label: "{{ "${{JUJU_VERSION}}" }}" + on-evaluation-failure: "dont-run" + steps: + - {{$builder}}: + test_name: '{{$.SuiteName}}' + setup_steps: '' + {{- if gt (len $node.SkipTasks) 1 }} + task_name: '{{$task_name}}' + skip_tasks: '{{$skip_tasks}}' + {{- else }} + task_name: '' + skip_tasks: '{{$node.ExcludedTasks}}' + {{- end}} + {{- else }} + # Only run on regexp version match. + # Accounts for tests which do not exist + # until a given Juju version. condition-kind: regex-match - regex: "{{ $cond }}" + regex: "{{ $minRegexp }}" label: "{{ "${{JUJU_VERSION}}" }}" on-evaluation-failure: "dont-run" steps: - {{$builder}}: test_name: '{{$.SuiteName}}' setup_steps: '' - {{- if gt (len $node.SkipTasks) 1 }} + {{- if gt (len $node.SkipTasks) 1 }} task_name: '{{$task_name}}' skip_tasks: '{{$skip_tasks}}' - {{- else }} + {{- else }} task_name: '' skip_tasks: '{{$node.ExcludedTasks}}' - {{- end}} + {{- end}} + {{- end }} + {{- end }} {{- else }} - {{$builder}}: test_name: '{{$.SuiteName}}' setup_steps: '' - {{- if gt (len $node.SkipTasks) 1 }} + {{- if gt (len $node.SkipTasks) 1 }} task_name: '{{$task_name}}' skip_tasks: '{{$skip_tasks}}' - {{- else }} + {{- else }} task_name: '' skip_tasks: '{{$node.ExcludedTasks}}' - {{- end}} -{{- end}} + {{- end}} +{{- end }} publishers: - integration-artifacts {{- end }}