From 167466bd07dc4b2b1e33a340258f2d9d5c89c312 Mon Sep 17 00:00:00 2001 From: Rob Ferguson Date: Tue, 1 Oct 2024 15:07:35 -0500 Subject: [PATCH 01/16] wip base layer --- packages/base/tasks.yaml | 9 ++++ packages/base/zarf.yaml | 47 +++++++++++++++++++ packages/standard/zarf.yaml | 14 +++--- src/pepr/config.ts | 2 + src/pepr/operator/index.ts | 16 +++++++ .../reconcilers/package-reconciler.ts | 15 ++++-- tasks.yaml | 8 +++- tasks/create.yaml | 8 ++++ tasks/deploy.yaml | 5 ++ 9 files changed, 113 insertions(+), 11 deletions(-) create mode 100644 packages/base/tasks.yaml create mode 100644 packages/base/zarf.yaml diff --git a/packages/base/tasks.yaml b/packages/base/tasks.yaml new file mode 100644 index 000000000..f5c835749 --- /dev/null +++ b/packages/base/tasks.yaml @@ -0,0 +1,9 @@ +includes: + - istio: ../../src/istio/tasks.yaml + - pepr: ../../src/pepr/tasks.yaml + +tasks: + - name: validate + actions: + - task: istio:validate + - task: pepr:validate \ No newline at end of file diff --git a/packages/base/zarf.yaml b/packages/base/zarf.yaml new file mode 100644 index 000000000..8ec49d49a --- /dev/null +++ b/packages/base/zarf.yaml @@ -0,0 +1,47 @@ +kind: ZarfPackageConfig +metadata: + name: core-base + description: "UDS Core (Base)" + authors: "Defense Unicorns - Product" + # x-release-please-start-version + version: "0.28.0" + # x-release-please-end + +components: + - name: uds-operator-config + required: true + import: + path: ../../src/pepr + + # CRDs + - name: prometheus-operator-crds + required: true + import: + path: ../../src/prometheus-stack + + # Istio + - name: istio-controlplane + required: true + import: + path: ../../src/istio + + - name: istio-admin-gateway + required: true + import: + path: ../../src/istio + + - name: istio-tenant-gateway + required: true + import: + path: ../../src/istio + + - name: istio-passthrough-gateway + required: false + import: + path: ../../src/istio + + # Pepr the world + - name: pepr-uds-core + required: true + import: + path: ../../src/pepr diff --git a/packages/standard/zarf.yaml b/packages/standard/zarf.yaml index ef5ff903b..e055b5b35 100644 --- a/packages/standard/zarf.yaml +++ b/packages/standard/zarf.yaml @@ -11,40 +11,40 @@ components: - name: uds-operator-config required: true import: - path: ../../src/pepr + path: ../base # CRDs - name: prometheus-operator-crds required: true import: - path: ../../src/prometheus-stack + path: ../base # Istio - name: istio-controlplane required: true import: - path: ../../src/istio + path: ../base - name: istio-admin-gateway required: true import: - path: ../../src/istio + path: ../base - name: istio-tenant-gateway required: true import: - path: ../../src/istio + path: ../base - name: istio-passthrough-gateway required: false import: - path: ../../src/istio + path: ../base # Pepr the world - name: pepr-uds-core required: true import: - path: ../../src/pepr + path: ../base # Metrics Server - name: metrics-server diff --git a/src/pepr/config.ts b/src/pepr/config.ts index 183f504d9..9f15b0cc7 100644 --- a/src/pepr/config.ts +++ b/src/pepr/config.ts @@ -27,6 +27,8 @@ export const UDSConfig = { // Redis URI for Authservice authserviceRedisUri, + + isIdentityDeployed: false, }; // configure subproject logger diff --git a/src/pepr/operator/index.ts b/src/pepr/operator/index.ts index 5c11232a2..f9e44c04d 100644 --- a/src/pepr/operator/index.ts +++ b/src/pepr/operator/index.ts @@ -16,6 +16,7 @@ import { UDSExemption, UDSPackage } from "./crd"; import { validator } from "./crd/validators/package-validator"; // Reconciler imports +import { UDSConfig } from "../config"; import { purgeAuthserviceClients } from "./controllers/keycloak/authservice/authservice"; import { exemptValidator } from "./crd/validators/exempt-validator"; import { packageReconciler } from "./reconcilers/package-reconciler"; @@ -63,3 +64,18 @@ When(UDSPackage) // Watch for Exemptions and validate When(UDSExemption).IsCreatedOrUpdated().Validate(exemptValidator); + + +// Watch for Functional Layers and update config +When(UDSPackage) + .IsCreatedOrUpdated() + .WithName("keycloak") + .Watch(() => { + UDSConfig.isIdentityDeployed = true; + }); +When(UDSPackage) + .IsDeleted() + .WithName("keycloak") + .Watch(() => { + UDSConfig.isIdentityDeployed = false; + }); \ No newline at end of file diff --git a/src/pepr/operator/reconcilers/package-reconciler.ts b/src/pepr/operator/reconcilers/package-reconciler.ts index 568f71afa..4798398fa 100644 --- a/src/pepr/operator/reconcilers/package-reconciler.ts +++ b/src/pepr/operator/reconcilers/package-reconciler.ts @@ -5,6 +5,7 @@ import { enableInjection } from "../controllers/istio/injection"; import { istioResources } from "../controllers/istio/istio-resources"; import { authservice } from "../controllers/keycloak/authservice/authservice"; import { keycloak } from "../controllers/keycloak/client-sync"; +import { Client } from "../controllers/keycloak/types"; import { podMonitor } from "../controllers/monitoring/pod-monitor"; import { serviceMonitor } from "../controllers/monitoring/service-monitor"; import { networkPolicies } from "../controllers/network/policies"; @@ -65,9 +66,17 @@ export async function packageReconciler(pkg: UDSPackage) { // Update the namespace to ensure the istio-injection label is set await enableInjection(pkg); - // Configure SSO - const ssoClients = await keycloak(pkg); - const authserviceClients = await authservice(pkg, ssoClients); + let ssoClients = new Map(); + let authserviceClients: string[] = []; + + if (UDSConfig.isIdentityDeployed) { + // Configure SSO + ssoClients = await keycloak(pkg); + authserviceClients = await authservice(pkg, ssoClients); + } else if (pkg.spec?.sso) { + // TODO: Create event for Package? Or maybe fail + log.error("SSO is not deployed, but the package has SSO configuration"); + } // Create the VirtualService and ServiceEntry for each exposed service endpoints = await istioResources(pkg, namespace!); diff --git a/tasks.yaml b/tasks.yaml index 0ee6c3e46..6b2a150af 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -10,9 +10,15 @@ includes: - deploy: ./tasks/deploy.yaml - test: ./tasks/test.yaml - lint: ./tasks/lint.yaml + - base: ./packages/base/tasks.yaml tasks: - + - name: deploy-base + actions: + - task: create:core-base + - task: setup:k3d-test-cluster + - task: deploy:core-base + - task: base:validate - name: default actions: - description: "Build, deploy and test UDS Core" diff --git a/tasks/create.yaml b/tasks/create.yaml index 9480a8944..6dd571ade 100644 --- a/tasks/create.yaml +++ b/tasks/create.yaml @@ -18,6 +18,14 @@ tasks: - description: "Create the UDS Core Standard Zarf Package" cmd: "uds zarf package create packages/standard --confirm --no-progress --flavor ${FLAVOR}" + - name: core-base + description: "Create the UDS Core Base Package" + actions: + - task: pepr-build + + - description: "Create the UDS Core Standard Zarf Package" + cmd: "uds zarf package create packages/base --confirm --no-progress --flavor ${FLAVOR}" + - name: k3d-standard-bundle description: "Create the K3d-UDS Core Bundle" actions: diff --git a/tasks/deploy.yaml b/tasks/deploy.yaml index 0d1ff8a88..e0eb59f4d 100644 --- a/tasks/deploy.yaml +++ b/tasks/deploy.yaml @@ -55,3 +55,8 @@ tasks: actions: - description: "Deploy the standard UDS Core zarf package" cmd: uds zarf package deploy build/zarf-package-core-${UDS_ARCH}-${VERSION}.tar.zst --confirm --no-progress --components '*' + + - name: core-base + actions: + - description: "Deploy the standard UDS Core (Base) zarf package" + cmd: uds zarf package deploy build/zarf-package-core-base-${UDS_ARCH}-${VERSION}.tar.zst --confirm --no-progress --components '*' From 840ae2248aeebcb01705a6ef603d53a6204c21aa Mon Sep 17 00:00:00 2001 From: Rob Ferguson Date: Tue, 1 Oct 2024 17:07:30 -0500 Subject: [PATCH 02/16] rework tasks and CI for functional layers --- .github/filters.yaml | 86 +------------------ .github/workflows/publish.yaml | 59 +++++++++++++ .github/workflows/test.yaml | 8 +- packages/identity-authorization/tasks.yaml | 9 ++ packages/identity-authorization/zarf.yaml | 21 +++++ packages/standard/zarf.yaml | 4 +- .../reconcilers/package-reconciler.ts | 1 + tasks.yaml | 7 -- tasks/create.yaml | 8 +- tasks/deploy.yaml | 6 ++ tasks/publish.yaml | 14 +++ tasks/test.yaml | 19 ++++ 12 files changed, 146 insertions(+), 96 deletions(-) create mode 100644 packages/identity-authorization/tasks.yaml create mode 100644 packages/identity-authorization/zarf.yaml diff --git a/.github/filters.yaml b/.github/filters.yaml index 77ec1bba8..75e12a0b3 100644 --- a/.github/filters.yaml +++ b/.github/filters.yaml @@ -1,96 +1,18 @@ all: - "**" -authservice: - - "src/authservice/**" - - "!**/*.md" - - "!**/*.jpg" - - "!**/*.png" - - "!**/*.gif" - - "!**/*.svg" - -grafana: - - "src/grafana/**" - - "!**/*.md" - - "!**/*.jpg" - - "!**/*.png" - - "!**/*.gif" - - "!**/*.svg" - -istio: +base: - "src/istio/**" + - "src/pepr/**" - "!**/*.md" - "!**/*.jpg" - "!**/*.png" - "!**/*.gif" - "!**/*.svg" -keycloak: +identity-authorization: - "src/keycloak/**" - - "!**/*.md" - - "!**/*.jpg" - - "!**/*.png" - - "!**/*.gif" - - "!**/*.svg" - -kiali: - - "src/kiali/**" - - "!**/*.md" - - "!**/*.jpg" - - "!**/*.png" - - "!**/*.gif" - - "!**/*.svg" - -loki: - - "src/loki/**" - - "!**/*.md" - - "!**/*.jpg" - - "!**/*.png" - - "!**/*.gif" - - "!**/*.svg" - -metrics-server: - - "src/metrics-server/**" - - "!**/*.md" - - "!**/*.jpg" - - "!**/*.png" - - "!**/*.gif" - - "!**/*.svg" - -neuvector: - - "src/neuvector/**" - - "!**/*.md" - - "!**/*.jpg" - - "!**/*.png" - - "!**/*.gif" - - "!**/*.svg" - -prometheus-stack: - - "src/prometheus-stack/**" - - "!**/*.md" - - "!**/*.jpg" - - "!**/*.png" - - "!**/*.gif" - - "!**/*.svg" - -vector: - - "src/vector/**" - - "!**/*.md" - - "!**/*.jpg" - - "!**/*.png" - - "!**/*.gif" - - "!**/*.svg" - -tempo: - - "src/tempo/**" - - "!**/*.md" - - "!**/*.jpg" - - "!**/*.png" - - "!**/*.gif" - - "!**/*.svg" - -velero: - - "src/velero/**" + - "src/authservice/**" - "!**/*.md" - "!**/*.jpg" - "!**/*.png" diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 6ed33ecbe..31c025c0e 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -87,3 +87,62 @@ jobs: uses: ./.github/actions/save-logs with: suffix: -${{ matrix.flavor }} + publish-uds-core-layers: + if: ${{ !inputs.snapshot }} + strategy: + matrix: + flavor: [upstream, registry1, unicorn] + # maybe handle base as a special case to avoid duplicate building the layer + layer: [base, identity-authorization] + runs-on: "ubuntu-latest" + name: Publish package layers + + permissions: + contents: read + packages: write + id-token: write # This is needed for OIDC federation. + + steps: + - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + + - name: Environment setup + uses: ./.github/actions/setup + with: + registry1Username: ${{ secrets.IRON_BANK_ROBOT_USERNAME }} + registry1Password: ${{ secrets.IRON_BANK_ROBOT_PASSWORD }} + ghToken: ${{ secrets.GITHUB_TOKEN }} + chainguardIdentity: ${{ secrets.CHAINGUARD_IDENTITY }} + + - name: Create Package Layers + env: + UDS_LAYER: ${{ matrix.layer }} + run: | + ZARF_ARCHITECTURE=amd64 uds run create:single-layer --no-progress --set FLAVOR=${{ matrix.flavor }} + + if [ "${{ matrix.flavor }}" != "registry1" ]; then + ZARF_ARCHITECTURE=arm64 uds run create:single-layer --no-progress --set FLAVOR=${{ matrix.flavor }} + fi + + # Core package layer test + - name: Test amd64 Bundle + env: + UDS_LAYER: ${{ matrix.layer }} + run: | + uds run test:base --set FLAVOR=${{ matrix.flavor }} + uds run deploy:single-layer + uds run -f packages/${{ matrix.layer }}/tasks.yaml validate --no-progress + + - name: Debug Output + if: ${{ always() && !inputs.snapshot }} + uses: ./.github/actions/debug-output + + - name: Publish Core Package Layer + env: + UDS_LAYER: ${{ matrix.layer }} + run: uds run -f tasks/publish.yaml single-layer --set FLAVOR=${{ matrix.flavor }} --no-progress + + - name: Save logs + if: always() + uses: ./.github/actions/save-logs + with: + suffix: -${{ matrix.layer }} \ No newline at end of file diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a5ed0ce24..a9d918e53 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -21,7 +21,7 @@ on: inputs: package: type: string - description: "The name of the source package to test" + description: "The name of the core package layer to test" required: true flavor: type: string @@ -43,7 +43,7 @@ jobs: timeout-minutes: 30 name: Test env: - UDS_PKG: ${{ inputs.package }} + UDS_LAYER: ${{ inputs.package }} steps: - name: Checkout repository @@ -57,9 +57,9 @@ jobs: ghToken: ${{ secrets.GITHUB_TOKEN }} chainguardIdentity: ${{ secrets.CHAINGUARD_IDENTITY }} - - name: Test a single source package + - name: Test a single layer package if: ${{ inputs.package != 'all' && inputs.test_type == 'install' }} - run: uds run test-single-package --set FLAVOR=${{ inputs.flavor }} --no-progress + run: uds run test:single-layer --set FLAVOR=${{ inputs.flavor }} --no-progress - name: Test UDS Core Install if: ${{ inputs.package == 'all' && inputs.test_type == 'install' }} diff --git a/packages/identity-authorization/tasks.yaml b/packages/identity-authorization/tasks.yaml new file mode 100644 index 000000000..f3320586c --- /dev/null +++ b/packages/identity-authorization/tasks.yaml @@ -0,0 +1,9 @@ +includes: + - keycloak: ../../src/keycloak/tasks.yaml + - authservice: ../../src/authservice/tasks.yaml + +tasks: + - name: validate + actions: + - task: keycloak:validate + - task: authservice:validate \ No newline at end of file diff --git a/packages/identity-authorization/zarf.yaml b/packages/identity-authorization/zarf.yaml new file mode 100644 index 000000000..ae60e60ec --- /dev/null +++ b/packages/identity-authorization/zarf.yaml @@ -0,0 +1,21 @@ +kind: ZarfPackageConfig +metadata: + name: core-identity-authorization + description: "UDS Core (Identity & Authorization)" + authors: "Defense Unicorns - Product" + # x-release-please-start-version + version: "0.28.0" + # x-release-please-end + +components: + # Keycloak + - name: keycloak + required: true + import: + path: ../../src/keycloak + + # Authservice + - name: authservice + required: true + import: + path: ../../src/authservice \ No newline at end of file diff --git a/packages/standard/zarf.yaml b/packages/standard/zarf.yaml index e055b5b35..b021aeabd 100644 --- a/packages/standard/zarf.yaml +++ b/packages/standard/zarf.yaml @@ -56,7 +56,7 @@ components: - name: keycloak required: true import: - path: ../../src/keycloak + path: ../identity-authorization # Neuvector - name: neuvector @@ -92,7 +92,7 @@ components: - name: authservice required: true import: - path: ../../src/authservice + path: ../identity-authorization # UDS Runtime - name: uds-runtime diff --git a/src/pepr/operator/reconcilers/package-reconciler.ts b/src/pepr/operator/reconcilers/package-reconciler.ts index 4798398fa..f84542d63 100644 --- a/src/pepr/operator/reconcilers/package-reconciler.ts +++ b/src/pepr/operator/reconcilers/package-reconciler.ts @@ -76,6 +76,7 @@ export async function packageReconciler(pkg: UDSPackage) { } else if (pkg.spec?.sso) { // TODO: Create event for Package? Or maybe fail log.error("SSO is not deployed, but the package has SSO configuration"); + // throw new Error("SSO is not deployed, but the package has SSO configuration"); } // Create the VirtualService and ServiceEntry for each exposed service diff --git a/tasks.yaml b/tasks.yaml index 6b2a150af..c330498e3 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -10,15 +10,8 @@ includes: - deploy: ./tasks/deploy.yaml - test: ./tasks/test.yaml - lint: ./tasks/lint.yaml - - base: ./packages/base/tasks.yaml tasks: - - name: deploy-base - actions: - - task: create:core-base - - task: setup:k3d-test-cluster - - task: deploy:core-base - - task: base:validate - name: default actions: - description: "Build, deploy and test UDS Core" diff --git a/tasks/create.yaml b/tasks/create.yaml index 6dd571ade..340b3266f 100644 --- a/tasks/create.yaml +++ b/tasks/create.yaml @@ -24,7 +24,7 @@ tasks: - task: pepr-build - description: "Create the UDS Core Standard Zarf Package" - cmd: "uds zarf package create packages/base --confirm --no-progress --flavor ${FLAVOR}" + cmd: "UDS_LAYER=base uds run create:single-layer --set FLAVOR=${FLAVOR}" - name: k3d-standard-bundle description: "Create the K3d-UDS Core Bundle" @@ -63,6 +63,12 @@ tasks: uds zarf package create src/istio --confirm --no-progress --flavor ${FLAVOR} fi + - name: single-layer + description: "Create a single Zarf Package, must set UDS_LAYER environment variable" + actions: + - description: "Create the requested Zarf Package (must set UDS_LAYER environment variable)" + cmd: "uds zarf package create packages/${UDS_LAYER} --confirm --no-progress --flavor ${FLAVOR}" + - name: pepr-build description: "Build the UDS Core Pepr Module" actions: diff --git a/tasks/deploy.yaml b/tasks/deploy.yaml index e0eb59f4d..7041e2bc6 100644 --- a/tasks/deploy.yaml +++ b/tasks/deploy.yaml @@ -41,6 +41,12 @@ tasks: - description: "Deploy the requested Zarf Package (must set UDS_PKG environment variable)" cmd: uds zarf package deploy build/zarf-package-uds-core-${UDS_PKG}-${UDS_ARCH}.tar.zst --confirm --no-progress --components '*' + - name: single-layer + description: "Deploy a single UDS Core layer, must set UDS_LAYER environment variable" + actions: + - description: "Deploy a single UDS Core Layer (must set UDS_LAYER environment variable)" + cmd: uds zarf package deploy build/zarf-package-core-${UDS_LAYER}-${UDS_ARCH}-${VERSION}.tar.zst --confirm --no-progress --components '*' + - name: latest-package-release actions: - task: utils:determine-repo diff --git a/tasks/publish.yaml b/tasks/publish.yaml index 30e70f557..90baa35a8 100644 --- a/tasks/publish.yaml +++ b/tasks/publish.yaml @@ -52,3 +52,17 @@ tasks: uds zarf tools registry copy ${pkgPath}:${VERSION} ${pkgPath}:latest pkgPath="${TARGET_REPO}/bundles/k3d-core-slim-dev" uds zarf tools registry copy ${pkgPath}:${VERSION} ${pkgPath}:latest + + - name: single-layer + description: "Publish UDS Core layer" + actions: + - task: utils:determine-repo + - description: "Publish amd64/arm64 packages per flavor" + cmd: | + echo "Publishing package to ${TARGET_REPO}" + uds zarf package publish build/zarf-package-core-${UDS_LAYER}-amd64-${VERSION}.tar.zst oci://${TARGET_REPO} + + # dont publish arm64 for registry1 since IB images are only amd64 + if [ "${FLAVOR}" != "registry1" ]; then + uds zarf package publish build/zarf-package-core-${UDS_LAYER}-arm64-${VERSION}.tar.zst oci://${TARGET_REPO} + fi diff --git a/tasks/test.yaml b/tasks/test.yaml index 185408e5a..4369841dc 100644 --- a/tasks/test.yaml +++ b/tasks/test.yaml @@ -3,6 +3,7 @@ includes: - setup: ./setup.yaml - deploy: ./deploy.yaml - compliance: https://raw.githubusercontent.com/defenseunicorns/uds-common/v0.13.1/tasks/compliance.yaml + - base-layer: ../packages/base/tasks.yaml tasks: - name: single-package @@ -15,6 +16,24 @@ tasks: - description: "Validate the package" cmd: uds run -f src/${UDS_PKG}/tasks.yaml validate --no-progress + - name: base + description: "Build and test the base layer" + actions: + - task: create:core-base + - task: setup:k3d-test-cluster + - task: deploy:core-base + - task: base-layer:validate + + - name: single-layer + description: "Build and test a single layer, must set UDS_LAYER environment variable" + actions: + - task: create:single-layer + - task: base + - task: deploy:single-layer + + - description: "Validate the package" + cmd: uds run -f packages/${UDS_LAYER}/tasks.yaml validate --no-progress + - name: validate-packages description: "Validated all packages" # loop through each src/* package and run the validate.yaml task From ef85acf97706626b66e3870a23bcf1ec58b9df12 Mon Sep 17 00:00:00 2001 From: Rob Ferguson Date: Tue, 1 Oct 2024 17:22:49 -0500 Subject: [PATCH 03/16] fix formatting --- src/pepr/operator/index.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pepr/operator/index.ts b/src/pepr/operator/index.ts index f9e44c04d..33ac7af40 100644 --- a/src/pepr/operator/index.ts +++ b/src/pepr/operator/index.ts @@ -65,7 +65,6 @@ When(UDSPackage) // Watch for Exemptions and validate When(UDSExemption).IsCreatedOrUpdated().Validate(exemptValidator); - // Watch for Functional Layers and update config When(UDSPackage) .IsCreatedOrUpdated() @@ -78,4 +77,4 @@ When(UDSPackage) .WithName("keycloak") .Watch(() => { UDSConfig.isIdentityDeployed = false; - }); \ No newline at end of file + }); From 0665fc40de245d68d64e400f91160aa9656a18c5 Mon Sep 17 00:00:00 2001 From: Rob Ferguson Date: Wed, 2 Oct 2024 12:01:58 -0500 Subject: [PATCH 04/16] slim dev with layers --- bundles/k3d-slim-dev/uds-bundle.yaml | 9 ++++- packages/slim-dev/README.md | 3 -- packages/slim-dev/zarf.yaml | 53 ---------------------------- 3 files changed, 8 insertions(+), 57 deletions(-) delete mode 100644 packages/slim-dev/README.md delete mode 100644 packages/slim-dev/zarf.yaml diff --git a/bundles/k3d-slim-dev/uds-bundle.yaml b/bundles/k3d-slim-dev/uds-bundle.yaml index e63b9793d..b29e4acef 100644 --- a/bundles/k3d-slim-dev/uds-bundle.yaml +++ b/bundles/k3d-slim-dev/uds-bundle.yaml @@ -31,7 +31,7 @@ packages: repository: ghcr.io/zarf-dev/packages/init ref: v0.40.1 - - name: core-slim-dev + - name: core-base path: ../../build/ # x-release-please-start-version ref: 0.28.0 @@ -66,6 +66,13 @@ packages: - name: TENANT_SERVICE_PORTS description: "The ports that are exposed from the tenant gateway LoadBalancer (useful for non-HTTP(S) traffic)" path: "service.ports" + + - name: core-identity-authorization + path: ../../build/ + # x-release-please-start-version + ref: 0.28.0 + # x-release-please-end + overrides: keycloak: keycloak: variables: diff --git a/packages/slim-dev/README.md b/packages/slim-dev/README.md deleted file mode 100644 index 1e3b1dce6..000000000 --- a/packages/slim-dev/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# UDS Core Slim Dev - -This is a special modified version of UDS Core that only includes the components needed to run Istio, UDS Operator and Keycloak. diff --git a/packages/slim-dev/zarf.yaml b/packages/slim-dev/zarf.yaml deleted file mode 100644 index 7a7de7a31..000000000 --- a/packages/slim-dev/zarf.yaml +++ /dev/null @@ -1,53 +0,0 @@ -kind: ZarfPackageConfig -metadata: - name: core-slim-dev - description: "UDS Core (Istio, UDS Operator and Keycloak)" - authors: "Defense Unicorns - Product" - # x-release-please-start-version - version: "0.28.0" - # x-release-please-end - -components: - - name: uds-operator-config - required: true - import: - path: ../../src/pepr - - # CRDs - - name: prometheus-operator-crds - required: true - import: - path: ../../src/prometheus-stack - - # Istio - - name: istio-controlplane - required: true - import: - path: ../../src/istio - - - name: istio-admin-gateway - required: true - import: - path: ../../src/istio - - - name: istio-tenant-gateway - required: true - import: - path: ../../src/istio - - - name: istio-passthrough-gateway - required: false - import: - path: ../../src/istio - - # Pepr the world - - name: pepr-uds-core - required: true - import: - path: ../../src/pepr - - # Keycloak - - name: keycloak - required: true - import: - path: ../../src/keycloak From a6497017b13497863487a505cd4028f89ebb66d5 Mon Sep 17 00:00:00 2001 From: Rob Ferguson Date: Wed, 2 Oct 2024 12:02:25 -0500 Subject: [PATCH 05/16] error package if sso is not enabled --- src/pepr/operator/reconcilers/package-reconciler.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pepr/operator/reconcilers/package-reconciler.ts b/src/pepr/operator/reconcilers/package-reconciler.ts index f84542d63..8bb193b1b 100644 --- a/src/pepr/operator/reconcilers/package-reconciler.ts +++ b/src/pepr/operator/reconcilers/package-reconciler.ts @@ -74,9 +74,8 @@ export async function packageReconciler(pkg: UDSPackage) { ssoClients = await keycloak(pkg); authserviceClients = await authservice(pkg, ssoClients); } else if (pkg.spec?.sso) { - // TODO: Create event for Package? Or maybe fail log.error("SSO is not deployed, but the package has SSO configuration"); - // throw new Error("SSO is not deployed, but the package has SSO configuration"); + throw new Error("Identity & Authorization is not deployed, but the package has SSO configuration"); } // Create the VirtualService and ServiceEntry for each exposed service From 9d9b4fcfe586059f80cc35f5e6283fa21a2a8028 Mon Sep 17 00:00:00 2001 From: Rob Ferguson Date: Wed, 2 Oct 2024 12:53:39 -0500 Subject: [PATCH 06/16] update tasks to use more variables and inputs --- .github/workflows/test.yaml | 2 +- tasks.yaml | 19 +++++++++++++++---- tasks/create.yaml | 24 ++++++++++-------------- tasks/deploy.yaml | 11 +++++------ tasks/test.yaml | 13 +++++++------ 5 files changed, 38 insertions(+), 31 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a9d918e53..abbaa3511 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -59,7 +59,7 @@ jobs: - name: Test a single layer package if: ${{ inputs.package != 'all' && inputs.test_type == 'install' }} - run: uds run test:single-layer --set FLAVOR=${{ inputs.flavor }} --no-progress + run: uds run test-single-layer --set FLAVOR=${{ inputs.flavor }} --set LAYER=${{ inputs.package }} --no-progress - name: Test UDS Core Install if: ${{ inputs.package == 'all' && inputs.test_type == 'install' }} diff --git a/tasks.yaml b/tasks.yaml index c330498e3..0e3a41e3e 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -4,6 +4,8 @@ variables: - name: PKG + - name: LAYER + includes: - create: ./tasks/create.yaml - setup: ./tasks/setup.yaml @@ -39,8 +41,15 @@ tasks: - name: slim-dev actions: - - description: "Create slim dev package" - task: create:slim-dev-package + - description: "Create base package" + task: create:single-layer + with: + layer: base + + - description: "Create identity-authorization package" + task: create:single-layer + with: + layer: identity-authorization - description: "Build slim dev bundle" task: create:k3d-slim-dev-bundle @@ -82,10 +91,12 @@ tasks: actions: - task: create:standard-package - - name: deploy-single-package + - name: test-single-layer description: "Deploy Pepr Module and a Zarf Package using UDS_PKG environment variable" actions: - - task: deploy:single-package + - task: test:base + - task: test:single-layer + if: ${{ ne .variables.LAYER "base" }} - name: deploy-standard-bundle actions: diff --git a/tasks/create.yaml b/tasks/create.yaml index 340b3266f..12715c874 100644 --- a/tasks/create.yaml +++ b/tasks/create.yaml @@ -9,6 +9,8 @@ variables: # renovate: datasource=docker depName=registry1.dso.mil/ironbank/opensource/defenseunicorns/pepr/controller versioning=semver default: registry1.dso.mil/ironbank/opensource/defenseunicorns/pepr/controller:v0.36.0 + - name: LAYER + tasks: - name: standard-package description: "Create the UDS Core Zarf Package" @@ -24,7 +26,7 @@ tasks: - task: pepr-build - description: "Create the UDS Core Standard Zarf Package" - cmd: "UDS_LAYER=base uds run create:single-layer --set FLAVOR=${FLAVOR}" + cmd: "uds run create:single-layer --set FLAVOR=${FLAVOR} --set=layer=base" - name: k3d-standard-bundle description: "Create the K3d-UDS Core Bundle" @@ -32,18 +34,10 @@ tasks: - description: "Create the UDS Core Standard Bundle" cmd: "uds create bundles/k3d-standard --confirm --no-progress --architecture=${ZARF_ARCHITECTURE}" - - name: slim-dev-package - description: "Create the UDS Core (Istio Only) Zarf Package" - actions: - - task: pepr-build - - - description: "Create the UDS Core Istio Zarf Package" - cmd: "uds zarf package create packages/slim-dev --confirm --no-progress --flavor ${FLAVOR}" - - name: k3d-slim-dev-bundle - description: "Create the K3d-UDS Core (Istio and Keycloak Only) Bundle" + description: "Create the slim dev bundle (Base and Identity)" actions: - - description: "Create the UDS Core Istio and Keycloak Only Bundle" + - description: "Create the slim dev bundle (Base and Identity)" cmd: "uds create bundles/k3d-slim-dev --confirm --no-progress --architecture=${ZARF_ARCHITECTURE}" - name: single-package @@ -64,10 +58,12 @@ tasks: fi - name: single-layer - description: "Create a single Zarf Package, must set UDS_LAYER environment variable" + inputs: + layer: + default: base + description: The UDS Core layer to build actions: - - description: "Create the requested Zarf Package (must set UDS_LAYER environment variable)" - cmd: "uds zarf package create packages/${UDS_LAYER} --confirm --no-progress --flavor ${FLAVOR}" + - cmd: uds zarf package create packages/${{ index .inputs "layer" }} --confirm --no-progress --flavor ${FLAVOR} - name: pepr-build description: "Build the UDS Core Pepr Module" diff --git a/tasks/deploy.yaml b/tasks/deploy.yaml index 7041e2bc6..e2082f598 100644 --- a/tasks/deploy.yaml +++ b/tasks/deploy.yaml @@ -43,9 +43,13 @@ tasks: - name: single-layer description: "Deploy a single UDS Core layer, must set UDS_LAYER environment variable" + inputs: + layer: + default: base + description: The UDS Core layer to deploy actions: - description: "Deploy a single UDS Core Layer (must set UDS_LAYER environment variable)" - cmd: uds zarf package deploy build/zarf-package-core-${UDS_LAYER}-${UDS_ARCH}-${VERSION}.tar.zst --confirm --no-progress --components '*' + cmd: uds zarf package deploy build/zarf-package-core-${{ index .inputs "layer" }}-${UDS_ARCH}-${VERSION}.tar.zst --confirm --no-progress --components '*' - name: latest-package-release actions: @@ -61,8 +65,3 @@ tasks: actions: - description: "Deploy the standard UDS Core zarf package" cmd: uds zarf package deploy build/zarf-package-core-${UDS_ARCH}-${VERSION}.tar.zst --confirm --no-progress --components '*' - - - name: core-base - actions: - - description: "Deploy the standard UDS Core (Base) zarf package" - cmd: uds zarf package deploy build/zarf-package-core-base-${UDS_ARCH}-${VERSION}.tar.zst --confirm --no-progress --components '*' diff --git a/tasks/test.yaml b/tasks/test.yaml index 4369841dc..f5a7d82e5 100644 --- a/tasks/test.yaml +++ b/tasks/test.yaml @@ -19,20 +19,21 @@ tasks: - name: base description: "Build and test the base layer" actions: - - task: create:core-base + - task: create:pepr-build - task: setup:k3d-test-cluster - - task: deploy:core-base - - task: base-layer:validate + - cmd: uds run -f tasks/test.yaml single-layer --set FLAVOR=${FLAVOR} --set=layer=base - name: single-layer description: "Build and test a single layer, must set UDS_LAYER environment variable" actions: - task: create:single-layer - - task: base + with: + layer: ${LAYER} - task: deploy:single-layer - + with: + layer: ${LAYER} - description: "Validate the package" - cmd: uds run -f packages/${UDS_LAYER}/tasks.yaml validate --no-progress + cmd: uds run -f packages/${LAYER}/tasks.yaml validate --no-progress - name: validate-packages description: "Validated all packages" From 6178244b89f42434dd2ca84be9395bdb9f9df669 Mon Sep 17 00:00:00 2001 From: Rob Ferguson Date: Wed, 2 Oct 2024 13:18:07 -0500 Subject: [PATCH 07/16] shuffle tasks around for inputs --- .github/workflows/publish.yaml | 12 ++++-------- tasks.yaml | 5 ----- tasks/create.yaml | 20 +++++--------------- tasks/deploy.yaml | 16 +++++----------- tasks/publish.yaml | 6 ++++-- 5 files changed, 18 insertions(+), 41 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 31c025c0e..56579ecf2 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -117,19 +117,17 @@ jobs: env: UDS_LAYER: ${{ matrix.layer }} run: | - ZARF_ARCHITECTURE=amd64 uds run create:single-layer --no-progress --set FLAVOR=${{ matrix.flavor }} + ZARF_ARCHITECTURE=amd64 uds run create:single-layer-callable --no-progress --set FLAVOR=${{ matrix.flavor }} --set LAYER=${{ matrix.layer }} if [ "${{ matrix.flavor }}" != "registry1" ]; then - ZARF_ARCHITECTURE=arm64 uds run create:single-layer --no-progress --set FLAVOR=${{ matrix.flavor }} + ZARF_ARCHITECTURE=arm64 uds run create:single-layer-callable --no-progress --set FLAVOR=${{ matrix.flavor }} --set LAYER=${{ matrix.layer }} fi # Core package layer test - name: Test amd64 Bundle - env: - UDS_LAYER: ${{ matrix.layer }} run: | uds run test:base --set FLAVOR=${{ matrix.flavor }} - uds run deploy:single-layer + uds run deploy:single-layer-callable --set LAYER=${{ matrix.layer }} uds run -f packages/${{ matrix.layer }}/tasks.yaml validate --no-progress - name: Debug Output @@ -137,9 +135,7 @@ jobs: uses: ./.github/actions/debug-output - name: Publish Core Package Layer - env: - UDS_LAYER: ${{ matrix.layer }} - run: uds run -f tasks/publish.yaml single-layer --set FLAVOR=${{ matrix.flavor }} --no-progress + run: uds run -f tasks/publish.yaml single-layer --set FLAVOR=${{ matrix.flavor }} --set LAYER=${{ matrix.layer }} --no-progress - name: Save logs if: always() diff --git a/tasks.yaml b/tasks.yaml index 0e3a41e3e..280457057 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -81,11 +81,6 @@ tasks: actions: - task: setup:k3d-test-cluster - - name: create-single-package - description: "Create a single Zarf Package, must set UDS_PKG environment variable" - actions: - - task: create:single-package - - name: create-standard-package description: "Create UDS Core Zarf Package, `upstream` flavor default, use --set FLAVOR={flavor} to change" actions: diff --git a/tasks/create.yaml b/tasks/create.yaml index 12715c874..22c2fd088 100644 --- a/tasks/create.yaml +++ b/tasks/create.yaml @@ -40,22 +40,12 @@ tasks: - description: "Create the slim dev bundle (Base and Identity)" cmd: "uds create bundles/k3d-slim-dev --confirm --no-progress --architecture=${ZARF_ARCHITECTURE}" - - name: single-package - description: "Create a single Zarf Package, must set UDS_PKG environment variable" + # This task is a wrapper to support --set LAYER=identity-authorization + - name: single-layer-callable actions: - - task: pepr-build - - - description: "Create the Pepr Zarf Package, if it exists" - cmd: "uds zarf package create src/pepr --confirm --no-progress" - - - description: "Create the requested Zarf Package (must set UDS_PKG environment variable)" - cmd: "uds zarf package create src/${UDS_PKG} --confirm --no-progress --flavor ${FLAVOR}" - - - description: "Create the Istio Zarf Package, if UDS_PKG != istio" - cmd: | - if [ "${UDS_PKG}" != "istio" ]; then - uds zarf package create src/istio --confirm --no-progress --flavor ${FLAVOR} - fi + - task: single-layer + with: + layer: $LAYER - name: single-layer inputs: diff --git a/tasks/deploy.yaml b/tasks/deploy.yaml index e2082f598..08c15b655 100644 --- a/tasks/deploy.yaml +++ b/tasks/deploy.yaml @@ -28,18 +28,12 @@ tasks: - description: "Deploy the UDS Core Slim Dev Only Bundle" cmd: uds deploy bundles/k3d-slim-dev/uds-bundle-k3d-core-slim-dev-${UDS_ARCH}-${VERSION}.tar.zst --confirm --no-progress - - name: single-package + # This task is a wrapper to support --set LAYER=identity-authorization + - name: single-layer-callable actions: - - description: "Deploy the Istio package, if UDS_PKG != istio" - cmd: | - if [ "${UDS_PKG}" != "istio" ]; then - uds zarf package deploy build/zarf-package-uds-core-istio-${UDS_ARCH}.tar.zst --confirm --no-progress --components '*' - fi - - description: "Deploy the Pepr Module" - cmd: | - uds zarf package deploy build/zarf-package-pepr-uds-core-${UDS_ARCH}.tar.zst --confirm --no-progress --set UDS_SINGLE_TEST=true --set PEPR_SERVICE_MONITORS=false - - description: "Deploy the requested Zarf Package (must set UDS_PKG environment variable)" - cmd: uds zarf package deploy build/zarf-package-uds-core-${UDS_PKG}-${UDS_ARCH}.tar.zst --confirm --no-progress --components '*' + - task: single-layer + with: + layer: $LAYER - name: single-layer description: "Deploy a single UDS Core layer, must set UDS_LAYER environment variable" diff --git a/tasks/publish.yaml b/tasks/publish.yaml index 90baa35a8..43a4d2b78 100644 --- a/tasks/publish.yaml +++ b/tasks/publish.yaml @@ -11,6 +11,8 @@ variables: default: "0.28.0" # x-release-please-end + - name: LAYER + tasks: - name: standard-package description: "Publish the UDS package" @@ -60,9 +62,9 @@ tasks: - description: "Publish amd64/arm64 packages per flavor" cmd: | echo "Publishing package to ${TARGET_REPO}" - uds zarf package publish build/zarf-package-core-${UDS_LAYER}-amd64-${VERSION}.tar.zst oci://${TARGET_REPO} + uds zarf package publish build/zarf-package-core-${LAYER}-amd64-${VERSION}.tar.zst oci://${TARGET_REPO} # dont publish arm64 for registry1 since IB images are only amd64 if [ "${FLAVOR}" != "registry1" ]; then - uds zarf package publish build/zarf-package-core-${UDS_LAYER}-arm64-${VERSION}.tar.zst oci://${TARGET_REPO} + uds zarf package publish build/zarf-package-core-${LAYER}-arm64-${VERSION}.tar.zst oci://${TARGET_REPO} fi From 6d0fff3681b780cb5ad5ae0c8833f84d73feaafc Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Wed, 2 Oct 2024 14:12:44 -0600 Subject: [PATCH 08/16] chore: switch publish logic to task v github workflow --- .github/workflows/publish.yaml | 185 ++++++++++------------ packages/identity-authorization/zarf.yaml | 2 +- tasks/publish.yaml | 17 +- 3 files changed, 95 insertions(+), 109 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 56579ecf2..78557e953 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -8,93 +8,99 @@ on: description: 'true - for snapshot release' required: true type: boolean + pull_request: # TODO: TEMP + types: [milestoned, opened, reopened, synchronize] jobs: - publish-uds-core: - strategy: - matrix: - flavor: [upstream, registry1, unicorn] - runs-on: "uds-ubuntu-big-boy-8-core" - name: Publish packages - - permissions: - contents: read - packages: write - id-token: write # This is needed for OIDC federation. - - steps: - - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + # publish-uds-core: + # strategy: + # matrix: + # flavor: [upstream, registry1, unicorn] + # runs-on: "uds-ubuntu-big-boy-8-core" + # name: Publish packages + + # permissions: + # contents: read + # packages: write + # id-token: write # This is needed for OIDC federation. + + # steps: + # - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + + # - name: Environment setup + # uses: ./.github/actions/setup + # with: + # registry1Username: ${{ secrets.IRON_BANK_ROBOT_USERNAME }} + # registry1Password: ${{ secrets.IRON_BANK_ROBOT_PASSWORD }} + # ghToken: ${{ secrets.GITHUB_TOKEN }} + # chainguardIdentity: ${{ secrets.CHAINGUARD_IDENTITY }} + + # - name: (Snapshot) Get snapshot version using git commit short sha and date + # if: ${{ inputs.snapshot }} + # run: | + # SHORT_SHA=$(git rev-parse --short HEAD) + # RELEASE_DATE=$(date +'%Y-%m-%d') + # echo "SNAPSHOT_VERSION=${RELEASE_DATE}-${SHORT_SHA}" >> $GITHUB_ENV + # echo "PUBLISH_ARGS=--set VERSION=${RELEASE_DATE}-${SHORT_SHA}" >> $GITHUB_ENV + + # - name: (Snapshot) Set versions to snapshot + # if: ${{ inputs.snapshot }} + # run: | + # yq -ei '.metadata.version=env(SNAPSHOT_VERSION), (.packages[]|select(has("ref"))|select(.name=="core")).ref=env(SNAPSHOT_VERSION)' bundles/k3d-standard/uds-bundle.yaml + # yq -ei '.metadata.version=env(SNAPSHOT_VERSION), (.packages[]|select(has("ref"))|select(.name=="core-slim-dev")).ref=env(SNAPSHOT_VERSION)' bundles/k3d-slim-dev/uds-bundle.yaml + # yq -ei '.metadata.version=env(SNAPSHOT_VERSION)' packages/standard/zarf.yaml + # yq -ei '.metadata.version=env(SNAPSHOT_VERSION)' packages/slim-dev/zarf.yaml + + # - name: Create Packages and Bundles + # run: | + # ZARF_ARCHITECTURE=amd64 uds run -f tasks/create.yaml standard-package --no-progress --set FLAVOR=${{ matrix.flavor }} + # ZARF_ARCHITECTURE=amd64 uds run -f tasks/create.yaml k3d-standard-bundle --no-progress + # ZARF_ARCHITECTURE=amd64 uds run -f tasks/create.yaml slim-dev-package --no-progress --set FLAVOR=${{ matrix.flavor }} + # ZARF_ARCHITECTURE=amd64 uds run -f tasks/create.yaml k3d-slim-dev-bundle --no-progress + + # if [ "${{ matrix.flavor }}" != "registry1" ]; then + # ZARF_ARCHITECTURE=arm64 uds run -f tasks/create.yaml standard-package --no-progress --set FLAVOR=${{ matrix.flavor }} + # ZARF_ARCHITECTURE=arm64 uds run -f tasks/create.yaml k3d-standard-bundle --no-progress + # ZARF_ARCHITECTURE=arm64 uds run -f tasks/create.yaml slim-dev-package --no-progress --set FLAVOR=${{ matrix.flavor }} + # ZARF_ARCHITECTURE=arm64 uds run -f tasks/create.yaml k3d-slim-dev-bundle --no-progress + # fi + + # # Standard Package by default tests full core + # - name: Test amd64 Bundle + # if: ${{ !inputs.snapshot }} + # run: | + # uds run deploy-standard-bundle --no-progress + # uds run -f tasks/test.yaml validate-packages --no-progress + + # - name: Debug Output + # if: ${{ always() && !inputs.snapshot }} + # uses: ./.github/actions/debug-output + + # # Publish package and bundle to destination repository + # - name: Publish Standard Package + # run: uds run -f tasks/publish.yaml standard-package --set FLAVOR=${{ matrix.flavor }} --set SNAPSHOT="${{ inputs.snapshot }}" ${PUBLISH_ARGS} --no-progress + + # - name: Publish Upstream Flavored Bundles + # if: ${{ matrix.flavor == 'upstream' }} + # run: uds run -f tasks/publish.yaml bundles --set SNAPSHOT="${{ inputs.snapshot }}" ${PUBLISH_ARGS} --no-progress + + # - name: Save logs + # if: always() + # uses: ./.github/actions/save-logs + # with: + # suffix: -${{ matrix.flavor }} - - name: Environment setup - uses: ./.github/actions/setup - with: - registry1Username: ${{ secrets.IRON_BANK_ROBOT_USERNAME }} - registry1Password: ${{ secrets.IRON_BANK_ROBOT_PASSWORD }} - ghToken: ${{ secrets.GITHUB_TOKEN }} - chainguardIdentity: ${{ secrets.CHAINGUARD_IDENTITY }} - - - name: (Snapshot) Get snapshot version using git commit short sha and date - if: ${{ inputs.snapshot }} - run: | - SHORT_SHA=$(git rev-parse --short HEAD) - RELEASE_DATE=$(date +'%Y-%m-%d') - echo "SNAPSHOT_VERSION=${RELEASE_DATE}-${SHORT_SHA}" >> $GITHUB_ENV - echo "PUBLISH_ARGS=--set VERSION=${RELEASE_DATE}-${SHORT_SHA}" >> $GITHUB_ENV - - - name: (Snapshot) Set versions to snapshot - if: ${{ inputs.snapshot }} - run: | - yq -ei '.metadata.version=env(SNAPSHOT_VERSION), (.packages[]|select(has("ref"))|select(.name=="core")).ref=env(SNAPSHOT_VERSION)' bundles/k3d-standard/uds-bundle.yaml - yq -ei '.metadata.version=env(SNAPSHOT_VERSION), (.packages[]|select(has("ref"))|select(.name=="core-slim-dev")).ref=env(SNAPSHOT_VERSION)' bundles/k3d-slim-dev/uds-bundle.yaml - yq -ei '.metadata.version=env(SNAPSHOT_VERSION)' packages/standard/zarf.yaml - yq -ei '.metadata.version=env(SNAPSHOT_VERSION)' packages/slim-dev/zarf.yaml - - - name: Create Packages and Bundles - run: | - ZARF_ARCHITECTURE=amd64 uds run -f tasks/create.yaml standard-package --no-progress --set FLAVOR=${{ matrix.flavor }} - ZARF_ARCHITECTURE=amd64 uds run -f tasks/create.yaml k3d-standard-bundle --no-progress - ZARF_ARCHITECTURE=amd64 uds run -f tasks/create.yaml slim-dev-package --no-progress --set FLAVOR=${{ matrix.flavor }} - ZARF_ARCHITECTURE=amd64 uds run -f tasks/create.yaml k3d-slim-dev-bundle --no-progress - - if [ "${{ matrix.flavor }}" != "registry1" ]; then - ZARF_ARCHITECTURE=arm64 uds run -f tasks/create.yaml standard-package --no-progress --set FLAVOR=${{ matrix.flavor }} - ZARF_ARCHITECTURE=arm64 uds run -f tasks/create.yaml k3d-standard-bundle --no-progress - ZARF_ARCHITECTURE=arm64 uds run -f tasks/create.yaml slim-dev-package --no-progress --set FLAVOR=${{ matrix.flavor }} - ZARF_ARCHITECTURE=arm64 uds run -f tasks/create.yaml k3d-slim-dev-bundle --no-progress - fi - - # Standard Package by default tests full core - - name: Test amd64 Bundle - if: ${{ !inputs.snapshot }} - run: | - uds run deploy-standard-bundle --no-progress - uds run -f tasks/test.yaml validate-packages --no-progress - - - name: Debug Output - if: ${{ always() && !inputs.snapshot }} - uses: ./.github/actions/debug-output - - # Publish package and bundle to destination repository - - name: Publish Standard Package - run: uds run -f tasks/publish.yaml standard-package --set FLAVOR=${{ matrix.flavor }} --set SNAPSHOT="${{ inputs.snapshot }}" ${PUBLISH_ARGS} --no-progress - - - name: Publish Upstream Flavored Bundles - if: ${{ matrix.flavor == 'upstream' }} - run: uds run -f tasks/publish.yaml bundles --set SNAPSHOT="${{ inputs.snapshot }}" ${PUBLISH_ARGS} --no-progress - - - name: Save logs - if: always() - uses: ./.github/actions/save-logs - with: - suffix: -${{ matrix.flavor }} publish-uds-core-layers: if: ${{ !inputs.snapshot }} strategy: matrix: flavor: [upstream, registry1, unicorn] - # maybe handle base as a special case to avoid duplicate building the layer layer: [base, identity-authorization] - runs-on: "ubuntu-latest" + arch: [amd64, arm64] + exclude: + - flavor: registry1 + arch: arm64 + runs-on: ${{ matrix.arch == 'arm64' && 'uds-ubuntu-arm64-4-core' || 'uds-ubuntu-big-boy-4-core' }} name: Publish package layers permissions: @@ -113,32 +119,15 @@ jobs: ghToken: ${{ secrets.GITHUB_TOKEN }} chainguardIdentity: ${{ secrets.CHAINGUARD_IDENTITY }} - - name: Create Package Layers - env: - UDS_LAYER: ${{ matrix.layer }} - run: | - ZARF_ARCHITECTURE=amd64 uds run create:single-layer-callable --no-progress --set FLAVOR=${{ matrix.flavor }} --set LAYER=${{ matrix.layer }} - - if [ "${{ matrix.flavor }}" != "registry1" ]; then - ZARF_ARCHITECTURE=arm64 uds run create:single-layer-callable --no-progress --set FLAVOR=${{ matrix.flavor }} --set LAYER=${{ matrix.layer }} - fi - - # Core package layer test - - name: Test amd64 Bundle - run: | - uds run test:base --set FLAVOR=${{ matrix.flavor }} - uds run deploy:single-layer-callable --set LAYER=${{ matrix.layer }} - uds run -f packages/${{ matrix.layer }}/tasks.yaml validate --no-progress + - name: Test and Publish Core Package Layer + run: uds run -f tasks/publish.yaml single-layer --set FLAVOR=${{ matrix.flavor }} --set LAYER=${{ matrix.layer }} --no-progress - name: Debug Output if: ${{ always() && !inputs.snapshot }} uses: ./.github/actions/debug-output - - name: Publish Core Package Layer - run: uds run -f tasks/publish.yaml single-layer --set FLAVOR=${{ matrix.flavor }} --set LAYER=${{ matrix.layer }} --no-progress - - name: Save logs if: always() uses: ./.github/actions/save-logs with: - suffix: -${{ matrix.layer }} \ No newline at end of file + suffix: -${{ matrix.flavor }}-${{ matrix.layer }} diff --git a/packages/identity-authorization/zarf.yaml b/packages/identity-authorization/zarf.yaml index ae60e60ec..b77e6502b 100644 --- a/packages/identity-authorization/zarf.yaml +++ b/packages/identity-authorization/zarf.yaml @@ -18,4 +18,4 @@ components: - name: authservice required: true import: - path: ../../src/authservice \ No newline at end of file + path: ../../src/authservice diff --git a/tasks/publish.yaml b/tasks/publish.yaml index 43a4d2b78..2379faee4 100644 --- a/tasks/publish.yaml +++ b/tasks/publish.yaml @@ -1,5 +1,6 @@ includes: - utils: utils.yaml + - test: test.yaml variables: - name: FLAVOR @@ -56,15 +57,11 @@ tasks: uds zarf tools registry copy ${pkgPath}:${VERSION} ${pkgPath}:latest - name: single-layer - description: "Publish UDS Core layer" + description: "Test and Publish UDS Core layer" actions: + - task: test:base + - task: test:single-layer + if: ${{ ne .variables.LAYER "base" }} - task: utils:determine-repo - - description: "Publish amd64/arm64 packages per flavor" - cmd: | - echo "Publishing package to ${TARGET_REPO}" - uds zarf package publish build/zarf-package-core-${LAYER}-amd64-${VERSION}.tar.zst oci://${TARGET_REPO} - - # dont publish arm64 for registry1 since IB images are only amd64 - if [ "${FLAVOR}" != "registry1" ]; then - uds zarf package publish build/zarf-package-core-${LAYER}-arm64-${VERSION}.tar.zst oci://${TARGET_REPO} - fi + - description: "Publish build of layer" + cmd: echo "uds zarf package publish build/zarf-package-core-${LAYER}-${UDS_ARCH}-${VERSION}.tar.zst oci://${TARGET_REPO}" From 56ed292253979fd953eae7a482f757f39d360cc3 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Wed, 2 Oct 2024 14:13:45 -0600 Subject: [PATCH 09/16] formatting --- src/pepr/operator/reconcilers/package-reconciler.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pepr/operator/reconcilers/package-reconciler.ts b/src/pepr/operator/reconcilers/package-reconciler.ts index 8bb193b1b..edf64ceb9 100644 --- a/src/pepr/operator/reconcilers/package-reconciler.ts +++ b/src/pepr/operator/reconcilers/package-reconciler.ts @@ -75,7 +75,9 @@ export async function packageReconciler(pkg: UDSPackage) { authserviceClients = await authservice(pkg, ssoClients); } else if (pkg.spec?.sso) { log.error("SSO is not deployed, but the package has SSO configuration"); - throw new Error("Identity & Authorization is not deployed, but the package has SSO configuration"); + throw new Error( + "Identity & Authorization is not deployed, but the package has SSO configuration", + ); } // Create the VirtualService and ServiceEntry for each exposed service From fcc5315a89cd49d554ddf4a69afb48d38b87be66 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Wed, 2 Oct 2024 14:16:51 -0600 Subject: [PATCH 10/16] fix: multiarch uds-cli --- .github/actions/setup/action.yaml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/actions/setup/action.yaml b/.github/actions/setup/action.yaml index 9c5b5040d..dff2d9157 100644 --- a/.github/actions/setup/action.yaml +++ b/.github/actions/setup/action.yaml @@ -14,6 +14,10 @@ inputs: chainguardIdentity: description: "ID for Chainguard Identity" required: true + udsCliVersion: + description: The uds-cli version to install + # renovate: datasource=github-tags depName=defenseunicorns/uds-cli versioning=semver + default: 0.16.0 runs: using: "composite" @@ -28,13 +32,14 @@ runs: # renovate: datasource=github-tags depName=k3d-io/k3d versioning=semver run: curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=v5.7.4 bash - - name: Set up Homebrew - uses: Homebrew/actions/setup-homebrew@master + - name: Set UDS CLI Arch + id: setUdsCliArch + shell: bash + run: INPUT=${{ runner.arch == 'X64' && 'amd64' || runner.arch }}; echo "ARCH=${INPUT,,}" >> "$GITHUB_OUTPUT" - name: Install UDS CLI shell: bash - # renovate: datasource=github-tags depName=defenseunicorns/uds-cli versioning=semver - run: brew install defenseunicorns/tap/uds@0.16.0 + run: curl -o /usr/local/bin/uds -L https://github.com/defenseunicorns/uds-cli/releases/download/v${{ inputs.udsCliVersion }}/uds-cli_v${{ inputs.udsCliVersion }}_Linux_${{steps.setUdsCliArch.outputs.ARCH}} && chmod +x /usr/local/bin/uds - name: Install Lula uses: defenseunicorns/lula-action/setup@badad8c4b1570095f57e66ffd62664847698a3b9 # v0.0.1 From 7999089b2480920e834b443ac401755e3a5e4aa2 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Wed, 2 Oct 2024 14:27:47 -0600 Subject: [PATCH 11/16] fix: lint, slim-dev, suffix --- .github/workflows/publish.yaml | 2 +- packages/base/tasks.yaml | 2 +- packages/identity-authorization/tasks.yaml | 2 +- tasks/create.yaml | 10 ++-------- tasks/test.yaml | 2 +- 5 files changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 78557e953..fcbc198af 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -130,4 +130,4 @@ jobs: if: always() uses: ./.github/actions/save-logs with: - suffix: -${{ matrix.flavor }}-${{ matrix.layer }} + suffix: -${{ matrix.flavor }}-${{ matrix.layer }}-${{ matrix.arch }} diff --git a/packages/base/tasks.yaml b/packages/base/tasks.yaml index f5c835749..1742dcbcf 100644 --- a/packages/base/tasks.yaml +++ b/packages/base/tasks.yaml @@ -6,4 +6,4 @@ tasks: - name: validate actions: - task: istio:validate - - task: pepr:validate \ No newline at end of file + - task: pepr:validate diff --git a/packages/identity-authorization/tasks.yaml b/packages/identity-authorization/tasks.yaml index f3320586c..1f1b92c3c 100644 --- a/packages/identity-authorization/tasks.yaml +++ b/packages/identity-authorization/tasks.yaml @@ -6,4 +6,4 @@ tasks: - name: validate actions: - task: keycloak:validate - - task: authservice:validate \ No newline at end of file + - task: authservice:validate diff --git a/tasks/create.yaml b/tasks/create.yaml index 22c2fd088..6fb1f2260 100644 --- a/tasks/create.yaml +++ b/tasks/create.yaml @@ -20,14 +20,6 @@ tasks: - description: "Create the UDS Core Standard Zarf Package" cmd: "uds zarf package create packages/standard --confirm --no-progress --flavor ${FLAVOR}" - - name: core-base - description: "Create the UDS Core Base Package" - actions: - - task: pepr-build - - - description: "Create the UDS Core Standard Zarf Package" - cmd: "uds run create:single-layer --set FLAVOR=${FLAVOR} --set=layer=base" - - name: k3d-standard-bundle description: "Create the K3d-UDS Core Bundle" actions: @@ -53,6 +45,8 @@ tasks: default: base description: The UDS Core layer to build actions: + - task: pepr-build + if: ${{ eq .inputs.layer "base"}} - cmd: uds zarf package create packages/${{ index .inputs "layer" }} --confirm --no-progress --flavor ${FLAVOR} - name: pepr-build diff --git a/tasks/test.yaml b/tasks/test.yaml index f5a7d82e5..a08680e8a 100644 --- a/tasks/test.yaml +++ b/tasks/test.yaml @@ -27,7 +27,7 @@ tasks: description: "Build and test a single layer, must set UDS_LAYER environment variable" actions: - task: create:single-layer - with: + with: layer: ${LAYER} - task: deploy:single-layer with: From 4446774d4ea5ed3a7157ea08c257656c1e513831 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Wed, 2 Oct 2024 15:08:06 -0600 Subject: [PATCH 12/16] chore: revert workflow --- .github/workflows/publish.yaml | 156 ++++++++++++++++----------------- tasks.yaml | 5 -- tasks/publish.yaml | 2 +- tasks/test.yaml | 10 --- 4 files changed, 78 insertions(+), 95 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index fcbc198af..e5d746cf2 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -8,87 +8,85 @@ on: description: 'true - for snapshot release' required: true type: boolean - pull_request: # TODO: TEMP - types: [milestoned, opened, reopened, synchronize] jobs: - # publish-uds-core: - # strategy: - # matrix: - # flavor: [upstream, registry1, unicorn] - # runs-on: "uds-ubuntu-big-boy-8-core" - # name: Publish packages - - # permissions: - # contents: read - # packages: write - # id-token: write # This is needed for OIDC federation. - - # steps: - # - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - - # - name: Environment setup - # uses: ./.github/actions/setup - # with: - # registry1Username: ${{ secrets.IRON_BANK_ROBOT_USERNAME }} - # registry1Password: ${{ secrets.IRON_BANK_ROBOT_PASSWORD }} - # ghToken: ${{ secrets.GITHUB_TOKEN }} - # chainguardIdentity: ${{ secrets.CHAINGUARD_IDENTITY }} - - # - name: (Snapshot) Get snapshot version using git commit short sha and date - # if: ${{ inputs.snapshot }} - # run: | - # SHORT_SHA=$(git rev-parse --short HEAD) - # RELEASE_DATE=$(date +'%Y-%m-%d') - # echo "SNAPSHOT_VERSION=${RELEASE_DATE}-${SHORT_SHA}" >> $GITHUB_ENV - # echo "PUBLISH_ARGS=--set VERSION=${RELEASE_DATE}-${SHORT_SHA}" >> $GITHUB_ENV - - # - name: (Snapshot) Set versions to snapshot - # if: ${{ inputs.snapshot }} - # run: | - # yq -ei '.metadata.version=env(SNAPSHOT_VERSION), (.packages[]|select(has("ref"))|select(.name=="core")).ref=env(SNAPSHOT_VERSION)' bundles/k3d-standard/uds-bundle.yaml - # yq -ei '.metadata.version=env(SNAPSHOT_VERSION), (.packages[]|select(has("ref"))|select(.name=="core-slim-dev")).ref=env(SNAPSHOT_VERSION)' bundles/k3d-slim-dev/uds-bundle.yaml - # yq -ei '.metadata.version=env(SNAPSHOT_VERSION)' packages/standard/zarf.yaml - # yq -ei '.metadata.version=env(SNAPSHOT_VERSION)' packages/slim-dev/zarf.yaml - - # - name: Create Packages and Bundles - # run: | - # ZARF_ARCHITECTURE=amd64 uds run -f tasks/create.yaml standard-package --no-progress --set FLAVOR=${{ matrix.flavor }} - # ZARF_ARCHITECTURE=amd64 uds run -f tasks/create.yaml k3d-standard-bundle --no-progress - # ZARF_ARCHITECTURE=amd64 uds run -f tasks/create.yaml slim-dev-package --no-progress --set FLAVOR=${{ matrix.flavor }} - # ZARF_ARCHITECTURE=amd64 uds run -f tasks/create.yaml k3d-slim-dev-bundle --no-progress - - # if [ "${{ matrix.flavor }}" != "registry1" ]; then - # ZARF_ARCHITECTURE=arm64 uds run -f tasks/create.yaml standard-package --no-progress --set FLAVOR=${{ matrix.flavor }} - # ZARF_ARCHITECTURE=arm64 uds run -f tasks/create.yaml k3d-standard-bundle --no-progress - # ZARF_ARCHITECTURE=arm64 uds run -f tasks/create.yaml slim-dev-package --no-progress --set FLAVOR=${{ matrix.flavor }} - # ZARF_ARCHITECTURE=arm64 uds run -f tasks/create.yaml k3d-slim-dev-bundle --no-progress - # fi - - # # Standard Package by default tests full core - # - name: Test amd64 Bundle - # if: ${{ !inputs.snapshot }} - # run: | - # uds run deploy-standard-bundle --no-progress - # uds run -f tasks/test.yaml validate-packages --no-progress - - # - name: Debug Output - # if: ${{ always() && !inputs.snapshot }} - # uses: ./.github/actions/debug-output - - # # Publish package and bundle to destination repository - # - name: Publish Standard Package - # run: uds run -f tasks/publish.yaml standard-package --set FLAVOR=${{ matrix.flavor }} --set SNAPSHOT="${{ inputs.snapshot }}" ${PUBLISH_ARGS} --no-progress - - # - name: Publish Upstream Flavored Bundles - # if: ${{ matrix.flavor == 'upstream' }} - # run: uds run -f tasks/publish.yaml bundles --set SNAPSHOT="${{ inputs.snapshot }}" ${PUBLISH_ARGS} --no-progress - - # - name: Save logs - # if: always() - # uses: ./.github/actions/save-logs - # with: - # suffix: -${{ matrix.flavor }} + publish-uds-core: + strategy: + matrix: + flavor: [upstream, registry1, unicorn] + runs-on: "uds-ubuntu-big-boy-8-core" + name: Publish packages + + permissions: + contents: read + packages: write + id-token: write # This is needed for OIDC federation. + + steps: + - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + + - name: Environment setup + uses: ./.github/actions/setup + with: + registry1Username: ${{ secrets.IRON_BANK_ROBOT_USERNAME }} + registry1Password: ${{ secrets.IRON_BANK_ROBOT_PASSWORD }} + ghToken: ${{ secrets.GITHUB_TOKEN }} + chainguardIdentity: ${{ secrets.CHAINGUARD_IDENTITY }} + + - name: (Snapshot) Get snapshot version using git commit short sha and date + if: ${{ inputs.snapshot }} + run: | + SHORT_SHA=$(git rev-parse --short HEAD) + RELEASE_DATE=$(date +'%Y-%m-%d') + echo "SNAPSHOT_VERSION=${RELEASE_DATE}-${SHORT_SHA}" >> $GITHUB_ENV + echo "PUBLISH_ARGS=--set VERSION=${RELEASE_DATE}-${SHORT_SHA}" >> $GITHUB_ENV + + - name: (Snapshot) Set versions to snapshot + if: ${{ inputs.snapshot }} + run: | + yq -ei '.metadata.version=env(SNAPSHOT_VERSION), (.packages[]|select(has("ref"))|select(.name=="core")).ref=env(SNAPSHOT_VERSION)' bundles/k3d-standard/uds-bundle.yaml + yq -ei '.metadata.version=env(SNAPSHOT_VERSION), (.packages[]|select(has("ref"))|select(.name=="core-slim-dev")).ref=env(SNAPSHOT_VERSION)' bundles/k3d-slim-dev/uds-bundle.yaml + yq -ei '.metadata.version=env(SNAPSHOT_VERSION)' packages/standard/zarf.yaml + yq -ei '.metadata.version=env(SNAPSHOT_VERSION)' packages/slim-dev/zarf.yaml + + - name: Create Packages and Bundles + run: | + ZARF_ARCHITECTURE=amd64 uds run -f tasks/create.yaml standard-package --no-progress --set FLAVOR=${{ matrix.flavor }} + ZARF_ARCHITECTURE=amd64 uds run -f tasks/create.yaml k3d-standard-bundle --no-progress + ZARF_ARCHITECTURE=amd64 uds run -f tasks/create.yaml slim-dev-package --no-progress --set FLAVOR=${{ matrix.flavor }} + ZARF_ARCHITECTURE=amd64 uds run -f tasks/create.yaml k3d-slim-dev-bundle --no-progress + + if [ "${{ matrix.flavor }}" != "registry1" ]; then + ZARF_ARCHITECTURE=arm64 uds run -f tasks/create.yaml standard-package --no-progress --set FLAVOR=${{ matrix.flavor }} + ZARF_ARCHITECTURE=arm64 uds run -f tasks/create.yaml k3d-standard-bundle --no-progress + ZARF_ARCHITECTURE=arm64 uds run -f tasks/create.yaml slim-dev-package --no-progress --set FLAVOR=${{ matrix.flavor }} + ZARF_ARCHITECTURE=arm64 uds run -f tasks/create.yaml k3d-slim-dev-bundle --no-progress + fi + + # Standard Package by default tests full core + - name: Test amd64 Bundle + if: ${{ !inputs.snapshot }} + run: | + uds run deploy-standard-bundle --no-progress + uds run -f tasks/test.yaml validate-packages --no-progress + + - name: Debug Output + if: ${{ always() && !inputs.snapshot }} + uses: ./.github/actions/debug-output + + # Publish package and bundle to destination repository + - name: Publish Standard Package + run: uds run -f tasks/publish.yaml standard-package --set FLAVOR=${{ matrix.flavor }} --set SNAPSHOT="${{ inputs.snapshot }}" ${PUBLISH_ARGS} --no-progress + + - name: Publish Upstream Flavored Bundles + if: ${{ matrix.flavor == 'upstream' }} + run: uds run -f tasks/publish.yaml bundles --set SNAPSHOT="${{ inputs.snapshot }}" ${PUBLISH_ARGS} --no-progress + + - name: Save logs + if: always() + uses: ./.github/actions/save-logs + with: + suffix: -${{ matrix.flavor }} publish-uds-core-layers: if: ${{ !inputs.snapshot }} diff --git a/tasks.yaml b/tasks.yaml index 280457057..df3526f23 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -97,11 +97,6 @@ tasks: actions: - task: deploy:k3d-standard-bundle - - name: test-single-package - description: "Build and test a single package, must set UDS_PKG environment variable" - actions: - - task: test:single-package - - name: test-uds-core description: "Build and test UDS Core" actions: diff --git a/tasks/publish.yaml b/tasks/publish.yaml index 2379faee4..553f386c0 100644 --- a/tasks/publish.yaml +++ b/tasks/publish.yaml @@ -64,4 +64,4 @@ tasks: if: ${{ ne .variables.LAYER "base" }} - task: utils:determine-repo - description: "Publish build of layer" - cmd: echo "uds zarf package publish build/zarf-package-core-${LAYER}-${UDS_ARCH}-${VERSION}.tar.zst oci://${TARGET_REPO}" + cmd: uds zarf package publish build/zarf-package-core-${LAYER}-${UDS_ARCH}-${VERSION}.tar.zst oci://${TARGET_REPO} diff --git a/tasks/test.yaml b/tasks/test.yaml index a08680e8a..90078cfd4 100644 --- a/tasks/test.yaml +++ b/tasks/test.yaml @@ -6,16 +6,6 @@ includes: - base-layer: ../packages/base/tasks.yaml tasks: - - name: single-package - description: "Build and test a single package, must set UDS_PKG environment variable" - actions: - - task: create:single-package - - task: setup:k3d-test-cluster - - task: deploy:single-package - - - description: "Validate the package" - cmd: uds run -f src/${UDS_PKG}/tasks.yaml validate --no-progress - - name: base description: "Build and test the base layer" actions: From ad00ec10c57d633dc16eec97dac480c17e4d9983 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Wed, 2 Oct 2024 15:29:30 -0600 Subject: [PATCH 13/16] fix: filter logic --- .github/filters.yaml | 12 ++---------- .github/workflows/pull-request-conditionals.yaml | 1 - .github/workflows/slim-dev-test.yaml | 4 +++- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/.github/filters.yaml b/.github/filters.yaml index 75e12a0b3..2d06cbf63 100644 --- a/.github/filters.yaml +++ b/.github/filters.yaml @@ -2,19 +2,11 @@ all: - "**" base: + - "packages/base/**" - "src/istio/**" - "src/pepr/**" - - "!**/*.md" - - "!**/*.jpg" - - "!**/*.png" - - "!**/*.gif" - - "!**/*.svg" identity-authorization: + - "packages/identity-authorization/**" - "src/keycloak/**" - "src/authservice/**" - - "!**/*.md" - - "!**/*.jpg" - - "!**/*.png" - - "!**/*.gif" - - "!**/*.svg" diff --git a/.github/workflows/pull-request-conditionals.yaml b/.github/workflows/pull-request-conditionals.yaml index c5ccf2adf..cc178e911 100644 --- a/.github/workflows/pull-request-conditionals.yaml +++ b/.github/workflows/pull-request-conditionals.yaml @@ -64,7 +64,6 @@ jobs: uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 with: filters: .github/filters.yaml - predicate-quantifier: every # This job triggers a separate workflow for each changed source package, if any. run-package-test: diff --git a/.github/workflows/slim-dev-test.yaml b/.github/workflows/slim-dev-test.yaml index a3b09ee32..e6c94746a 100644 --- a/.github/workflows/slim-dev-test.yaml +++ b/.github/workflows/slim-dev-test.yaml @@ -8,9 +8,11 @@ on: paths: - src/pepr/** - src/keycloak/** + - src/authservice/** - src/istio/** - src/prometheus-stack/** - - packages/slim-dev/** + - packages/base/** + - packages/identity-authorization/** - bundles/k3d-slim-dev/** - .github/workflows/slim-dev** - "!**/*.md" From a235215205408d22eccbccb516c7ce37c1922c24 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Wed, 2 Oct 2024 16:46:03 -0600 Subject: [PATCH 14/16] chore: dev-setup fix, layers cleanup, ... --- .github/workflows/compliance.yaml | 2 -- README.md | 10 +++++----- docs/deployment/uds-deploy.md | 6 ++---- src/istio/common/zarf.yaml | 5 +++++ src/istio/values/registry1-values.yaml | 4 ++-- src/istio/values/unicorn-values.yaml | 4 ++-- src/istio/values/upstream-values.yaml | 4 ++-- src/pepr/logger.ts | 1 + src/pepr/operator/index.ts | 7 +++++++ src/velero/README.md | 20 +++----------------- tasks.yaml | 18 ++++++------------ 11 files changed, 35 insertions(+), 46 deletions(-) diff --git a/.github/workflows/compliance.yaml b/.github/workflows/compliance.yaml index eb4f6598c..c9649a18c 100644 --- a/.github/workflows/compliance.yaml +++ b/.github/workflows/compliance.yaml @@ -25,8 +25,6 @@ jobs: runs-on: ubuntu-latest name: Evaluate continue-on-error: true - # env: - # UDS_PKG: ${{ inputs.package }} steps: # Used to execute the uds run command - name: Checkout repository diff --git a/README.md b/README.md index c139b7387..8bc9719fc 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ uds deploy k3d-core-slim-dev:0.28.0 #### Developing UDS Core -UDS Core development leverages the `uds zarf dev deploy` command. For convenience, a UDS Task is provided to setup the environment. You'll need to have [NodeJS](https://nodejs.org/en/download/) 20 or later installed to continue. Here's an example of a flow developing the [metrics-server package](./src/metrics-server/README.md): +UDS Core development leverages the `uds zarf dev deploy` command. For convenience, a UDS Task is provided to setup the environment. You'll need to have [NodeJS](https://nodejs.org/en/download/) 20 or later installed to continue. Here's an example of a flow developing the [identity-authorization layer](./package/identity-authorization/README.md): ```bash # Create the dev environment @@ -90,8 +90,8 @@ npx pepr dev # If not developing the Pepr module (can be run multiple times): npx pepr deploy -# Deploy the package (can be run multiple times) -uds run dev-deploy --set PKG=metrics-server +# Deploy the layer (can be run multiple times) +uds run dev-deploy --set LAYER=identity-authorization ``` #### Testing UDS Core @@ -102,10 +102,10 @@ You can perform a complete test of UDS Core by running the following command: uds run test-uds-core ``` -This will create a local k3d cluster, install UDS Core, and run a series of tests against it, the same tests that are run in CI. If you want to run the tests against a specific package, you can use the `PKG` env variable. The following example runs the tests against the metrics-server package: +This will create a local k3d cluster, install UDS Core, and run a series of tests against it, the same tests that are run in CI. If you want to run the tests against a specific core layer, you can use the `LAYER` task variable. The following example runs the tests against the identity-authorization layer: ```bash -UDS_PKG=metrics-server uds run test-single-package +uds run test-single-layer --set LAYER=identity-authorization ``` Note you can specify the `--set FLAVOR=registry1` flag to test using Iron Bank images instead of the upstream images. diff --git a/docs/deployment/uds-deploy.md b/docs/deployment/uds-deploy.md index 97455a7c8..90b05291e 100644 --- a/docs/deployment/uds-deploy.md +++ b/docs/deployment/uds-deploy.md @@ -110,12 +110,10 @@ You can perform a complete test of UDS Core by running the following command: uds run test-uds-core ``` -This command initiates the creation of a local k3d cluster, installs UDS Core, and executes a set of tests identical to those performed in CI. If you wish to run tests targeting a specific package, you can utilize the `PKG` environment variable. - -The example below runs tests against the metrics-server package: +This will create a local k3d cluster, install UDS Core, and run a series of tests against it, the same tests that are run in CI. If you want to run the tests against a specific core layer, you can use the `LAYER` task variable. The following example runs the tests against the identity-authorization layer: ```cli -UDS_PKG=metrics-server uds run test-single-package +uds run test-single-layer --set LAYER=identity-authorization ``` {{% alert-note %}} diff --git a/src/istio/common/zarf.yaml b/src/istio/common/zarf.yaml index 69e10b0f7..bf6eb4911 100644 --- a/src/istio/common/zarf.yaml +++ b/src/istio/common/zarf.yaml @@ -4,6 +4,11 @@ metadata: description: "UDS Core Istio Common" url: https://istio.io/latest/ +variables: + - name: ISTIO_ZARF_REGISTRY + description: Allow registry override for dev deploy, workaround for https://github.com/zarf-dev/zarf/issues/2713 + default: "###ZARF_REGISTRY###" + components: - name: istio-controlplane required: true diff --git a/src/istio/values/registry1-values.yaml b/src/istio/values/registry1-values.yaml index 37921bf2c..750b83a9b 100644 --- a/src/istio/values/registry1-values.yaml +++ b/src/istio/values/registry1-values.yaml @@ -3,7 +3,7 @@ pilot: global: proxy_init: # renovate: image=registry1.dso.mil/ironbank/tetrate/istio/proxyv2 - image: "###ZARF_REGISTRY###/ironbank/tetrate/istio/proxyv2:1.23.2-tetratefips-v0" + image: "###ZARF_VAR_ISTIO_ZARF_REGISTRY###/ironbank/tetrate/istio/proxyv2:1.23.2-tetratefips-v0" proxy: # renovate: image=registry1.dso.mil/ironbank/tetrate/istio/proxyv2 - image: "###ZARF_REGISTRY###/ironbank/tetrate/istio/proxyv2:1.23.2-tetratefips-v0" + image: "###ZARF_VAR_ISTIO_ZARF_REGISTRY###/ironbank/tetrate/istio/proxyv2:1.23.2-tetratefips-v0" diff --git a/src/istio/values/unicorn-values.yaml b/src/istio/values/unicorn-values.yaml index 28c091285..68f715901 100644 --- a/src/istio/values/unicorn-values.yaml +++ b/src/istio/values/unicorn-values.yaml @@ -3,7 +3,7 @@ pilot: global: proxy_init: # renovate: image=cgr.dev/du-uds-defenseunicorns/istio-proxy-fips - image: "###ZARF_REGISTRY###/du-uds-defenseunicorns/istio-proxy-fips:1.23.2" + image: "###ZARF_VAR_ISTIO_ZARF_REGISTRY###/du-uds-defenseunicorns/istio-proxy-fips:1.23.2" proxy: # renovate: image=cgr.dev/du-uds-defenseunicorns/istio-proxy-fips - image: "###ZARF_REGISTRY###/du-uds-defenseunicorns/istio-proxy-fips:1.23.2" + image: "###ZARF_VAR_ISTIO_ZARF_REGISTRY###/du-uds-defenseunicorns/istio-proxy-fips:1.23.2" diff --git a/src/istio/values/upstream-values.yaml b/src/istio/values/upstream-values.yaml index 63b88c989..5df6c66af 100644 --- a/src/istio/values/upstream-values.yaml +++ b/src/istio/values/upstream-values.yaml @@ -3,7 +3,7 @@ pilot: global: proxy_init: # renovate: image=docker.io/istio/proxyv2 - image: "###ZARF_REGISTRY###/istio/proxyv2:1.23.2-distroless" + image: "###ZARF_VAR_ISTIO_ZARF_REGISTRY###/istio/proxyv2:1.23.2-distroless" proxy: # renovate: image=docker.io/istio/proxyv2 - image: "###ZARF_REGISTRY###/istio/proxyv2:1.23.2-distroless" + image: "###ZARF_VAR_ISTIO_ZARF_REGISTRY###/istio/proxyv2:1.23.2-distroless" diff --git a/src/pepr/logger.ts b/src/pepr/logger.ts index 8f505faed..b40eccb4a 100644 --- a/src/pepr/logger.ts +++ b/src/pepr/logger.ts @@ -4,6 +4,7 @@ export enum Component { STARTUP = "startup", CONFIG = "config", ISTIO = "istio", + OPERATOR = "operator", OPERATOR_EXEMPTIONS = "operator.exemptions", OPERATOR_ISTIO = "operator.istio", OPERATOR_KEYCLOAK = "operator.keycloak", diff --git a/src/pepr/operator/index.ts b/src/pepr/operator/index.ts index 33ac7af40..43f629b5a 100644 --- a/src/pepr/operator/index.ts +++ b/src/pepr/operator/index.ts @@ -17,6 +17,7 @@ import { validator } from "./crd/validators/package-validator"; // Reconciler imports import { UDSConfig } from "../config"; +import { Component, setupLogger } from "../logger"; import { purgeAuthserviceClients } from "./controllers/keycloak/authservice/authservice"; import { exemptValidator } from "./crd/validators/exempt-validator"; import { packageReconciler } from "./reconcilers/package-reconciler"; @@ -24,6 +25,8 @@ import { packageReconciler } from "./reconcilers/package-reconciler"; // Export the operator capability for registration in the root pepr.ts export { operator } from "./common"; +const log = setupLogger(Component.OPERATOR); + // Pre-populate the API server CIDR since we are not persisting the EndpointSlice // Note ignore any errors since the watch will still be running hereafter void initAPIServerCIDR(); @@ -68,13 +71,17 @@ When(UDSExemption).IsCreatedOrUpdated().Validate(exemptValidator); // Watch for Functional Layers and update config When(UDSPackage) .IsCreatedOrUpdated() + .InNamespace("keycloak") .WithName("keycloak") .Watch(() => { + // todo: wait for keycloak and authservice to be running? + log.info("Identity and Authorization layer deployed, operator configured to handle SSO."); UDSConfig.isIdentityDeployed = true; }); When(UDSPackage) .IsDeleted() .WithName("keycloak") .Watch(() => { + log.info("Identity and Authorization layer removed, operator will NOT handle SSO."); UDSConfig.isIdentityDeployed = false; }); diff --git a/src/velero/README.md b/src/velero/README.md index 534761d6a..5d9c08e19 100644 --- a/src/velero/README.md +++ b/src/velero/README.md @@ -75,28 +75,14 @@ This package currently assumes the availability of S3 API compatible object stor ## Deploy -### Build and Deploy Everything locally via UDS tasks +### Build Deploy, and Test locally via UDS tasks -```bash -# build the bundle for testing -UDS_PKG=velero uds run create-single-package - -# setup a k3d test env -uds run setup-test-cluster - -# deploy the bundle -UDS_PKG=velero uds run deploy-single-package -``` - -### Test the package via UDS tasks -Running the following will check that the velero deployment exists in the cluster and attempt to execute a backup: +Velero is included in the backup-restore functional layer (WIP). This layer can be created, deployed, and tested with a single UDS run command: ```bash -uds run -f src/velero/tasks.yaml validate +uds run test-single-layer --set LAYER=backup-restore ``` -> Alternatively, you can combine package creation, cluster setup, package deploy and the test command with a simple `UDS_PKG=velero uds run test-single-package` - ## Manually trigger the default backup for testing purposes ```bash diff --git a/tasks.yaml b/tasks.yaml index 0f6afc1e8..0934a233f 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -2,8 +2,6 @@ variables: - name: FLAVOR default: upstream - - name: PKG - - name: LAYER includes: @@ -24,9 +22,8 @@ tasks: actions: - description: "Create the dev cluster" task: setup:create-k3d-cluster - - description: "Deploy the Istio source package with Zarf Dev" - cmd: "uds zarf dev deploy src/istio --flavor ${FLAVOR} --no-progress" + cmd: "uds zarf dev deploy src/istio --flavor upstream --no-progress --deploy-set ISTIO_ZARF_REGISTRY=docker.io" # Note, this abuses the --flavor flag to only install the CRDs from this package - the "crds-only" flavor is not an explicit flavor of the package - description: "Deploy the Prometheus-Stack source package with Zarf Dev to only install the CRDs" @@ -65,16 +62,13 @@ tasks: - description: "Deploy Pepr" cmd: "npx pepr deploy --confirm" - - description: "Deploy Keycloak" - cmd: "uds run dev-deploy --set PKG=keycloak" - - - description: "Deploy Authservice" - cmd: "uds run dev-deploy --set PKG=authservice" + - description: "Deploy Keycloak + Authservice" + cmd: "uds run dev-deploy --set LAYER=identity-authorization --no-progress" - name: dev-deploy - description: "Deploy the given source package with Zarf Dev" + description: "Deploy the given core layer with Zarf Dev" actions: - - cmd: "uds zarf dev deploy src/${PKG} --flavor ${FLAVOR}" + - cmd: "uds zarf dev deploy packages/${LAYER} --flavor ${FLAVOR} --no-progress" - name: setup-cluster description: "Create a k3d Cluster and Initialize with Zarf" @@ -87,7 +81,7 @@ tasks: - task: create:standard-package - name: test-single-layer - description: "Deploy Pepr Module and a Zarf Package using UDS_PKG environment variable" + description: "Deploy Pepr Module and a Zarf Package using LAYER variable" actions: - task: test:base - task: test:single-layer From dadcdb461fbab1e46c7792c2aff507908035ea70 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Thu, 3 Oct 2024 10:27:03 -0600 Subject: [PATCH 15/16] chore: revert dev-setup changes --- src/istio/common/zarf.yaml | 5 ----- src/istio/values/registry1-values.yaml | 4 ++-- src/istio/values/unicorn-values.yaml | 4 ++-- src/istio/values/upstream-values.yaml | 4 ++-- tasks.yaml | 5 ++++- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/istio/common/zarf.yaml b/src/istio/common/zarf.yaml index bf6eb4911..69e10b0f7 100644 --- a/src/istio/common/zarf.yaml +++ b/src/istio/common/zarf.yaml @@ -4,11 +4,6 @@ metadata: description: "UDS Core Istio Common" url: https://istio.io/latest/ -variables: - - name: ISTIO_ZARF_REGISTRY - description: Allow registry override for dev deploy, workaround for https://github.com/zarf-dev/zarf/issues/2713 - default: "###ZARF_REGISTRY###" - components: - name: istio-controlplane required: true diff --git a/src/istio/values/registry1-values.yaml b/src/istio/values/registry1-values.yaml index 750b83a9b..37921bf2c 100644 --- a/src/istio/values/registry1-values.yaml +++ b/src/istio/values/registry1-values.yaml @@ -3,7 +3,7 @@ pilot: global: proxy_init: # renovate: image=registry1.dso.mil/ironbank/tetrate/istio/proxyv2 - image: "###ZARF_VAR_ISTIO_ZARF_REGISTRY###/ironbank/tetrate/istio/proxyv2:1.23.2-tetratefips-v0" + image: "###ZARF_REGISTRY###/ironbank/tetrate/istio/proxyv2:1.23.2-tetratefips-v0" proxy: # renovate: image=registry1.dso.mil/ironbank/tetrate/istio/proxyv2 - image: "###ZARF_VAR_ISTIO_ZARF_REGISTRY###/ironbank/tetrate/istio/proxyv2:1.23.2-tetratefips-v0" + image: "###ZARF_REGISTRY###/ironbank/tetrate/istio/proxyv2:1.23.2-tetratefips-v0" diff --git a/src/istio/values/unicorn-values.yaml b/src/istio/values/unicorn-values.yaml index 68f715901..28c091285 100644 --- a/src/istio/values/unicorn-values.yaml +++ b/src/istio/values/unicorn-values.yaml @@ -3,7 +3,7 @@ pilot: global: proxy_init: # renovate: image=cgr.dev/du-uds-defenseunicorns/istio-proxy-fips - image: "###ZARF_VAR_ISTIO_ZARF_REGISTRY###/du-uds-defenseunicorns/istio-proxy-fips:1.23.2" + image: "###ZARF_REGISTRY###/du-uds-defenseunicorns/istio-proxy-fips:1.23.2" proxy: # renovate: image=cgr.dev/du-uds-defenseunicorns/istio-proxy-fips - image: "###ZARF_VAR_ISTIO_ZARF_REGISTRY###/du-uds-defenseunicorns/istio-proxy-fips:1.23.2" + image: "###ZARF_REGISTRY###/du-uds-defenseunicorns/istio-proxy-fips:1.23.2" diff --git a/src/istio/values/upstream-values.yaml b/src/istio/values/upstream-values.yaml index 5df6c66af..63b88c989 100644 --- a/src/istio/values/upstream-values.yaml +++ b/src/istio/values/upstream-values.yaml @@ -3,7 +3,7 @@ pilot: global: proxy_init: # renovate: image=docker.io/istio/proxyv2 - image: "###ZARF_VAR_ISTIO_ZARF_REGISTRY###/istio/proxyv2:1.23.2-distroless" + image: "###ZARF_REGISTRY###/istio/proxyv2:1.23.2-distroless" proxy: # renovate: image=docker.io/istio/proxyv2 - image: "###ZARF_VAR_ISTIO_ZARF_REGISTRY###/istio/proxyv2:1.23.2-distroless" + image: "###ZARF_REGISTRY###/istio/proxyv2:1.23.2-distroless" diff --git a/tasks.yaml b/tasks.yaml index 0934a233f..e2013be78 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -22,8 +22,11 @@ tasks: actions: - description: "Create the dev cluster" task: setup:create-k3d-cluster + + # Note: This currently is broken until https://github.com/zarf-dev/zarf/issues/2713 is resolved + # As a workaround you can edit the `src/istio/values/upstream-values.yaml` file to change ###ZARF_REGISTRY### to docker.io before running - description: "Deploy the Istio source package with Zarf Dev" - cmd: "uds zarf dev deploy src/istio --flavor upstream --no-progress --deploy-set ISTIO_ZARF_REGISTRY=docker.io" + cmd: "uds zarf dev deploy src/istio --flavor upstream --no-progress" # Note, this abuses the --flavor flag to only install the CRDs from this package - the "crds-only" flavor is not an explicit flavor of the package - description: "Deploy the Prometheus-Stack source package with Zarf Dev to only install the CRDs" From f8a373b21f5591faf35b35c4bdfe2503d2f372a6 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Thu, 3 Oct 2024 11:26:21 -0600 Subject: [PATCH 16/16] chore: use uds cli action --- .github/actions/setup/action.yaml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/actions/setup/action.yaml b/.github/actions/setup/action.yaml index dff2d9157..62b2f0d65 100644 --- a/.github/actions/setup/action.yaml +++ b/.github/actions/setup/action.yaml @@ -14,10 +14,6 @@ inputs: chainguardIdentity: description: "ID for Chainguard Identity" required: true - udsCliVersion: - description: The uds-cli version to install - # renovate: datasource=github-tags depName=defenseunicorns/uds-cli versioning=semver - default: 0.16.0 runs: using: "composite" @@ -32,14 +28,11 @@ runs: # renovate: datasource=github-tags depName=k3d-io/k3d versioning=semver run: curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=v5.7.4 bash - - name: Set UDS CLI Arch - id: setUdsCliArch - shell: bash - run: INPUT=${{ runner.arch == 'X64' && 'amd64' || runner.arch }}; echo "ARCH=${INPUT,,}" >> "$GITHUB_OUTPUT" - - name: Install UDS CLI - shell: bash - run: curl -o /usr/local/bin/uds -L https://github.com/defenseunicorns/uds-cli/releases/download/v${{ inputs.udsCliVersion }}/uds-cli_v${{ inputs.udsCliVersion }}_Linux_${{steps.setUdsCliArch.outputs.ARCH}} && chmod +x /usr/local/bin/uds + uses: defenseunicorns/setup-uds@b987a32bac3baeb67bfb08f5e1544e2f9076ee8a # v1.0.0 + with: + # renovate: datasource=github-tags depName=defenseunicorns/uds-cli versioning=semver + version: v0.16.0 - name: Install Lula uses: defenseunicorns/lula-action/setup@badad8c4b1570095f57e66ffd62664847698a3b9 # v0.0.1