From a3e55a3fb6d2bdbb96f03cfb354835800184dd64 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Mon, 1 Jul 2024 10:32:27 -0700 Subject: [PATCH 01/15] Experimenting with passing matrix parameters and outputs from previous set of matrix jobs I need a way to reuse the tags, image names, contexts which are parameters used for building the image. Currently these are available in the matrix jobs but I need to access these in the subsequent jobs so that I can rebuild images from cache. Using custom github action for this: Write outputs: https://github.com/cloudposse/github-action-matrix-outputs-write Read outputs: https://github.com/cloudposse/github-action-matrix-outputs-read --- .github/workflows/cicd.yaml | 39 ++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 88db314c..774b878e 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -103,18 +103,55 @@ jobs: cache-from: type=gha,scope=${{ matrix.image_name }} cache-to: type=gha,mode=max,scope=${{ matrix.image_name }} + - name: Write Matrix Build Parameters to Output + uses: cloudposse/github-action-matrix-outputs-write@v1 + id: matrix-write + with: + matrix-step-name: ${{ github.job }} + matrix-key: ${{ matrix.image_name }} + outputs: |- + host_namespace: ${{ matrix.host_namespace }} + image: ${{ matrix.image_name }} + context: ${{ matrix.context }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + + ## Read matrix outputs + read-matrix-build-parameters: + runs-on: ubuntu-latest + needs: [docker-build-and-push-images] + steps: + - uses: cloudposse/github-action-matrix-outputs-read@v1 + id: matrix-read + with: + matrix-step-name: docker-build-and-push-images + + outputs: + result: "${{ steps.matrix-read.outputs.result }}" + + run-demo-automated-test: - needs: docker-build-and-push-images + needs: [docker-build-and-push-images, read-matrix-build-parameters] runs-on: ubuntu-latest + env: + MATRIX_PARAMETERS: ${{ fromJson(needs.read-matrix-build-parameters.outputs.result) }} + steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 + + - name: Print Matrix Build Parameters + run: | + echo "Entire matrix parameters: ${{ env.MATRIX_PARAMETERS }}" + - name: List Docker images run: docker images - name: Run automated test script + timeout-minutes: 30 run: bash demo-automated-testing.sh From d51b117416cbd05a3653ebe0e56a9be2f4c65ecd Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Mon, 1 Jul 2024 10:43:44 -0700 Subject: [PATCH 02/15] Update cicd.yaml Got an error but values are being passed between steps of matrix jobs. Perhaps missing double quotes was the problem, let's see if that resolves it. ------ Got this error: Error: Outputs should be valid YAML --------------------- host_namespace: ghcr.io/everest/everest-demo image: mqtt-server context: ./mosquitto tags: ghcr.io/everest/everest-demo/mqtt-server:0.0.16 ghcr.io/everest/everest-demo/mqtt-server:latest labels: org.opencontainers.image.created=2024-07-01T17:35:09.789Z --------------------- YAMLParseError: Implicit keys need to be on a single line at line 5, column 1: tags: ghcr.io/everest/everest-demo/mqtt-server:0.0.16 ghcr.io/everest/everest-demo/mqtt-server:latest ^ ---------- --- .github/workflows/cicd.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 774b878e..82d975a8 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -110,11 +110,11 @@ jobs: matrix-step-name: ${{ github.job }} matrix-key: ${{ matrix.image_name }} outputs: |- - host_namespace: ${{ matrix.host_namespace }} - image: ${{ matrix.image_name }} - context: ${{ matrix.context }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + host_namespace: "${{ matrix.host_namespace }}" + image: "${{ matrix.image_name }}" + context: "${{ matrix.context }}" + tags: "${{ steps.meta.outputs.tags }}" + labels: "${{ steps.meta.outputs.labels }}" ## Read matrix outputs From 1a9ab549f5c6cb2d16b2d3beb35445d97ee45f82 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Mon, 1 Jul 2024 11:09:55 -0700 Subject: [PATCH 03/15] Using block level scalar >- to handle multilines So the quotes weren't the issue actually but the fact that the tags passed from the meta step were multiline values. Hence using block level scalar >- to let YAML interpret them as multi-line commands. --- .github/workflows/cicd.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 82d975a8..9adad601 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -109,12 +109,14 @@ jobs: with: matrix-step-name: ${{ github.job }} matrix-key: ${{ matrix.image_name }} - outputs: |- - host_namespace: "${{ matrix.host_namespace }}" - image: "${{ matrix.image_name }}" - context: "${{ matrix.context }}" - tags: "${{ steps.meta.outputs.tags }}" - labels: "${{ steps.meta.outputs.labels }}" + outputs: | + host_namespace: ${{ matrix.host_namespace }} + image: ${{ matrix.image_name }} + context: ${{ matrix.context }} + tags: >- + ${{ steps.meta.outputs.tags }} + labels: >- + ${{ steps.meta.outputs.labels }} ## Read matrix outputs From d8806a1d904a707550673b0c54e80a9992aa9895 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Mon, 1 Jul 2024 11:14:39 -0700 Subject: [PATCH 04/15] Using | instead of >- to attempt to resolve YAML Parser error Still got the same error for multiline values when using >- YAMLParseError: Implicit keys need to be on a single line at line 6, column 1: ghcr.io/everest/everest-demo/mqtt-server:0.0.16 ghcr.io/everest/everest-demo/mqtt-server:latest ^ --- .github/workflows/cicd.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 9adad601..d7a73912 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -113,9 +113,9 @@ jobs: host_namespace: ${{ matrix.host_namespace }} image: ${{ matrix.image_name }} context: ${{ matrix.context }} - tags: >- + tags: | ${{ steps.meta.outputs.tags }} - labels: >- + labels: | ${{ steps.meta.outputs.labels }} From cc5998f288015db40960e16c21bd05b86bf3c91d Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Mon, 1 Jul 2024 12:30:16 -0700 Subject: [PATCH 05/15] Accessing Tags and Labels via JSON object output of meta action The direct access to tags and labels was giving errors since tags and labels were multiline values whereas the matrix output custom action needed single line values for the YAML parser to correctly parse the parameters. I found that we can access the tags and labels via the json object which is an output from the meta action. So, trying that approach now. ------- Another possible approach that I came across, but didn't try yet. I found that with the meta action, we can specify the delimiter / separator type for tags and labels: https://github.com/docker/metadata-action#inputs While tags can have a CSV datatype in docker build push action, labels can only have a List datatype: https://github.com/docker/build-push-action?tab=readme-ov-file#inputs Hence I believe the aforementioned approach would not work. ------- --- .github/workflows/cicd.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index d7a73912..c5e98635 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -113,10 +113,7 @@ jobs: host_namespace: ${{ matrix.host_namespace }} image: ${{ matrix.image_name }} context: ${{ matrix.context }} - tags: | - ${{ steps.meta.outputs.tags }} - labels: | - ${{ steps.meta.outputs.labels }} + tags_labels: ${{ fromJSON(steps.meta.outputs.json) }} ## Read matrix outputs From 8c82d52999e539079b61b2d91f7d69563c815d7c Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Mon, 1 Jul 2024 13:18:48 -0700 Subject: [PATCH 06/15] Got a YAML / JSON related error again The custom write and read seems to have passed. But error when trying to read data in the env variables part. Error: The template is not valid. .github/workflows/cicd.yaml (Line: 138, Col: 26): A mapping was not expected ----- Now trying to output results directly first, let's see. --- .github/workflows/cicd.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index c5e98635..3ed156cd 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -20,9 +20,9 @@ jobs: strategy: matrix: include: - - host_namespace: ghcr.io/everest/everest-demo - image_name: manager - context: ./manager + # - host_namespace: ghcr.io/everest/everest-demo + # image_name: manager + # context: ./manager - host_namespace: ghcr.io/everest/everest-demo image_name: mqtt-server context: ./mosquitto @@ -126,6 +126,10 @@ jobs: with: matrix-step-name: docker-build-and-push-images + - name: Print matrix read values + run: | + echo "Matrix values result: ${{ steps.matrix-read.outputs.result }}" + outputs: result: "${{ steps.matrix-read.outputs.result }}" From b2895a896996d14f3e623dbc0b12871ecf90be8f Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Mon, 1 Jul 2024 13:22:57 -0700 Subject: [PATCH 07/15] Tags Labels value not passed correctly via matrix outputs custom write / read action JSON object data is simply passed as "Object" for tags_labels. Expected list of tags and labels. --- .github/workflows/cicd.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 3ed156cd..93c74289 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -115,6 +115,9 @@ jobs: context: ${{ matrix.context }} tags_labels: ${{ fromJSON(steps.meta.outputs.json) }} + - name: Print meta JSON + run: | + echo "meta step JSON output values: ${{ fromJSON(steps.meta.outputs.json) }}" ## Read matrix outputs read-matrix-build-parameters: From a1c4f8d65684ba0b9f07ab8e0ad0cd35ef129b43 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Mon, 1 Jul 2024 13:36:57 -0700 Subject: [PATCH 08/15] Printing out tags and labels --- .github/workflows/cicd.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 93c74289..5f94b036 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -118,6 +118,8 @@ jobs: - name: Print meta JSON run: | echo "meta step JSON output values: ${{ fromJSON(steps.meta.outputs.json) }}" + echo "meta step JSON tags values: ${{ fromJSON(steps.meta.outputs.json).tags }}" + echo "meta step JSON labels values: ${{ fromJSON(steps.meta.outputs.json).labels }}" ## Read matrix outputs read-matrix-build-parameters: From 9b029822b14f886989a59a5922308faed6dcb3cb Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Mon, 1 Jul 2024 13:41:31 -0700 Subject: [PATCH 09/15] Print using fromJSON --- .github/workflows/cicd.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 5f94b036..2bb6b680 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -134,6 +134,7 @@ jobs: - name: Print matrix read values run: | echo "Matrix values result: ${{ steps.matrix-read.outputs.result }}" + echo "FromJSON Matrix values result: ${{ fromJson(steps.matrix-read.outputs.result) }}" outputs: result: "${{ steps.matrix-read.outputs.result }}" From fc23388695249f885850313f99ac294faa50f406 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Mon, 1 Jul 2024 13:43:21 -0700 Subject: [PATCH 10/15] ENV vars removed fromJSON --- .github/workflows/cicd.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 2bb6b680..80201f13 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -144,8 +144,9 @@ jobs: needs: [docker-build-and-push-images, read-matrix-build-parameters] runs-on: ubuntu-latest - env: - MATRIX_PARAMETERS: ${{ fromJson(needs.read-matrix-build-parameters.outputs.result) }} + # env: + # MATRIX_PARAMETERS: ${{ fromJson(needs.read-matrix-build-parameters.outputs.result) }} + MATRIX_PARAMETERS: ${{ needs.read-matrix-build-parameters.outputs.result }} steps: - name: Checkout From c6eeeaa5165c170cd932a43d72952ed02729188f Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Mon, 1 Jul 2024 13:43:21 -0700 Subject: [PATCH 11/15] ENV vars removed fromJSON --- .github/workflows/cicd.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 2bb6b680..c7a1f90d 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -145,7 +145,8 @@ jobs: runs-on: ubuntu-latest env: - MATRIX_PARAMETERS: ${{ fromJson(needs.read-matrix-build-parameters.outputs.result) }} + # MATRIX_PARAMETERS: ${{ fromJson(needs.read-matrix-build-parameters.outputs.result) }} + MATRIX_PARAMETERS: ${{ needs.read-matrix-build-parameters.outputs.result }} steps: - name: Checkout From 450619a6f66e67c812d959642cf7fdef2af7fcb4 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Mon, 1 Jul 2024 13:47:31 -0700 Subject: [PATCH 12/15] Printing different values --- .github/workflows/cicd.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index c7a1f90d..7710e76b 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -134,7 +134,9 @@ jobs: - name: Print matrix read values run: | echo "Matrix values result: ${{ steps.matrix-read.outputs.result }}" - echo "FromJSON Matrix values result: ${{ fromJson(steps.matrix-read.outputs.result) }}" + echo "FromJSON Matrix values result: ${{ fromJson(steps.matrix-read.outputs.result).image }}" + echo "Matrix values image: ${{ steps.matrix-read.outputs.result.image }}" + echo "FromJSON Matrix values image: ${{ fromJson(steps.matrix-read.outputs.result).image }}" outputs: result: "${{ steps.matrix-read.outputs.result }}" From e0025aeada78bfe984f153221a977ad8ab8a8a5e Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Mon, 1 Jul 2024 13:53:36 -0700 Subject: [PATCH 13/15] Removed from JSON from tags_labels in matrix write action --- .github/workflows/cicd.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 7710e76b..8a5b0ca7 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -113,7 +113,8 @@ jobs: host_namespace: ${{ matrix.host_namespace }} image: ${{ matrix.image_name }} context: ${{ matrix.context }} - tags_labels: ${{ fromJSON(steps.meta.outputs.json) }} + tags_labels: ${{ steps.meta.outputs.json }} + # tags_labels: ${{ fromJSON(steps.meta.outputs.json) }} - name: Print meta JSON run: | From fb527a7e8dec8134d48d3b722cb42c2b796194f7 Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Mon, 1 Jul 2024 14:07:51 -0700 Subject: [PATCH 14/15] Added build push action step to build images --- .github/workflows/cicd.yaml | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 8a5b0ca7..aab6e5eb 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -103,6 +103,9 @@ jobs: cache-from: type=gha,scope=${{ matrix.image_name }} cache-to: type=gha,mode=max,scope=${{ matrix.image_name }} + - name: List Docker images + run: docker images + - name: Write Matrix Build Parameters to Output uses: cloudposse/github-action-matrix-outputs-write@v1 id: matrix-write @@ -157,11 +160,30 @@ jobs: with: fetch-depth: 0 - - name: Print Matrix Build Parameters run: | echo "Entire matrix parameters: ${{ env.MATRIX_PARAMETERS }}" + - name: Build and push mqtt-server + uses: docker/build-push-action@v5 + with: + context: ${{ env.MATRIX_PARAMETERS.context.mqtt-server }} + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ env.MATRIX_PARAMETERS.tags_labels.mqtt-server.tags }} + labels: ${{ env.MATRIX_PARAMETERS.tags_labels.mqtt-server.labels }} + cache-from: type=gha,scope=${{ env.MATRIX_PARAMETERS.image.mqtt-server }} + cache-to: type=gha,mode=max,scope=${{ env.MATRIX_PARAMETERS.image.mqtt-server }} + + - name: Build and push nodered + uses: docker/build-push-action@v5 + with: + context: ${{ env.MATRIX_PARAMETERS.context.nodered }} + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ env.MATRIX_PARAMETERS.tags_labels.nodered.tags }} + labels: ${{ env.MATRIX_PARAMETERS.tags_labels.nodered.labels }} + cache-from: type=gha,scope=${{ env.MATRIX_PARAMETERS.image.nodered }} + cache-to: type=gha,mode=max,scope=${{ env.MATRIX_PARAMETERS.image.nodered }} + - name: List Docker images run: docker images From 94aebc7c71638728de75774eaa3c91476985d2eb Mon Sep 17 00:00:00 2001 From: "Mahadik, Mukul Chandrakant" Date: Mon, 1 Jul 2024 14:11:10 -0700 Subject: [PATCH 15/15] Printing values --- .github/workflows/cicd.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index aab6e5eb..0dbb29b4 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -163,6 +163,9 @@ jobs: - name: Print Matrix Build Parameters run: | echo "Entire matrix parameters: ${{ env.MATRIX_PARAMETERS }}" + echo "Matrix image: ${{ env.MATRIX_PARAMETERS.image }}" + echo "mqtt-server image: ${{ env.MATRIX_PARAMETERS.image.mqtt-server }}" + echo "nodered image: ${{ env.MATRIX_PARAMETERS.image.nodered }}" - name: Build and push mqtt-server uses: docker/build-push-action@v5