github: backport docker build

See PR#1496 Try to build and push for hub.docker.com In the previous versions, we use "Docker Build Cloud". It is useful by kicking via Web UI to trigger pre-defined build pipeline. Instead, there are limitation of pre-defined build pipelines. The max number of build pipelines is limited to 25. Thus, we can not maintain some target or architecture such as syslog, s3/arm64 and so on. And more, it means that we can't maintain v1.16 stable branch and v1.17 at the same time. Therefore we need to use alternative way - GitHub Actions. To setup actions, set Repository secrets: DOCKER_HUB_ORGS: fluent DOCKER_HUB_USERNAME: someone's account DOCKER_HUB_ACCESS_TOKEN: someone's access token With using GitHub Actions, the following image can be published again: * debian-elasticsearch7 arm64 * debian-logentries arm64 * debian-loggly arm64 * debian-logzio arm64 * debian-s3 arm64 * debian-papertrail * debian-papartrail arm64 Signed-off-by: Kentaro Hayashi <[email protected]>
kenhys · Jun 25, 2024 · c66ef27 · c66ef27
1 parent 798f283
commit c66ef27
Show file tree

Hide file tree

Showing 2 changed files with 208 additions and 0 deletions.
diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
@@ -0,0 +1,205 @@
+name: Publish Docker Image
+
+on:
+  push:
+    branches:
+      - v1.16
+    tags:
+      - v1.*
+
+env:
+  REPOSITORY: ${{ secrets.DOCKER_HUB_ORGS }}/fluentd-kubernetes-daemonset
+jobs:
+  define-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      components: ${{ steps.set-components.outputs.components }}
+    steps:
+      - id: set-components
+        run: |
+          components=$(echo '[
+            "debian-azureblob",
+            "debian-cloudwatch",
+            "debian-elasticsearch7",
+            "debian-elasticsearch8",
+            "debian-forward",
+            "debian-gcs",
+            "debian-graylog",
+            "debian-kafka",
+            "debian-kafka2",
+            "debian-kinesis",
+            "debian-logentries",
+            "debian-loggly",
+            "debian-logzio",
+            "debian-opensearch",
+            "debian-papertrail",
+            "debian-s3",
+            "debian-syslog"
+            ]' | jq -c)
+          echo "components=$components" >> "$GITHUB_OUTPUT"
+  amd64:
+    needs: define-matrix
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        component: ${{ fromJSON(needs.define-matrix.outputs.components) }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          platforms: linux/amd64
+      - uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+      - name: Setup tags
+        run: |
+          set -x
+          component=${{ matrix.component }}
+          echo "CONTEXT=docker-image/v1.16/${component}" >> ${GITHUB_ENV}
+          for target in $(make echo-all-images); do
+            case $target in
+              *$component-amd64*)
+                tags=$(echo $target | cut -d':' -f2-)
+                tag1=$(echo $tags | cut -d',' -f1)
+                tag2=$(echo $tags | cut -d',' -f2)
+                echo "AMD64TAGS=${{ env.REPOSITORY }}:${tag1},${{ env.REPOSITORY }}:${tag2}" >> ${GITHUB_ENV}
+                ;;
+            esac
+          done
+      - name: Build and push for amd64
+        uses: docker/build-push-action@v6
+        with:
+          context: ${{ env.CONTEXT }}
+          provenance: false
+          push: true
+          platforms: linux/amd64
+          tags: ${{ env.AMD64TAGS }}
+          # dare to use old mediatype (application/vnd.docker.distribution.manifest.v2+json)
+          outputs: oci-mediatypes=false
+  arm64:
+    needs: define-matrix
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        component: ${{ fromJSON(needs.define-matrix.outputs.components) }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          platforms: linux/arm64
+      - uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+      - name: Setup tags
+        run: |
+          set -x
+          component=${{ matrix.component }}
+          echo "CONTEXT=docker-image/v1.16/arm64/${component}" >> ${GITHUB_ENV}
+          for target in $(make echo-all-images); do
+            case $target in
+              *$component-arm64*)
+                tags=$(echo $target | cut -d':' -f2-)
+                tag1=$(echo $tags | cut -d',' -f1)
+                tag2=$(echo $tags | cut -d',' -f2)
+                echo "ARM64TAGS=${{ env.REPOSITORY }}:${tag1},${{ env.REPOSITORY }}:${tag2}" >> ${GITHUB_ENV}
+                ;;
+            esac
+          done
+      - name: Build and push for arm64
+        uses: docker/build-push-action@v6
+        with:
+          context: ${{ env.CONTEXT }}
+          provenance: false
+          push: true
+          platforms: linux/arm64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          tags: ${{ env.ARM64TAGS }}
+          # dare to use old mediatype (application/vnd.docker.distribution.manifest.v2+json)
+          outputs: oci-mediatypes=false
+  manifest:
+    needs: [define-matrix, amd64, arm64]
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        component: ${{ fromJSON(needs.define-matrix.outputs.components) }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+      - name: Create manifest
+        run: |
+          component=${{ matrix.component }}
+          for target in $(make echo-all-images); do
+            case $target in
+              *$component-amd64*)
+                tags=$(echo $target | cut -d':' -f2-)
+                tag1=$(echo $tags | cut -d',' -f1)
+                tag2=$(echo $tags | cut -d',' -f2)
+                MULTIARCH_AMD64_TAG=${tag1/amd64-/}
+                MULTIARCH_AMD64_SHORT_TAG=${tag2/amd64-/}
+                AMD64TAG=${tag1}
+                SHORT_AMD64TAG=${tag2}
+                ;;
+              *$component-arm64*)
+                tags=$(echo $target | cut -d':' -f2-)
+                tag1=$(echo $tags | cut -d',' -f1)
+                tag2=$(echo $tags | cut -d',' -f2)
+                MULTIARCH_ARM64_TAG=${tag1/arm64-/}
+                MULTIARCH_ARM64_SHORT_TAG=${tag2/arm64-/}
+                ARM64TAG=${tag1}
+                SHORT_ARM64TAG=${tag2}
+                ;;
+            esac
+          done
+          # v1.xx.x-debian-(component)-xxx
+          if [ ${MULTIARCH_AMD64_TAG} != ${MULTIARCH_ARM64_TAG} ]; then
+            echo "Multiarch tag (v1.xx.x-debian-(component)-xxx) must be same for amd64 and arm64: ${MULTIARCH_AMD64_TAG} !=  ${MULTIARCH_ARM64_TAG}"
+          else
+            docker buildx imagetools create -t ${{ env.REPOSITORY }}:${MULTIARCH_AMD64_TAG} \
+              ${{ env.REPOSITORY }}:${AMD64TAG} \
+              ${{ env.REPOSITORY }}:${ARM64TAG}
+          fi
+          # v1.xx-debian-(component)-xxx
+          if [ ${MULTIARCH_AMD64_SHORT_TAG} != ${MULTIARCH_ARM64_SHORT_TAG} ]; then
+            echo "Multiarch tag (v1.xx-debian-(component)-xxx) must be same for amd64 and arm64: ${MULTIARCH_AMD64_SHORT_TAG} !=  ${MULTIARCH_ARM64_SHORT_TAG}"
+          else
+            docker buildx imagetools create -t ${{ env.REPOSITORY }}:${MULTIARCH_AMD64_SHORT_TAG} \
+              ${{ env.REPOSITORY }}:${SHORT_AMD64TAG} \
+              ${{ env.REPOSITORY }}:${SHORT_ARM64TAG}
+          fi
+          echo "MULTIARCH_AMD64_TAG=${MULTIARCH_AMD64_TAG}" >> ${GITHUB_ENV}
+          echo "MULTIARCH_AMD64_SHORT_TAG=${MULTIARCH_AMD64_SHORT_TAG}" >> ${GITHUB_ENV}
+          echo "AMD64TAG=${AMD64TAG}" >> ${GITHUB_ENV}
+          echo "SHORT_AMD64TAG=${SHORT_ARM64TAG}" >> ${GITHUB_ENV}
+          echo "MULTIARCH_ARM64_TAG=${MULTIARCH_ARM64_TAG}" >> ${GITHUB_ENV}
+          echo "MULTIARCH_ARM64_SHORT_TAG=${MULTIARCH_ARM64_SHORT_TAG}" >> ${GITHUB_ENV}
+          echo "ARM64TAG=${ARM64TAG}" >> ${GITHUB_ENV}
+          echo "SHORT_ARM64TAG=${SHORT_ARM64TAG}" >> ${GITHUB_ENV}
+      - name: Inspect manifest ${{ env.AMD64TAG }}
+        run: docker manifest inspect ${{ env.REPOSITORY }}:${{ env.AMD64TAG }}
+      - name: Inspect manifest ${{ env.ARM64TAG }}
+        run: docker manifest inspect ${{ env.REPOSITORY }}:${{ env.ARM64TAG }}
+      - name: Inspect manifest ${{ env.MULTIARCH_AMD64_TAG }}
+        run: docker manifest inspect ${{ env.REPOSITORY }}:${{ env.MULTIARCH_AMD64_TAG }}
+      - name: Inspect manifest ${{ env.MULTIARCH_AMD64_SHORT_TAG }}
+        run: docker manifest inspect ${{ env.REPOSITORY }}:${{ env.MULTIARCH_AMD64_SHORT_TAG }}
+      - name: Inspect ${{ env.AMD64TAG }} with buildx
+        run: docker buildx imagetools inspect ${{ env.REPOSITORY }}:${{ env.AMD64TAG }}
+      - name: Inspect ${{ env.ARM64TAG }} with buildx
+        run: docker buildx imagetools inspect ${{ env.REPOSITORY }}:${{ env.ARM64TAG }}
+      - name: Inspect ${{ env.MULTIARCH_AMD64_TAG }} with buildx
+        run: docker buildx imagetools inspect ${{ env.REPOSITORY }}:${{ env.MULTIARCH_AMD64_TAG }}
+      - name: Inspect ${{ env.MULTIARCH_AMD64_SHORT_TAG }} with buildx
+        run: docker buildx imagetools inspect ${{ env.REPOSITORY }}:${{ env.MULTIARCH_AMD64_SHORT_TAG }}
diff --git a/Makefile b/Makefile
@@ -83,6 +83,9 @@ eq = $(if $(or $(1),$(2)),$(and $(findstring $(1),$(2)),\
 
 no-cache-arg = $(if $(call eq, $(no-cache), yes), --no-cache, $(empty))
 
+echo-all-images:
+	@echo $(ALL_IMAGES)
+
 # Build Docker image.
 #
 # Usage: