From c66ef27a2498d1f05db7a632f92de7e1d531823c Mon Sep 17 00:00:00 2001 From: Kentaro Hayashi Date: Wed, 19 Jun 2024 16:11:15 +0900 Subject: [PATCH] github: backport docker build See PR#1496 Try to build and push for hub.docker.com In the previous versions, we use "Docker Build Cloud". It is useful by kicking via Web UI to trigger pre-defined build pipeline. Instead, there are limitation of pre-defined build pipelines. The max number of build pipelines is limited to 25. Thus, we can not maintain some target or architecture such as syslog, s3/arm64 and so on. And more, it means that we can't maintain v1.16 stable branch and v1.17 at the same time. Therefore we need to use alternative way - GitHub Actions. To setup actions, set Repository secrets: DOCKER_HUB_ORGS: fluent DOCKER_HUB_USERNAME: someone's account DOCKER_HUB_ACCESS_TOKEN: someone's access token With using GitHub Actions, the following image can be published again: * debian-elasticsearch7 arm64 * debian-logentries arm64 * debian-loggly arm64 * debian-logzio arm64 * debian-s3 arm64 * debian-papertrail * debian-papartrail arm64 Signed-off-by: Kentaro Hayashi --- .github/workflows/docker-build.yml | 205 +++++++++++++++++++++++++++++ Makefile | 3 + 2 files changed, 208 insertions(+) create mode 100644 .github/workflows/docker-build.yml diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml new file mode 100644 index 000000000..8d9cc9cfa --- /dev/null +++ b/.github/workflows/docker-build.yml @@ -0,0 +1,205 @@ +name: Publish Docker Image + +on: + push: + branches: + - v1.16 + tags: + - v1.* + +env: + REPOSITORY: ${{ secrets.DOCKER_HUB_ORGS }}/fluentd-kubernetes-daemonset +jobs: + define-matrix: + runs-on: ubuntu-latest + outputs: + components: ${{ steps.set-components.outputs.components }} + steps: + - id: set-components + run: | + components=$(echo '[ + "debian-azureblob", + "debian-cloudwatch", + "debian-elasticsearch7", + "debian-elasticsearch8", + "debian-forward", + "debian-gcs", + "debian-graylog", + "debian-kafka", + "debian-kafka2", + "debian-kinesis", + "debian-logentries", + "debian-loggly", + "debian-logzio", + "debian-opensearch", + "debian-papertrail", + "debian-s3", + "debian-syslog" + ]' | jq -c) + echo "components=$components" >> "$GITHUB_OUTPUT" + amd64: + needs: define-matrix + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + component: ${{ fromJSON(needs.define-matrix.outputs.components) }} + steps: + - uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + platforms: linux/amd64 + - uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + - name: Setup tags + run: | + set -x + component=${{ matrix.component }} + echo "CONTEXT=docker-image/v1.16/${component}" >> ${GITHUB_ENV} + for target in $(make echo-all-images); do + case $target in + *$component-amd64*) + tags=$(echo $target | cut -d':' -f2-) + tag1=$(echo $tags | cut -d',' -f1) + tag2=$(echo $tags | cut -d',' -f2) + echo "AMD64TAGS=${{ env.REPOSITORY }}:${tag1},${{ env.REPOSITORY }}:${tag2}" >> ${GITHUB_ENV} + ;; + esac + done + - name: Build and push for amd64 + uses: docker/build-push-action@v6 + with: + context: ${{ env.CONTEXT }} + provenance: false + push: true + platforms: linux/amd64 + tags: ${{ env.AMD64TAGS }} + # dare to use old mediatype (application/vnd.docker.distribution.manifest.v2+json) + outputs: oci-mediatypes=false + arm64: + needs: define-matrix + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + component: ${{ fromJSON(needs.define-matrix.outputs.components) }} + steps: + - uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + platforms: linux/arm64 + - uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + - name: Setup tags + run: | + set -x + component=${{ matrix.component }} + echo "CONTEXT=docker-image/v1.16/arm64/${component}" >> ${GITHUB_ENV} + for target in $(make echo-all-images); do + case $target in + *$component-arm64*) + tags=$(echo $target | cut -d':' -f2-) + tag1=$(echo $tags | cut -d',' -f1) + tag2=$(echo $tags | cut -d',' -f2) + echo "ARM64TAGS=${{ env.REPOSITORY }}:${tag1},${{ env.REPOSITORY }}:${tag2}" >> ${GITHUB_ENV} + ;; + esac + done + - name: Build and push for arm64 + uses: docker/build-push-action@v6 + with: + context: ${{ env.CONTEXT }} + provenance: false + push: true + platforms: linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + tags: ${{ env.ARM64TAGS }} + # dare to use old mediatype (application/vnd.docker.distribution.manifest.v2+json) + outputs: oci-mediatypes=false + manifest: + needs: [define-matrix, amd64, arm64] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + component: ${{ fromJSON(needs.define-matrix.outputs.components) }} + steps: + - uses: actions/checkout@v4 + - uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + - name: Create manifest + run: | + component=${{ matrix.component }} + for target in $(make echo-all-images); do + case $target in + *$component-amd64*) + tags=$(echo $target | cut -d':' -f2-) + tag1=$(echo $tags | cut -d',' -f1) + tag2=$(echo $tags | cut -d',' -f2) + MULTIARCH_AMD64_TAG=${tag1/amd64-/} + MULTIARCH_AMD64_SHORT_TAG=${tag2/amd64-/} + AMD64TAG=${tag1} + SHORT_AMD64TAG=${tag2} + ;; + *$component-arm64*) + tags=$(echo $target | cut -d':' -f2-) + tag1=$(echo $tags | cut -d',' -f1) + tag2=$(echo $tags | cut -d',' -f2) + MULTIARCH_ARM64_TAG=${tag1/arm64-/} + MULTIARCH_ARM64_SHORT_TAG=${tag2/arm64-/} + ARM64TAG=${tag1} + SHORT_ARM64TAG=${tag2} + ;; + esac + done + # v1.xx.x-debian-(component)-xxx + if [ ${MULTIARCH_AMD64_TAG} != ${MULTIARCH_ARM64_TAG} ]; then + echo "Multiarch tag (v1.xx.x-debian-(component)-xxx) must be same for amd64 and arm64: ${MULTIARCH_AMD64_TAG} != ${MULTIARCH_ARM64_TAG}" + else + docker buildx imagetools create -t ${{ env.REPOSITORY }}:${MULTIARCH_AMD64_TAG} \ + ${{ env.REPOSITORY }}:${AMD64TAG} \ + ${{ env.REPOSITORY }}:${ARM64TAG} + fi + # v1.xx-debian-(component)-xxx + if [ ${MULTIARCH_AMD64_SHORT_TAG} != ${MULTIARCH_ARM64_SHORT_TAG} ]; then + echo "Multiarch tag (v1.xx-debian-(component)-xxx) must be same for amd64 and arm64: ${MULTIARCH_AMD64_SHORT_TAG} != ${MULTIARCH_ARM64_SHORT_TAG}" + else + docker buildx imagetools create -t ${{ env.REPOSITORY }}:${MULTIARCH_AMD64_SHORT_TAG} \ + ${{ env.REPOSITORY }}:${SHORT_AMD64TAG} \ + ${{ env.REPOSITORY }}:${SHORT_ARM64TAG} + fi + echo "MULTIARCH_AMD64_TAG=${MULTIARCH_AMD64_TAG}" >> ${GITHUB_ENV} + echo "MULTIARCH_AMD64_SHORT_TAG=${MULTIARCH_AMD64_SHORT_TAG}" >> ${GITHUB_ENV} + echo "AMD64TAG=${AMD64TAG}" >> ${GITHUB_ENV} + echo "SHORT_AMD64TAG=${SHORT_ARM64TAG}" >> ${GITHUB_ENV} + echo "MULTIARCH_ARM64_TAG=${MULTIARCH_ARM64_TAG}" >> ${GITHUB_ENV} + echo "MULTIARCH_ARM64_SHORT_TAG=${MULTIARCH_ARM64_SHORT_TAG}" >> ${GITHUB_ENV} + echo "ARM64TAG=${ARM64TAG}" >> ${GITHUB_ENV} + echo "SHORT_ARM64TAG=${SHORT_ARM64TAG}" >> ${GITHUB_ENV} + - name: Inspect manifest ${{ env.AMD64TAG }} + run: docker manifest inspect ${{ env.REPOSITORY }}:${{ env.AMD64TAG }} + - name: Inspect manifest ${{ env.ARM64TAG }} + run: docker manifest inspect ${{ env.REPOSITORY }}:${{ env.ARM64TAG }} + - name: Inspect manifest ${{ env.MULTIARCH_AMD64_TAG }} + run: docker manifest inspect ${{ env.REPOSITORY }}:${{ env.MULTIARCH_AMD64_TAG }} + - name: Inspect manifest ${{ env.MULTIARCH_AMD64_SHORT_TAG }} + run: docker manifest inspect ${{ env.REPOSITORY }}:${{ env.MULTIARCH_AMD64_SHORT_TAG }} + - name: Inspect ${{ env.AMD64TAG }} with buildx + run: docker buildx imagetools inspect ${{ env.REPOSITORY }}:${{ env.AMD64TAG }} + - name: Inspect ${{ env.ARM64TAG }} with buildx + run: docker buildx imagetools inspect ${{ env.REPOSITORY }}:${{ env.ARM64TAG }} + - name: Inspect ${{ env.MULTIARCH_AMD64_TAG }} with buildx + run: docker buildx imagetools inspect ${{ env.REPOSITORY }}:${{ env.MULTIARCH_AMD64_TAG }} + - name: Inspect ${{ env.MULTIARCH_AMD64_SHORT_TAG }} with buildx + run: docker buildx imagetools inspect ${{ env.REPOSITORY }}:${{ env.MULTIARCH_AMD64_SHORT_TAG }} diff --git a/Makefile b/Makefile index 20171ea2a..11472a281 100644 --- a/Makefile +++ b/Makefile @@ -83,6 +83,9 @@ eq = $(if $(or $(1),$(2)),$(and $(findstring $(1),$(2)),\ no-cache-arg = $(if $(call eq, $(no-cache), yes), --no-cache, $(empty)) +echo-all-images: + @echo $(ALL_IMAGES) + # Build Docker image. # # Usage: