Build multiplatform image for amd64 and arm64 architectures on release

RasaHQ · Jul 9, 2024 · 3c18a61 · 3c18a61
1 parent cb4ac02
commit 3c18a61
Show file tree

Hide file tree

Showing 7 changed files with 291 additions and 248 deletions.
diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml
@@ -10,11 +10,18 @@ on:
 
 env:
  IS_TAG_BUILD: ${{ startsWith(github.event.ref, 'refs/tags') }}
+ DEV_REPOSITORY: 329710836760.dkr.ecr.us-east-1.amazonaws.com/rasa-sdk-dev
+ AWS_REGION: us-east-1
 
 # SECRETS
 # - PYPI_TOKEN: publishing token for amn41 account, needs to be maintainer of
 # RasaHQ/rasa-sdk on pypi (account credentials in 1password)
 
+permissions:
+ id-token: write # This is required for requesting the JWT
+ contents: read # This is required for actions/checkout
+
+
 jobs:
  quality:
  name: Code Quality
@@ -128,204 +135,34 @@ jobs:
  with:
  dockerfile: "Dockerfile"
 
- build_docker_image_set_env:
- name: Prepare environment for Docker build
- runs-on: ubuntu-22.04
- outputs:
- # Tag name used for intermediate images created during Docker image builds, e.g. 3886 - a PR number
- image_tag: ${{ steps.set_output.outputs.image_tag }}
- # Return 'true' if tag version is equal or higher than the latest tagged rasa-sdk version
- is_newest_version: ${{ steps.rasa_sdk_get_version.outputs.is_newest_version }}
- steps:
- # Due to an issue with checking out a wrong commit, we make sure
- # to checkout HEAD commit for a pull request.
- # More details: https://github.com/actions/checkout/issues/299
- - name: Checkout pull request HEAD commit instead of merge commit 🕝
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3
- if: github.event_name == 'pull_request'
- with:
- ref: ${{ github.event.pull_request.head.sha }}
-
- - name: Checkout git repository 🕝
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3
- if: github.event_name != 'pull_request'
-
- # Set environment variables for a pull request
- #
- # In this scenario, we've created a PR #1234
- #
- # Example output:
- # IMAGE_TAG=1234
- - name: Set environment variables - pull_request
- if: github.event_name == 'pull_request' && env.IS_TAG_BUILD == 'false'
- run: |
- echo "IMAGE_TAG=${{ github.event.number }}" >> $GITHUB_ENV
-
- # Set environment variables for a tag
- #
- # In this scenario, we've pushed the '2.4.1' tag
- #
- # Example output:
- # IMAGE_TAG=2.4.1
- - name: Set environment variables - push - tag
- if: github.event_name == 'push' && env.IS_TAG_BUILD == 'true'
- run: |
- TAG_NAME=${GITHUB_REF#refs/tags/}
- echo "IMAGE_TAG=${TAG_NAME}" >> $GITHUB_ENV
-
- # Set environment variables for a branch
- #
- # In this scenario, we've pushed changes into the main branch
- #
- # Example output:
- # IMAGE_TAG=main
- - name: Set environment variables - push - branch
- if: github.event_name == 'push' && env.IS_TAG_BUILD == 'false'
- run: |
- BRANCH_NAME=${GITHUB_REF#refs/heads/}
- echo "IMAGE_TAG=${BRANCH_NAME}" >> $GITHUB_ENV
-
- - name: Set output
- id: set_output
- run: |
- echo "image_tag=${{ env.IMAGE_TAG }}" >> $GITHUB_OUTPUT
-
- - name: Check if tag version is equal or higher than the latest tagged rasa-sdk version
- id: rasa_sdk_get_version
- if: env.IS_TAG_BUILD == 'true'
- run: |
- # Get latest tagged rasa-sdk version
- git fetch --depth=1 origin "+refs/tags/*:refs/tags/*"
- # Fetch branch history
- git fetch --prune --unshallow
- LATEST_TAGGED_NON_ALPHA_RASA_SDK_VERSION=$(git tag | sort -r -V | grep -E "^[0-9.]+$" | head -n1)
- CURRENT_TAG=${GITHUB_REF#refs/tags/}
- # Return 'true' if tag version is equal or higher than the latest tagged rasa-sdk version
- IS_NEWEST_VERSION=$((printf '%s\n%s\n' "${LATEST_TAGGED_NON_ALPHA_RASA_SDK_VERSION}" "$CURRENT_TAG" \
- | sort -V -C && echo true || echo false) || true)
-
-
- if [[ "${IS_NEWEST_VERSION}" == "true" && "$CURRENT_TAG" =~ ^[0-9.]+$ ]]; then
- echo "is_newest_version=true" >> $GITHUB_OUTPUT
- else
- echo "is_newest_version=false" >> $GITHUB_OUTPUT
- fi
-
- build_docker_image:
- name: Build Docker image
+ rasa-sdk-dev-docker-image:
+ name: Build dev Docker image
  runs-on: ubuntu-22.04
- needs: [quality, test, docker_linter, build_docker_image_set_env]
 
  steps:
- # Due to an issue with checking out a wrong commit, we make sure
- # to checkout HEAD commit for a pull request.
- # More details: https://github.com/actions/checkout/issues/299
- - name: Checkout pull request HEAD commit instead of merge commit 🕝
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3
- if: github.event_name == 'pull_request'
- with:
- ref: ${{ github.event.pull_request.head.sha }}
+ - name: Check out code
+ uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
- - name: Checkout git repository 🕝
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3
- if: github.event_name != 'pull_request'
-
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c
+ - name: Configure AWS credentials
+ uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 # v3.0.1
  with:
- version: v0.5.1
- driver: docker
+ role-to-assume: ${{ secrets.AWS_ASSUME_ROLE_SESSION_TOKEN }}
+ aws-region: ${{ env.AWS_REGION }}
 
- - name: Set environment variables
- run: |
- echo "IMAGE_TAG=${{ needs.build_docker_image_set_env.outputs.image_tag }}" >> $GITHUB_ENV
-
- - name: Login to Docker Hub Registry 🔢
- if: github.event_name == 'push' && env.IS_TAG_BUILD == 'true' && github.repository == 'RasaHQ/rasa-sdk'
- run: echo ${{ secrets.DOCKER_HUB_PASSWORD }} | docker login -u ${{ secrets.DOCKER_HUB_USERNAME }} --password-stdin || true
-
- - name: Build and Push Docker image 📦
- if: github.event_name == 'push' && env.IS_TAG_BUILD == 'true' && github.repository == 'RasaHQ/rasa-sdk'
- run: |
- IS_NEWEST_VERSION=${{ needs.build_docker_image_set_env.outputs.is_newest_version }}
-
- # Push image
- IMAGE_TAG=${IMAGE_TAG} docker buildx bake --set *.platform=linux/amd64,linux/arm64 -f docker-bake.hcl default --push
-
- # Tag the image as latest
- if [[ "${IS_NEWEST_VERSION}" == "true" ]]; then
- docker tag rasa/rasa-sdk:${IMAGE_TAG} rasa/rasa-sdk:latest
- docker push rasa/rasa-sdk:latest
- fi
-
- deploy:
- name: Deploy to PyPI
- runs-on: ubuntu-22.04
-
- # deploy will only be run when there is a tag available
- if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
- needs: [quality, test, build_docker_image] # only run after all other stages succeeded
-
- steps:
- - name: Checkout git repository 🕝
- uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3
-
- - name: Set up Python 3.10 🐍
- uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435
+ - name: Login to Amazon ECR
+ id: login-ecr
+ uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1
  with:
- python-version: '3.10'
+ mask-password: "true"
 
- - name: Read Poetry Version 🔢
- run: |
- echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
- shell: bash
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
 
- - name: Install poetry 🦄
- uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844
- with:
- poetry-version: ${{ env.POETRY_VERSION }}
-
- - name: Build ⚒️ Distributions
- run: poetry build
-
- - name: Publish to PyPI 📦
- uses: pypa/gh-action-pypi-publish@bea5cda687c2b79989126d589ef4411bedce0195
- with:
- user: __token__
- password: ${{ secrets.PYPI_TOKEN }}
- skip_existing: true
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0
 
- - name: Publish Release Notes 🗞
- if: env.IS_TAG_BUILD
- env:
- GITHUB_TAG: ${{ github.ref }}
- GITHUB_REPO_SLUG: ${{ github.repository }}
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ - name: Build and push docker image to AWS
  run: |
- GITHUB_TAG=${GITHUB_TAG/refs\/tags\//}
- sudo apt-get update
- sudo apt-get -y install pandoc
- pip install -U github3.py pep440_version_utils
- python3 ${GITHUB_WORKSPACE}/scripts/publish_gh_release_notes.py
-
- - name: Get RASA SDK Version
- env:
- RASA_SDK_VERSION: ${{ github.ref }}
- run: |
- echo "RASA_SDK_VERSION=${RASA_SDK_VERSION/refs\/tags\//}" >> $GITHUB_ENV
-
- - name: Notify Slack 💬
- if: env.IS_TAG_BUILD && success()
- env:
- SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ASSISTANT_RELEASE_WEBHOOK }}
- uses: Ilshidur/[email protected]
- with:
- args: "⚡ New *Rasa SDK* version ${{ env.RASA_SDK_VERSION }} has been released! Changelog: https://github.com/RasaHQ/rasa-sdk/blob/${{ env.RASA_SDK_VERSION }}/CHANGELOG.mdx"
-
- - name: Notify Slack of Failure ⛔
- if: env.IS_TAG_BUILD && failure()
- env:
- SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ASSISTANT_DEV_TRIBE_WEBHOOK }}
- uses: Ilshidur/[email protected]
- with:
- args: "⛔️ *Rasa SDK* version ${{ env.RASA_SDK_VERSION }} could not be released 😱 GitHub Actions: https://github.com/RasaHQ/rasa-sdk/actions?query=branch%3A${{ env.RASA_SDK_VERSION }}"
+ IMAGE_NAME=${{ env.DEV_REPOSITORY }} \
+ IMAGE_TAG=pr${{ github.event.number }} \
+ make build-and-push-multi-platform-docker
diff --git a/.github/workflows/pr-cleanup.yml b/.github/workflows/pr-cleanup.yml
@@ -0,0 +1,51 @@
+name: CI On Pull Request Closure
+on:
+ pull_request:
+ types: [closed]
+
+env:
+ AWS_REGION: us-east-1
+
+permissions:
+ id-token: write # This is required for requesting the JWT
+ contents: read # This is required for actions/checkout
+
+jobs: 
+ aws-ecr:
+ # Skip any PR created by dependabot to avoid permission issues
+ if: (github.actor != 'dependabot[bot]')
+ name: Delete PR Docker Images
+ runs-on: ubuntu-22.04
+ steps:
+ - name: Configure AWS credentials
+ uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 #v3.0.1
+ with:
+ role-to-assume: ${{ secrets.AWS_ASSUME_ROLE_SESSION_TOKEN }}
+ aws-region: ${{ env.AWS_REGION }}
+
+ - name: Login to Amazon ECR
+ id: login-ecr
+ uses: aws-actions/amazon-ecr-login@2fc7aceee09e9e4a7105c0d060c656fad0b4f63d #v1.6
+ with:
+ mask-password: "true"
+
+ - name: Remove docker images from all ECR repositories
+ run: |
+ REPO_NAMES=("rasa-sdk-dev")
+ SEARCH_STRING="pr${{ github.event.number }}"
+ # Iterate through each repository
+ for REPO_NAME in "${REPO_NAMES[@]}"
+ do
+ # List all image tags in the repository that match the search string
+ # Escape empty strings and null character
+ IMAGES=$(aws ecr list-images --repository-name $REPO_NAME --query "imageIds[?imageTag!=null&&imageTag!=''&&contains(imageTag, '$SEARCH_STRING')].imageTag" --output text)
+ 
+ # Print the image tags
+ echo "Images to be deleted in repository $REPO_NAME:"
+ echo "$IMAGES"
+ # Delete each image by its tag
+ for IMAGE in $IMAGES
+ do
+ aws ecr batch-delete-image --repository-name $REPO_NAME --image-ids imageTag=$IMAGE
+ done
+ done
diff --git a/...b/workflows/rasa-sdk-dev-docker-image.yml → .github/workflows/pr-merged.yml b/...b/workflows/rasa-sdk-dev-docker-image.yml → .github/workflows/pr-merged.yml
@@ -1,11 +1,9 @@
-name: Create Dev Docker Images
+name: Pull Request Merged to Main
 
 on:
- schedule:
- # Run cron job at 8AM Monday to Sunday.
- - cron: '0 8 * * *'
- workflow_dispatch:
- pull_request:
+ push:
+ branches:
+ - main
 
 env:
  AWS_REGION: us-east-1
@@ -21,18 +19,13 @@ permissions:
 
 jobs:
  rasa-sdk-dev-docker-image:
- name: rasa-sdk dev docker image
+ name: Build Dev Docker Image and Push to AWS
  runs-on: ubuntu-22.04
 
  steps:
  - name: Check out code
  uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
- - name: Read Poetry Version 🔢
- run: |
- echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
- shell: bash
-
  - name: Configure AWS credentials
  uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 # v3.0.1
  with:
@@ -45,11 +38,14 @@ jobs:
  with:
  mask-password: "true"
 
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
+
  - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c
- with:
- version: v0.5.1
+ uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0
 
  - name: Build and push docker image to AWS
  run: |
- docker buildx bake --set *.platform=linux/amd64,linux/arm64 --set default.tags=$REPOSITORY:latest --push
+ IMAGE_NAME=${{ env.REPOSITORY }} \
+ IMAGE_TAG=latest \
+ make build-and-push-multi-platform-docker