Skip to content

Commit

Permalink
Port build of multi-platform dev docker images from 3.9.x to main (
Browse files Browse the repository at this point in the history
…#1125)

* add support for ARM arch

* add changelog entry

* trigger dev image build on PR

* remove docker driver from dev image workflow

* use --push instead of load flag

* update override flag

* remove target from override flag

* Build multiplatform image for amd64 and arm64 architectures on release

* Set POETRY_VERSION as build argument in Dockerfile

* Add workflow_dispatch to release-artifacts GitHub workflow to allow manual release of Docker images and PyPI packages

---------

Co-authored-by: Anca Lita <[email protected]>
  • Loading branch information
radovanZRasa and ancalita authored Jul 11, 2024
1 parent 1f3fe9d commit 6423418
Show file tree
Hide file tree
Showing 8 changed files with 299 additions and 258 deletions.
221 changes: 27 additions & 194 deletions .github/workflows/continuous-integration.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,18 @@ on:

env:
IS_TAG_BUILD: ${{ startsWith(github.event.ref, 'refs/tags') }}
DEV_REPOSITORY: 329710836760.dkr.ecr.us-east-1.amazonaws.com/rasa-sdk-dev
AWS_REGION: us-east-1

# SECRETS
# - PYPI_TOKEN: publishing token for amn41 account, needs to be maintainer of
# RasaHQ/rasa-sdk on pypi (account credentials in 1password)

permissions:
id-token: write # This is required for requesting the JWT
contents: read # This is required for actions/checkout


jobs:
quality:
name: Code Quality
Expand Down Expand Up @@ -128,208 +135,34 @@ jobs:
with:
dockerfile: "Dockerfile"

build_docker_image_set_env:
name: Prepare environment for Docker build
runs-on: ubuntu-22.04
outputs:
# Tag name used for intermediate images created during Docker image builds, e.g. 3886 - a PR number
image_tag: ${{ steps.set_output.outputs.image_tag }}
# Return 'true' if tag version is equal or higher than the latest tagged rasa-sdk version
is_newest_version: ${{ steps.rasa_sdk_get_version.outputs.is_newest_version }}
steps:
# Due to an issue with checking out a wrong commit, we make sure
# to checkout HEAD commit for a pull request.
# More details: https://github.com/actions/checkout/issues/299
- name: Checkout pull request HEAD commit instead of merge commit 🕝
uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3
if: github.event_name == 'pull_request'
with:
ref: ${{ github.event.pull_request.head.sha }}

- name: Checkout git repository 🕝
uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3
if: github.event_name != 'pull_request'

# Set environment variables for a pull request
#
# In this scenario, we've created a PR #1234
#
# Example output:
# IMAGE_TAG=1234
- name: Set environment variables - pull_request
if: github.event_name == 'pull_request' && env.IS_TAG_BUILD == 'false'
run: |
echo "IMAGE_TAG=${{ github.event.number }}" >> $GITHUB_ENV
# Set environment variables for a tag
#
# In this scenario, we've pushed the '2.4.1' tag
#
# Example output:
# IMAGE_TAG=2.4.1
- name: Set environment variables - push - tag
if: github.event_name == 'push' && env.IS_TAG_BUILD == 'true'
run: |
TAG_NAME=${GITHUB_REF#refs/tags/}
echo "IMAGE_TAG=${TAG_NAME}" >> $GITHUB_ENV
# Set environment variables for a branch
#
# In this scenario, we've pushed changes into the main branch
#
# Example output:
# IMAGE_TAG=main
- name: Set environment variables - push - branch
if: github.event_name == 'push' && env.IS_TAG_BUILD == 'false'
run: |
BRANCH_NAME=${GITHUB_REF#refs/heads/}
echo "IMAGE_TAG=${BRANCH_NAME}" >> $GITHUB_ENV
- name: Set output
id: set_output
run: |
echo "image_tag=${{ env.IMAGE_TAG }}" >> $GITHUB_OUTPUT
- name: Check if tag version is equal or higher than the latest tagged rasa-sdk version
id: rasa_sdk_get_version
if: env.IS_TAG_BUILD == 'true'
run: |
# Get latest tagged rasa-sdk version
git fetch --depth=1 origin "+refs/tags/*:refs/tags/*"
# Fetch branch history
git fetch --prune --unshallow
LATEST_TAGGED_NON_ALPHA_RASA_SDK_VERSION=$(git tag | sort -r -V | grep -E "^[0-9.]+$" | head -n1)
CURRENT_TAG=${GITHUB_REF#refs/tags/}
# Return 'true' if tag version is equal or higher than the latest tagged rasa-sdk version
IS_NEWEST_VERSION=$((printf '%s\n%s\n' "${LATEST_TAGGED_NON_ALPHA_RASA_SDK_VERSION}" "$CURRENT_TAG" \
| sort -V -C && echo true || echo false) || true)
if [[ "${IS_NEWEST_VERSION}" == "true" && "$CURRENT_TAG" =~ ^[0-9.]+$ ]]; then
echo "is_newest_version=true" >> $GITHUB_OUTPUT
else
echo "is_newest_version=false" >> $GITHUB_OUTPUT
fi
build_docker_image:
name: Build Docker image
rasa-sdk-dev-docker-image:
name: Build dev Docker image
runs-on: ubuntu-22.04
needs: [quality, test, docker_linter, build_docker_image_set_env]

steps:
# Due to an issue with checking out a wrong commit, we make sure
# to checkout HEAD commit for a pull request.
# More details: https://github.com/actions/checkout/issues/299
- name: Checkout pull request HEAD commit instead of merge commit 🕝
uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3
if: github.event_name == 'pull_request'
with:
ref: ${{ github.event.pull_request.head.sha }}

- name: Checkout git repository 🕝
uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3
if: github.event_name != 'pull_request'
- name: Check out code
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 # v3.0.1
with:
version: v0.5.1
driver: docker

- name: Set environment variables
run: |
echo "IMAGE_TAG=${{ needs.build_docker_image_set_env.outputs.image_tag }}" >> $GITHUB_ENV
- name: Build Docker image
run: |
docker buildx bake --load
- name: Login to Docker Hub Registry 🔢
if: github.event_name == 'push' && env.IS_TAG_BUILD == 'true' && github.repository == 'RasaHQ/rasa-sdk'
run: echo ${{ secrets.DOCKER_HUB_PASSWORD }} | docker login -u ${{ secrets.DOCKER_HUB_USERNAME }} --password-stdin || true

- name: Push Docker image 📦
if: github.event_name == 'push' && env.IS_TAG_BUILD == 'true' && github.repository == 'RasaHQ/rasa-sdk'
run: |
IS_NEWEST_VERSION=${{ needs.build_docker_image_set_env.outputs.is_newest_version }}
# Push image
docker push rasa/rasa-sdk:${IMAGE_TAG}
# Tag the image as latest
if [[ "${IS_NEWEST_VERSION}" == "true" ]]; then
docker tag rasa/rasa-sdk:${IMAGE_TAG} rasa/rasa-sdk:latest
docker push rasa/rasa-sdk:latest
fi
role-to-assume: ${{ secrets.AWS_ASSUME_ROLE_SESSION_TOKEN }}
aws-region: ${{ env.AWS_REGION }}

deploy:
name: Deploy to PyPI
runs-on: ubuntu-22.04

# deploy will only be run when there is a tag available
if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
needs: [quality, test, build_docker_image] # only run after all other stages succeeded

steps:
- name: Checkout git repository 🕝
uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3

- name: Set up Python 3.10 🐍
uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435
with:
python-version: '3.10'

- name: Read Poetry Version 🔢
run: |
echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
shell: bash

- name: Install poetry 🦄
uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844
- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1
with:
poetry-version: ${{ env.POETRY_VERSION }}
mask-password: "true"

- name: Build ⚒️ Distributions
run: poetry build
- name: Set up QEMU
uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0

- name: Publish to PyPI 📦
uses: pypa/gh-action-pypi-publish@bea5cda687c2b79989126d589ef4411bedce0195
with:
user: __token__
password: ${{ secrets.PYPI_TOKEN }}
skip_existing: true

- name: Publish Release Notes 🗞
if: env.IS_TAG_BUILD
env:
GITHUB_TAG: ${{ github.ref }}
GITHUB_REPO_SLUG: ${{ github.repository }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
GITHUB_TAG=${GITHUB_TAG/refs\/tags\//}
sudo apt-get update
sudo apt-get -y install pandoc
pip install -U github3.py pep440_version_utils
python3 ${GITHUB_WORKSPACE}/scripts/publish_gh_release_notes.py
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0

- name: Get RASA SDK Version
env:
RASA_SDK_VERSION: ${{ github.ref }}
- name: Build and push docker image to AWS
run: |
echo "RASA_SDK_VERSION=${RASA_SDK_VERSION/refs\/tags\//}" >> $GITHUB_ENV
- name: Notify Slack 💬
if: env.IS_TAG_BUILD && success()
env:
SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ASSISTANT_RELEASE_WEBHOOK }}
uses: Ilshidur/[email protected]
with:
args: "⚡ New *Rasa SDK* version ${{ env.RASA_SDK_VERSION }} has been released! Changelog: https://github.com/RasaHQ/rasa-sdk/blob/${{ env.RASA_SDK_VERSION }}/CHANGELOG.mdx"

- name: Notify Slack of Failure ⛔
if: env.IS_TAG_BUILD && failure()
env:
SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ASSISTANT_DEV_TRIBE_WEBHOOK }}
uses: Ilshidur/[email protected]
with:
args: "⛔️ *Rasa SDK* version ${{ env.RASA_SDK_VERSION }} could not be released 😱 GitHub Actions: https://github.com/RasaHQ/rasa-sdk/actions?query=branch%3A${{ env.RASA_SDK_VERSION }}"
IMAGE_NAME=${{ env.DEV_REPOSITORY }} \
IMAGE_TAG=pr${{ github.event.number }} \
make build-and-push-multi-platform-docker
51 changes: 51 additions & 0 deletions .github/workflows/pr-cleanup.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
name: CI On Pull Request Closure
on:
pull_request:
types: [closed]

env:
AWS_REGION: us-east-1

permissions:
id-token: write # This is required for requesting the JWT
contents: read # This is required for actions/checkout

jobs:
aws-ecr:
# Skip any PR created by dependabot to avoid permission issues
if: (github.actor != 'dependabot[bot]')
name: Delete PR Docker Images
runs-on: ubuntu-22.04
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 #v3.0.1
with:
role-to-assume: ${{ secrets.AWS_ASSUME_ROLE_SESSION_TOKEN }}
aws-region: ${{ env.AWS_REGION }}

- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@2fc7aceee09e9e4a7105c0d060c656fad0b4f63d #v1.6
with:
mask-password: "true"

- name: Remove docker images from all ECR repositories
run: |
REPO_NAMES=("rasa-sdk-dev")
SEARCH_STRING="pr${{ github.event.number }}"
# Iterate through each repository
for REPO_NAME in "${REPO_NAMES[@]}"
do
# List all image tags in the repository that match the search string
# Escape empty strings and null character
IMAGES=$(aws ecr list-images --repository-name $REPO_NAME --query "imageIds[?imageTag!=null&&imageTag!=''&&contains(imageTag, '$SEARCH_STRING')].imageTag" --output text)
# Print the image tags
echo "Images to be deleted in repository $REPO_NAME:"
echo "$IMAGES"
# Delete each image by its tag
for IMAGE in $IMAGES
do
aws ecr batch-delete-image --repository-name $REPO_NAME --image-ids imageTag=$IMAGE
done
done
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
name: Create Dev Docker Images
name: Pull Request Merged to Main

on:
schedule:
# Run cron job at 8AM Monday to Sunday.
- cron: '0 8 * * *'
workflow_dispatch:
push:
branches:
- main

env:
AWS_REGION: us-east-1
Expand All @@ -20,18 +19,13 @@ permissions:

jobs:
rasa-sdk-dev-docker-image:
name: rasa-sdk dev docker image
name: Build Dev Docker Image and Push to AWS
runs-on: ubuntu-22.04

steps:
- name: Check out code
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

- name: Read Poetry Version 🔢
run: |
echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
shell: bash

- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 # v3.0.1
with:
Expand All @@ -44,17 +38,14 @@ jobs:
with:
mask-password: "true"

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c
with:
version: v0.5.1
driver: docker
- name: Set up QEMU
uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0

- name: Build docker image
run: |
docker buildx bake --load
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0

- name: Tag and push docker image to AWS
- name: Build and push docker image to AWS
run: |
docker tag rasa/rasa-sdk:main $REPOSITORY:latest
docker push $REPOSITORY:latest
IMAGE_NAME=${{ env.REPOSITORY }} \
IMAGE_TAG=latest \
make build-and-push-multi-platform-docker
Loading

0 comments on commit 6423418

Please sign in to comment.