From 868e0d5da5939eca2a92ec9fd7614e5711ecc067 Mon Sep 17 00:00:00 2001 From: Mason Tran Date: Tue, 18 Jun 2024 18:04:14 -0400 Subject: [PATCH] [github-actions] build docker image then run `arm-gcc` with it This changes the relationship of the workflows so that the `arm-gcc` job in `build.yml` is run using the docker image generated by the docker `build-image` job. Previously, the `arm-gcc` job would just use the `:latest` tag of `siliconlabsinc/ot-efr32-dev` from DockerHub. This presents a problem when there are changes to the bootstrap script which are required for the build script to succeed. These changes would not be present in the currently published `siliconlabsinc/ot-efr32-dev:latest` since that image was build off of `SiliconLabs/ot-efr32:main`, not the current branch. --- .github/workflows/build.yml | 111 +++++++++++++++++----------- .github/workflows/docker.yml | 127 +++++++++++++++------------------ .github/workflows/metadata.yml | 62 ++++++++++++++++ docker/Dockerfile | 15 +++- docker/base.Dockerfile | 25 +++++++ docker/ot-efr32-dev.Dockerfile | 21 ++++++ 6 files changed, 249 insertions(+), 112 deletions(-) create mode 100644 .github/workflows/metadata.yml create mode 100644 docker/base.Dockerfile create mode 100644 docker/ot-efr32-dev.Dockerfile diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5a428645..9db7c1ca 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,11 +41,23 @@ concurrency: cancel-in-progress: true jobs: + docker: + name: Docker + uses: ./.github/workflows/docker.yml + + metadata: + name: Metadata + uses: ./.github/workflows/metadata.yml + arm-gcc: - name: arm-gcc-${{ matrix.gcc_ver }} + name: Arm GNU Toolchain ${{ matrix.gcc_ver }} runs-on: ubuntu-22.04 + needs: [metadata, docker] + continue-on-error: true + permissions: + packages: write container: - image: siliconlabsinc/ot-efr32-dev:latest + image: ${{ needs.metadata.outputs.ghcr_image }} options: --user 1001 strategy: fail-fast: false @@ -59,46 +71,61 @@ jobs: gcc_extract_dir: arm-gnu-toolchain-12.3.rel1-x86_64-arm-none-eabi steps: - - uses: actions/checkout@v4 - with: - submodules: true + - uses: actions/checkout@v4 + with: + submodules: true - - name: Create LFS file hash list - run: git -C third_party/silabs/gecko_sdk lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-assets-id + - name: Create LFS file hash list + run: git -C third_party/silabs/gecko_sdk lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-assets-id - - name: Restore gecko_sdk LFS cache - uses: actions/cache@v4 - id: lfs-cache - with: - path: .git/modules/third_party/silabs/gecko_sdk/lfs - key: lfs-${{ hashFiles('.lfs-assets-id') }} - - - name: Git LFS Pull - run: git -C third_party/silabs/gecko_sdk lfs pull - - - name: Bootstrap ARM Toolchain - run: | - script/bootstrap arm_toolchain ${{ matrix.gcc_download_url }} ${{ matrix.gcc_extract_dir }} - - - name: Build - run: | - export PATH=${HOME}/.local/${{ matrix.gcc_extract_dir }}/bin:$PATH - script/test - - - name: Gather SLC generated files - if: failure() - run: | - rm -rf artifact && mkdir artifact - for b in build/*/slc; do - board=$(basename $(dirname "${b}")) - - echo "Artifacting '${board}'" - mkdir -p "artifact/${board}" - mv "build/${board}/slc" "artifact/${board}" - done - - - uses: actions/upload-artifact@v4 - if: failure() + - name: Restore gecko_sdk LFS cache + uses: actions/cache@v4 + id: lfs-cache + with: + path: .git/modules/third_party/silabs/gecko_sdk/lfs + key: lfs-${{ hashFiles('.lfs-assets-id') }} + + - name: Git LFS Pull + run: git -C third_party/silabs/gecko_sdk lfs pull + + - name: Bootstrap ARM Toolchain + run: | + script/bootstrap arm_toolchain ${{ matrix.gcc_download_url }} ${{ matrix.gcc_extract_dir }} + + - name: Build + run: | + export PATH=${HOME}/.local/${{ matrix.gcc_extract_dir }}/bin:$PATH + script/test + + - name: Gather SLC generated files + if: failure() + run: | + rm -rf artifact && mkdir artifact + for b in build/*/slc; do + board=$(basename $(dirname "${b}")) + + echo "Artifacting '${board}'" + mkdir -p "artifact/${board}" + mv "build/${board}/slc" "artifact/${board}" + done + + - uses: actions/upload-artifact@v4 + if: failure() + with: + name: build-${{ matrix.gcc_ver }} + path: artifact + + cleanup: + name: Cleanup + runs-on: ubuntu-22.04 + needs: [metadata, docker, arm-gcc] + steps: + - name: Delete ot-efr32-dev image + uses: bots-house/ghcr-delete-image-action@v1.1.0 + if: always() + continue-on-error: true with: - name: build-${{ matrix.gcc_ver }} - path: artifact + owner: ${{ needs.metadata.outputs.owner }} + name: ot-efr32-dev + token: ${{ secrets.GITHUB_TOKEN }} + tag: ${{ github.sha }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4bd9e463..bb91fd4d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -28,58 +28,53 @@ name: Docker -env: - TEST_TAG: siliconlabsinc/ot-efr32-dev:test - SHA_TAG: siliconlabsinc/ot-efr32-dev:${{ github.sha }} - LATEST_TAG: siliconlabsinc/ot-efr32-dev:latest - on: - push: - branches-ignore: - - 'dependabot/**' - pull_request: - branches: - - 'main' - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.repository == 'openthread/ot-efr32' && github.run_id) || github.ref }} - cancel-in-progress: true + workflow_call: -permissions: # added using https://github.com/step-security/secure-workflows - contents: read +env: + DOCKERHUB_LATEST_TAG: siliconlabsinc/ot-efr32-dev:latest jobs: - build: - name: Build Docker Image + metadata: + uses: ./.github/workflows/metadata.yml + + image: + name: Build image runs-on: ubuntu-22.04 + needs: metadata + permissions: + packages: write # Required to push Docker image to GitHub Container Registry steps: - name: Harden Runner uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 with: egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs - - uses: actions/checkout@v4 # v3.3.0 + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: submodules: true - name: Set up Docker Buildx uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 + with: + platforms: linux/amd64 - name: Build and export to Docker context - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: + build-args: | + BUILD_DATE=${{ needs.metadata.outputs.date }} context: . file: docker/Dockerfile load: true - build-args: | - - BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') - platforms: - linux/amd64 tags: | - ${{ env.TEST_TAG }} + ${{ needs.metadata.outputs.ghcr_image }} cache-from: type=gha cache-to: type=gha,mode=max + - name: Inspect Docker image + run: docker inspect ${{ needs.metadata.outputs.ghcr_image }} + - name: Container image sanity checks run: | # Download container-structure-test @@ -87,49 +82,45 @@ jobs: && chmod +x container-structure-test-linux-amd64 # Run tests - ./container-structure-test-linux-amd64 test --config docker/test-ot-efr32-dev.yml --image ${{ env.TEST_TAG }} - - - name: Create LFS file hash list - run: git -C third_party/silabs/gecko_sdk lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-assets-id + ./container-structure-test-linux-amd64 test --config docker/test-ot-efr32-dev.yml --image ${{ needs.metadata.outputs.ghcr_image }} - - name: Restore gecko_sdk LFS cache - uses: actions/cache@v4 - id: lfs-cache + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 with: - path: .git/modules/third_party/silabs/gecko_sdk/lfs - key: lfs-${{ hashFiles('.lfs-assets-id') }} + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - - name: Git LFS Pull - run: git -C third_party/silabs/gecko_sdk lfs pull - - - name: Test build inside container + - name: Push Docker image run: | - docker run -v ${{ github.workspace }}:/ot-efr32/ --user $(id -u) --rm ${{ env.TEST_TAG }} script/build --skip-silabs-apps brd4151a - - - name: Login to DockerHub - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 - if: | - success() && - github.repository == 'SiliconLabs/ot-efr32' && - github.event_name != 'pull_request' && - github.ref == 'refs/heads/main' - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Build and push - uses: docker/build-push-action@v5 - if: | - success() && - github.repository == 'SiliconLabs/ot-efr32' && - github.event_name != 'pull_request' && - github.ref == 'refs/heads/main' - with: - context: . - file: docker/Dockerfile - push: true - tags: | - ${{ env.LATEST_TAG }} - ${{ env.SHA_TAG }} - platforms: - linux/amd64 + docker push ${{ needs.metadata.outputs.ghcr_image }} + + publish-dockerhub: + name: Tag `latest` and publish to DockerHub + runs-on: ubuntu-22.04 + needs: [metadata, image] + permissions: + packages: read # Required to pull Docker image from GitHub Container Registry + if: | + github.repository == 'SiliconLabs/ot-efr32' && + github.event_name != 'pull_request' && + github.ref == 'refs/heads/main' + steps: + - name: Login to DockerHub + if: github.ref == 'refs/heads/main' + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Pull Docker image + run: | + docker pull ${{ needs.metadata.outputs.ghcr_image }} + + - name: Tag Docker image + run: | + docker tag ${{ needs.metadata.outputs.ghcr_image }} ${{ env.DOCKERHUB_LATEST_TAG }} + + - name: Push Docker image + run: | + docker push ${{ env.DOCKERHUB_LATEST_TAG }} diff --git a/.github/workflows/metadata.yml b/.github/workflows/metadata.yml new file mode 100644 index 00000000..0ee922e0 --- /dev/null +++ b/.github/workflows/metadata.yml @@ -0,0 +1,62 @@ +# +# Copyright (c) 2024, The OpenThread Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# + +name: Metadata + +on: + workflow_call: + outputs: + owner: + description: 'The lowercase GitHub repository owner' + value: ${{ jobs.metadata.outputs.owner }} + date: + description: 'The current date' + value: ${{ jobs.metadata.outputs.date }} + ghcr_image: + description: 'The GHCR image name' + value: ${{ jobs.metadata.outputs.ghcr_image }} + +jobs: + metadata: + name: Generate required metadata + runs-on: ubuntu-22.04 + outputs: + owner: ${{ steps.lowercase_owner.outputs.owner }} + date: ${{ steps.date.outputs.date }} + ghcr_image: ${{ steps.ghcr_image.outputs.ghcr_image }} + steps: + - name: Lowercase GitHub repository owner + id: lowercase_owner + run: echo "owner=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')" >> ${GITHUB_OUTPUT} + - name: Get current date + id: date + run: echo "date=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> ${GITHUB_OUTPUT} + - name: GHCR image name + id: ghcr_image + run: echo "ghcr_image=ghcr.io/${{ steps.lowercase_owner.outputs.owner }}/ot-efr32-dev:${{ github.sha }}" >> ${GITHUB_OUTPUT} + diff --git a/docker/Dockerfile b/docker/Dockerfile index b9d9d6bd..afb87b5c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,6 @@ -FROM ubuntu:22.04 +ARG BASE_IMAGE='ubuntu:22.04' + +FROM ${BASE_IMAGE} AS base ENV TZ="America/New_York" ENV repo_dir="/ot-efr32" @@ -23,7 +25,11 @@ RUN ./script/bootstrap arm_toolchain COPY ./requirements.txt . RUN ./script/bootstrap python +# ============================================================================== + # Label the build date before downloading slc to force slc to always be downloaded during a docker build + +FROM base AS ot-efr32-dev ARG BUILD_DATE LABEL build_date=${BUILD_DATE} @@ -34,8 +40,13 @@ RUN mkdir ${SLC_INSTALL_DIR} && \ ./script/bootstrap silabs +# Change workdir to root temporarily +WORKDIR / + # Clone repo for convenience ARG REPO_URL="https://github.com/openthread/ot-efr32" -WORKDIR / +ENV repo_dir="/ot-efr32" RUN rm -rf ${repo_dir} && git clone ${REPO_URL} ${repo_dir} + +# Change workdir back to repo WORKDIR ${repo_dir} diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile new file mode 100644 index 00000000..212e33af --- /dev/null +++ b/docker/base.Dockerfile @@ -0,0 +1,25 @@ +ARG BASE_IMAGE='ubuntu:22.04' +FROM ${BASE_IMAGE} as base + +ENV TZ="America/New_York" +ENV repo_dir="/ot-efr32" +WORKDIR ${repo_dir} + +# Install packages +RUN apt-get update && \ + apt-get -y install --no-install-recommends \ + sudo \ + tzdata \ + && rm -rf /var/lib/apt/lists/* + +# Copy scripts +COPY ./script/bootstrap \ + ./script/ + +# Install system packages and ARM toolchain +RUN ./script/bootstrap packages && rm -rf /var/lib/apt/lists/* +RUN ./script/bootstrap arm_toolchain + +# Install Python packages +COPY ./requirements.txt . +RUN ./script/bootstrap python diff --git a/docker/ot-efr32-dev.Dockerfile b/docker/ot-efr32-dev.Dockerfile new file mode 100644 index 00000000..79bb0ca2 --- /dev/null +++ b/docker/ot-efr32-dev.Dockerfile @@ -0,0 +1,21 @@ +ARG BASE_IMAGE='siliconlabsinc/ot-efr32-dev:base' + +FROM ${BASE_IMAGE} AS ot-efr32-dev + +COPY ./script/bootstrap_silabs \ + ./script/ +ENV SLC_INSTALL_DIR=/opt/slc_cli +RUN mkdir ${SLC_INSTALL_DIR} && \ + ./script/bootstrap silabs + + +# Change workdir to root temporarily +WORKDIR / + +# Clone repo for convenience +ARG REPO_URL="https://github.com/openthread/ot-efr32" +ENV repo_dir="/ot-efr32" +RUN rm -rf ${repo_dir} && git clone ${REPO_URL} ${repo_dir} + +# Change workdir back to repo +WORKDIR ${repo_dir}