From 141e1474522ef648ae445b2a596c01f48346ccf9 Mon Sep 17 00:00:00 2001 From: Mason Tran Date: Thu, 8 Feb 2024 12:44:41 -0500 Subject: [PATCH] [github-actions][docker] misc. improvements to docker image build (#793) This commit makes various improvements to the build of the siliconlabsinc/ot-efr32-dev docker image. - Remove usage of openthread/script/bootstrap. This will provide more control over what goes into the docker image - Add sanity checks after building the docker image - Link ARM toolchain and SLC executables to /usr/local/bin if the bootstrap scripts are run as root - General improvements to Dockerfile --- .dockerignore | 1 + .github/workflows/build.yml | 4 +- .github/workflows/docker.yml | 42 +++++++++++------- Dockerfile => docker/Dockerfile | 15 ++++--- docker/test-ot-efr32-dev.yml | 13 ++++++ script/bootstrap | 79 ++++++++++++++++++++------------- script/bootstrap_silabs | 14 ++++++ 7 files changed, 111 insertions(+), 57 deletions(-) rename Dockerfile => docker/Dockerfile (77%) create mode 100644 docker/test-ot-efr32-dev.yml diff --git a/.dockerignore b/.dockerignore index 529de381..d51be7a0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,4 @@ .git **/build/ +**/slc_cli/ third_party/silabs/ diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 85c15752..5a428645 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -46,6 +46,7 @@ jobs: runs-on: ubuntu-22.04 container: image: siliconlabsinc/ot-efr32-dev:latest + options: --user 1001 strategy: fail-fast: false matrix: @@ -75,9 +76,8 @@ jobs: - name: Git LFS Pull run: git -C third_party/silabs/gecko_sdk lfs pull - - name: Bootstrap + - name: Bootstrap ARM Toolchain run: | - script/bootstrap packages script/bootstrap arm_toolchain ${{ matrix.gcc_download_url }} ${{ matrix.gcc_extract_dir }} - name: Build diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 222542f3..d29cdbe1 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -49,8 +49,8 @@ permissions: # added using https://github.com/step-security/secure-workflows contents: read jobs: - docker: - name: Build docker image + build: + name: Build Docker Image runs-on: ubuntu-22.04 steps: - name: Harden Runner @@ -62,19 +62,6 @@ jobs: with: submodules: true - - name: Create LFS file hash list - run: git -C third_party/silabs/gecko_sdk lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-assets-id - - - name: Restore gecko_sdk LFS cache - uses: actions/cache@v4 - id: lfs-cache - with: - path: .git/modules/third_party/silabs/gecko_sdk/lfs - key: lfs-${{ hashFiles('.lfs-assets-id') }} - - - name: Git LFS Pull - run: git -C third_party/silabs/gecko_sdk lfs pull - - name: Set up Docker Buildx uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 @@ -82,6 +69,7 @@ jobs: uses: docker/build-push-action@v5 with: context: . + file: docker/Dockerfile load: true build-args: | - BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') @@ -92,9 +80,31 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max + - name: Container image sanity checks + run: | + # Download container-structure-test + curl -LO https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-amd64 \ + && chmod +x container-structure-test-linux-amd64 + + # Run tests + ./container-structure-test-linux-amd64 test --config docker/test-ot-efr32-dev.yml --image ${{ env.TEST_TAG }} + + - name: Create LFS file hash list + run: git -C third_party/silabs/gecko_sdk lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-assets-id + + - name: Restore gecko_sdk LFS cache + uses: actions/cache@v4 + id: lfs-cache + with: + path: .git/modules/third_party/silabs/gecko_sdk/lfs + key: lfs-${{ hashFiles('.lfs-assets-id') }} + + - name: Git LFS Pull + run: git -C third_party/silabs/gecko_sdk lfs pull + - name: Test build inside container run: | - docker run -v ${{ github.workspace }}:/ot-efr32/ --rm ${{ env.TEST_TAG }} script/build --skip-silabs-apps brd4151a + docker run -v ${{ github.workspace }}:/ot-efr32/ --user $(id -u) --rm ${{ env.TEST_TAG }} script/build --skip-silabs-apps brd4151a - name: Login to DockerHub uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 diff --git a/Dockerfile b/docker/Dockerfile similarity index 77% rename from Dockerfile rename to docker/Dockerfile index c1321d87..b9d9d6bd 100644 --- a/Dockerfile +++ b/docker/Dockerfile @@ -12,21 +12,23 @@ RUN apt-get update && \ && rm -rf /var/lib/apt/lists/* # Copy scripts -COPY ./openthread/script ./openthread/script COPY ./script/bootstrap \ - ./script/bootstrap_silabs \ ./script/ -COPY ./requirements.txt . -# Bootstrap +# Install system packages and ARM toolchain RUN ./script/bootstrap packages && rm -rf /var/lib/apt/lists/* +RUN ./script/bootstrap arm_toolchain -RUN ./script/bootstrap openthread && rm -rf /var/lib/apt/lists/* +# Install Python packages +COPY ./requirements.txt . +RUN ./script/bootstrap python -# Label the build date before downloading slc to force slc to always be download during a docker build +# Label the build date before downloading slc to force slc to always be downloaded during a docker build ARG BUILD_DATE LABEL build_date=${BUILD_DATE} +COPY ./script/bootstrap_silabs \ + ./script/ ENV SLC_INSTALL_DIR=/opt/slc_cli RUN mkdir ${SLC_INSTALL_DIR} && \ ./script/bootstrap silabs @@ -37,4 +39,3 @@ ARG REPO_URL="https://github.com/openthread/ot-efr32" WORKDIR / RUN rm -rf ${repo_dir} && git clone ${REPO_URL} ${repo_dir} WORKDIR ${repo_dir} - diff --git a/docker/test-ot-efr32-dev.yml b/docker/test-ot-efr32-dev.yml new file mode 100644 index 00000000..068f5b51 --- /dev/null +++ b/docker/test-ot-efr32-dev.yml @@ -0,0 +1,13 @@ +schemaVersion: '2.0.0' +commandTests: + - name: 'Check arm-eabi-none-gcc version' + command: "arm-none-eabi-gcc" + args: ["--version"] + expectedOutput: ["arm-none-eabi-gcc .* 12.[0-9]+.[0-9]+"] + - name: 'Check if slc-cli is installed' + command: "slc" + args: ["--version"] + - name: 'Check if git-lfs is installed' + command: "git" + args: ["lfs", "--version"] + expectedOutput: ["^git-lfs/[0-9]+.[0-9]+.[0-9]+"] diff --git a/script/bootstrap b/script/bootstrap index 260d512e..8700bedf 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Copyright (c) 2021, The OpenThread Authors. # All rights reserved. @@ -41,22 +41,25 @@ script_dir="$(realpath "$(dirname "${script_path}")")" repo_dir="$(dirname "${script_dir}")" # ============================================================================== - install_packages_apt() { echo 'Installing script dependencies...' - # apt-get update and install dependencies + # Install dependencies sudo apt-get update - sudo apt-get --no-install-recommends install -y \ - coreutils \ - openjdk-17-jre \ - python3-setuptools \ - python3-pip \ - git-lfs \ - unzip \ - wget \ + apt_packages=( + coreutils + git + git-lfs + openjdk-17-jre-headless + ninja-build + python3-pip + python3-setuptools + unzip + wget xz-utils + ) + sudo apt-get --no-install-recommends install -y "${apt_packages[@]}" } install_packages_opkg() @@ -72,11 +75,18 @@ install_packages_rpm() install_packages_brew() { echo 'Installing script dependencies...' - brew install \ - coreutils \ - openjdk@11 \ - grep \ + + brew_packages=( + coreutils + git + git-lfs + grep + ninja-build + openjdk@17 + unzip wget + ) + brew install "${brew_packages[@]}" } install_packages_source() @@ -102,16 +112,6 @@ install_packages() PM=brew fi install_packages_$PM - - if command -v pip3; then - install_packages_pip3 - fi -} - -do_bootstrap_openthread() -{ - echo "Bootstrapping openthread" - "${repo_dir}"/openthread/script/bootstrap } do_bootstrap_silabs() @@ -125,7 +125,13 @@ install_arm_toolchain() local url=${1-"https://developer.arm.com/-/media/Files/downloads/gnu/12.2.rel1/binrel/arm-gnu-toolchain-12.2.rel1-x86_64-arm-none-eabi.tar.xz"} local extract_dir=${2-"arm-gnu-toolchain-12.2.rel1-x86_64-arm-none-eabi"} - local toolchain_dir="${HOME}/.local" + local toolchain_dir + # Link if root + if [ "$EUID" -eq 0 ]; then + toolchain_dir=/opt + else + toolchain_dir="${HOME}/.local" + fi # Check if the toolchain is already present at the desired location if command -v "${toolchain_dir}/${extract_dir}/bin/arm-none-eabi-gcc"; then @@ -144,8 +150,19 @@ install_arm_toolchain() # Extract tar xf "${toolchain_dir}/${tarball}" --directory "${toolchain_dir}/${extract_dir}" --strip-components=1 - # Link - sudo ln -s -f "${toolchain_dir}"/"${extract_dir}"/bin/* /usr/local/bin/ + # Link if root + if [ "$EUID" -eq 0 ]; then + ln -s -f "${toolchain_dir}"/"${extract_dir}"/bin/* /usr/local/bin/ + ls -alh /usr/local/bin/arm* + else + set +x + echo "" + echo "The bootstrap script was run without sudo" + echo "" + echo "Make sure to add '${toolchain_dir}/${extract_dir}/bin/' to PATH" + echo "" + set -x + fi # Cleanup rm -rf "${toolchain_dir:?}/${tarball:?}" @@ -155,21 +172,19 @@ main() { if [ $# == 0 ]; then install_packages - do_bootstrap_openthread + install_packages_pip3 do_bootstrap_silabs elif [ "$1" == 'packages' ]; then install_packages elif [ "$1" == 'arm_toolchain' ]; then shift 1 install_arm_toolchain "$@" - elif [ "$1" == 'openthread' ]; then - do_bootstrap_openthread elif [ "$1" == 'python' ]; then install_packages_pip3 elif [ "$1" == 'silabs' ]; then do_bootstrap_silabs else - echo >&2 "Unsupported action: $1. Supported: openthread, silabs" + echo >&2 "Unsupported action: $1. Supported: packages, arm_toolchain, python, silabs" # 128 for Invalid arguments exit 128 fi diff --git a/script/bootstrap_silabs b/script/bootstrap_silabs index b45aa775..0551bb39 100755 --- a/script/bootstrap_silabs +++ b/script/bootstrap_silabs @@ -61,6 +61,20 @@ install_slc_cli() wget "${zip_url}" -O "${zip_path}" unzip -q -o "${zip_path}" -d "$(dirname ${zip_path})" && rm "${zip_path}" popd + + # Link if root + if [ "$EUID" -eq 0 ]; then + ln -s -f "${SLC_INSTALL_DIR}"/slc_cli/slc* /usr/local/bin/ + ls -alh /usr/local/bin/slc + else + set +x + echo "" + echo "The bootstrap script was run without sudo" + echo "" + echo "Make sure to add '${SLC_INSTALL_DIR}/slc_cli/' to PATH" + echo "" + set -x + fi } main()