From ddebd15e560d7a50ebd18d48132238ff6cb7cc9f Mon Sep 17 00:00:00 2001 From: Alvin Schiller <103769832+AlvinSchiller@users.noreply.github.com> Date: Fri, 20 Oct 2023 10:16:58 +0200 Subject: [PATCH] Unified Test runs in Docker (#2082) * removed obsolete v3.x workflow files * set user and group as build args to test altuser removed now obsolete "*_altuser" Dockerfile * set GIT_BRANCH and GIT_URL as build args makes sure that the installation script uses the current repo/branch * use correct variables for refname resolving * harmonized workflow and Dockerfiles * optimize test docker performance. copy changed only include the scripts that are needed. Otherwise every code change will invalidate the container cache. changed internal path of container and test scripts * optimize test docker performance. refactored changed thestructure to use the cache as much as possible. repo url -> packages -> user -> scripts * changed cache key with ref instead of ref_name pull request will get a different cache * get username with whoami to be compatible with containers * implemented different strategies for uniformity * set noninteractive on debconf get rid of warning "debconf: unable to initialize frontend: Dialog" export DEBIAN_FRONTEND=noninteractive does not work with sudo * unified workflow for debian. delete obsolete files * fix branch name on pr * refactored into subworkflow * fixed apt-key deprecation * unified Dockerfile. renamed workflow files * fix repo name on pullrequest --- .github/workflows/docker_bullseye.yml | 66 ------- .github/workflows/docker_bullseye_altuser.yml | 66 ------- .github/workflows/docker_buster.yml | 66 ------- .github/workflows/pythonpackage_future3.yml | 58 ------ .github/workflows/test_docker_debian.yml | 37 ++++ .../test_docker_debian_versionname_sub.yml | 172 ++++++++++++++++++ ci/Dockerfile.bullseye.test_install.amd64 | 35 ---- ci/Dockerfile.bullseye.test_install.armv7 | 37 ---- ...erfile.bullseye.test_install_altuser.armv7 | 37 ---- ci/Dockerfile.buster.amd64 | 32 ---- ci/Dockerfile.buster.armv7 | 32 ---- ci/Dockerfile.buster.test_install.armv7 | 37 ---- ci/Dockerfile.debian | 101 ++++++++++ ci/README.md | 45 ++--- ...buster-install-default-with-autohotspot.sh | 6 +- .../installscripts/buster-install-default.sh | 8 +- .../tests/run_installation_tests.sh | 8 +- .../tests/run_installation_tests2.sh | 6 +- .../tests/run_installation_tests3.sh | 6 +- .../installscripts/tests/test_installation.sh | 4 +- 20 files changed, 348 insertions(+), 511 deletions(-) delete mode 100644 .github/workflows/docker_bullseye.yml delete mode 100644 .github/workflows/docker_bullseye_altuser.yml delete mode 100644 .github/workflows/docker_buster.yml delete mode 100644 .github/workflows/pythonpackage_future3.yml create mode 100644 .github/workflows/test_docker_debian.yml create mode 100644 .github/workflows/test_docker_debian_versionname_sub.yml delete mode 100644 ci/Dockerfile.bullseye.test_install.amd64 delete mode 100644 ci/Dockerfile.bullseye.test_install.armv7 delete mode 100644 ci/Dockerfile.bullseye.test_install_altuser.armv7 delete mode 100644 ci/Dockerfile.buster.amd64 delete mode 100644 ci/Dockerfile.buster.armv7 delete mode 100644 ci/Dockerfile.buster.test_install.armv7 create mode 100644 ci/Dockerfile.debian diff --git a/.github/workflows/docker_bullseye.yml b/.github/workflows/docker_bullseye.yml deleted file mode 100644 index e15ae2173..000000000 --- a/.github/workflows/docker_bullseye.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: Test Install Scripts for Bullseye on Docker - -on: - schedule: - # run at 5 every sunday - - cron: '0 5 * * 0' - push: - branches-ignore: - - 'future3/**' - pull_request: - # The branches below must be a subset of the branches above - branches: [ develop ] - -jobs: - - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 - - - name: Build Bullseye ARMv7 - uses: docker/build-push-action@v5 - with: - context: . - load: true - push: false - file: ./ci/Dockerfile.bullseye.test_install.armv7 - platforms: linux/arm/v7 - tags: rpi-jukebox-rfid-bullseye:latest - cache-from: type=gha,scope=$GITHUB_REF_NAME-bullseye - cache-to: type=gha,mode=max,scope=$GITHUB_REF_NAME-bullseye - - - name: Run run_installation_tests.sh Bullseye ARMv7 - uses: tj-actions/docker-run@v2 - with: - image: rpi-jukebox-rfid-bullseye:latest - name: bullseye_run_installation_tests.sh - options: --platform linux/arm/v7 - args: | - /code/scripts/installscripts/tests/run_installation_tests.sh - - - name: Run run_installation_tests2.sh Bullseye ARMv7 - uses: tj-actions/docker-run@v2 - with: - image: rpi-jukebox-rfid-bullseye:latest - name: bullseye_run_installation_tests2.sh - options: --platform linux/arm/v7 - args: | - /code/scripts/installscripts/tests/run_installation_tests2.sh - - - name: Run run_installation_tests3.sh Bullseye ARMv7 - uses: tj-actions/docker-run@v2 - with: - image: rpi-jukebox-rfid-bullseye:latest - name: bullseye_run_installation_tests3.sh - options: --platform linux/arm/v7 - args: | - /code/scripts/installscripts/tests/run_installation_tests3.sh diff --git a/.github/workflows/docker_bullseye_altuser.yml b/.github/workflows/docker_bullseye_altuser.yml deleted file mode 100644 index 59e6d91d6..000000000 --- a/.github/workflows/docker_bullseye_altuser.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: Test Install Scripts for Bullseye (alternative user) on Docker - -on: - schedule: - # run at 5 every sunday - - cron: '0 5 * * 0' - push: - branches-ignore: - - 'future3/**' - pull_request: - # The branches below must be a subset of the branches above - branches: [ develop ] - -jobs: - - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 - - - name: Build altuser Bullseye ARMv7 - uses: docker/build-push-action@v5 - with: - context: . - load: true - push: false - file: ./ci/Dockerfile.bullseye.test_install_altuser.armv7 - platforms: linux/arm/v7 - tags: rpi-jukebox-rfid-bullseye-altuser:latest - cache-from: type=gha,scope=$GITHUB_REF_NAME-bullseye-altuser - cache-to: type=gha,mode=max,scope=$GITHUB_REF_NAME-bullseye-altuser - - - name: Run run_installation_tests.sh Bullseye ARMv7 - uses: tj-actions/docker-run@v2 - with: - image: rpi-jukebox-rfid-bullseye-altuser:latest - options: --platform linux/arm/v7 - name: run_installation_tests.sh - args: | - /code/scripts/installscripts/tests/run_installation_tests.sh - - - name: Run run_installation_tests2.sh Bullseye ARMv7 - uses: tj-actions/docker-run@v2 - with: - image: rpi-jukebox-rfid-bullseye-altuser:latest - options: --platform linux/arm/v7 - name: run_installation_tests2.sh - args: | - /code/scripts/installscripts/tests/run_installation_tests2.sh - - - name: Run run_installation_tests3.sh Bullseye ARMv7 - uses: tj-actions/docker-run@v2 - with: - image: rpi-jukebox-rfid-bullseye-altuser:latest - options: --platform linux/arm/v7 - name: run_installation_tests3.sh - args: | - /code/scripts/installscripts/tests/run_installation_tests3.sh diff --git a/.github/workflows/docker_buster.yml b/.github/workflows/docker_buster.yml deleted file mode 100644 index 69690ed4f..000000000 --- a/.github/workflows/docker_buster.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: Test Install Scripts for Buster on Docker - -on: - schedule: - # run at 5 every sunday - - cron: '0 5 * * 0' - push: - branches-ignore: - - 'future3/**' - pull_request: - # The branches below must be a subset of the branches above - branches: [ develop ] - -jobs: - - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 - - - name: Build Buster ARMv7 - uses: docker/build-push-action@v5 - with: - context: . - load: true - push: false - file: ./ci/Dockerfile.buster.test_install.armv7 - platforms: linux/arm/v7 - tags: rpi-jukebox-rfid-buster:latest - cache-from: type=gha,scope=$GITHUB_REF_NAME-buster - cache-to: type=gha,mode=max,scope=$GITHUB_REF_NAME-buster - - - name: Run run_installation_tests.sh Buster ARMv7 - uses: tj-actions/docker-run@v2 - with: - image: rpi-jukebox-rfid-buster:latest - options: --platform linux/arm/v7 - name: run_installation_tests.sh - args: | - /code/scripts/installscripts/tests/run_installation_tests.sh - - - name: Run run_installation_tests2.sh Buster ARMv7 - uses: tj-actions/docker-run@v2 - with: - image: rpi-jukebox-rfid-buster:latest - options: --platform linux/arm/v7 - name: run_installation_tests2.sh - args: | - /code/scripts/installscripts/tests/run_installation_tests2.sh - - - name: Run run_installation_tests3.sh Buster ARMv7 - uses: tj-actions/docker-run@v2 - with: - image: rpi-jukebox-rfid-buster:latest - options: --platform linux/arm/v7 - name: run_installation_tests3.sh - args: | - /code/scripts/installscripts/tests/run_installation_tests3.sh diff --git a/.github/workflows/pythonpackage_future3.yml b/.github/workflows/pythonpackage_future3.yml deleted file mode 100644 index 9844039b8..000000000 --- a/.github/workflows/pythonpackage_future3.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: Python + Docs Checks and Tests - -on: - push: - branches: - - 'future3/**' - paths: - - '**.py' - - '**.py.*' - - 'docs/sphinx/**' - pull_request: - branches: - - 'future3/**' - paths: - - '**.py' - - '**.py.*' - - 'docs/sphinx/**' - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - max-parallel: 4 - matrix: - python-version: [3.7, 3.8] - - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y libasound2-dev - python -m pip install --upgrade pip - pip install wheel - pip install spidev - pip install -r requirements.txt - # For operation of the Jukebox, ZMQ must be compiled from sources due to Websocket support - # When just building the docs, the regular ZMQ package is sufficient - pip install -r docs/sphinx/requirements_pyzmq.txt - pip install -r docs/sphinx/requirements.txt - # Also install all optional dependencies - pip install -r src/jukebox/components/rfid/fake_reader_gui/requirements.txt - - name: Lint with flake8 - run: | - pip install flake8 - # Stop the build if linting fails - ./run_flake8.sh - - name: Build the docs - working-directory: ./docs/sphinx - run: | - # Stop the build if documentation cannot be built - # Treat all warnings as errors - sphinx-build -W --keep-going -T -a -E -b html . _build diff --git a/.github/workflows/test_docker_debian.yml b/.github/workflows/test_docker_debian.yml new file mode 100644 index 000000000..ab0b9ec8a --- /dev/null +++ b/.github/workflows/test_docker_debian.yml @@ -0,0 +1,37 @@ +name: Test Install Scripts Debian + +on: + schedule: + # run at 5:00 every sunday + - cron: '0 5 * * 0' + push: + branches-ignore: + - 'future3/**' + pull_request: + # The branches below must be a subset of the branches above + branches: [ develop ] + +# let only one instance run the test so cache is not corrupted. +# cancel already running instances as only the last run will be relevant +concurrency: + group: ${{ github.ref }}-test-debian + cancel-in-progress: true + +jobs: + + # Build container and run tests + run: + name: ${{ matrix.debian_version_name }} + strategy: + fail-fast: false + matrix: + debian_version_name: ['bullseye', 'buster'] + uses: ./.github/workflows/test_docker_debian_versionname_sub.yml + with: + runs_on: ubuntu-latest + platform: linux/arm/v7 + docker_image_name: rpi-jukebox-rfid + cache_scope: ${{ github.ref }}-test-debian + matrix_usernames: "['pi', 'hans']" + matrix_test_scripts: "['run_installation_tests.sh', 'run_installation_tests2.sh', 'run_installation_tests3.sh']" + debian_version_name: ${{ matrix.debian_version_name }} diff --git a/.github/workflows/test_docker_debian_versionname_sub.yml b/.github/workflows/test_docker_debian_versionname_sub.yml new file mode 100644 index 000000000..24be6ef74 --- /dev/null +++ b/.github/workflows/test_docker_debian_versionname_sub.yml @@ -0,0 +1,172 @@ +name: Subworkflow Test Install Scripts Debian + +on: + workflow_call: + inputs: + runs_on: + required: true + type: string + platform: + required: true + type: string + debian_version_name: + required: true + type: string + cache_scope: + required: true + type: string + docker_image_name: + required: true + type: string + matrix_usernames: + required: true + type: string + matrix_test_scripts: + required: true + type: string + local_registry_port: + required: false + type: number + default: 5000 + +# let only one instance run the test so cache is not corrupted. +# cancel already running instances as only the last run will be relevant +concurrency: + group: ${{ inputs.cache_scope }}-${{ inputs.debian_version_name }} + cancel-in-progress: true + +jobs: + + # Build container for test execution + build: + runs-on: ${{ inputs.runs_on }} + + outputs: + cache_key: ${{ steps.vars.outputs.cache_key }} + image_file_path: ${{ steps.vars.outputs.image_file_path }} + image_tag_name: ${{ steps.vars.outputs.image_tag_name }} + + # create local docker registry to use locally build images + services: + registry: + image: registry:2 + ports: + - ${{ inputs.local_registry_port }}:5000 + + steps: + - uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.0.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.0.0 + with: + # network=host driver-opt needed to push to local registry + driver-opts: network=host + + - name: Set Output pre-vars + id: pre-vars + env: + DEBIAN_VERSION_NAME: ${{ inputs.debian_version_name }} + DOCKER_IMAGE_NAME: ${{ inputs.docker_image_name }} + CACHE_SCOPE: ${{ inputs.cache_scope }} + run: | + echo "image_tag_name=${{ env.DOCKER_IMAGE_NAME }}:${{ env.DEBIAN_VERSION_NAME }}-test" >> $GITHUB_OUTPUT + echo "cache_scope=${{ env.CACHE_SCOPE }}-${{ env.DEBIAN_VERSION_NAME }}" >> $GITHUB_OUTPUT + + - name: Set Output vars + id: vars + env: + DEBIAN_VERSION_NAME: ${{ inputs.debian_version_name }} + DOCKER_IMAGE_NAME: ${{ inputs.docker_image_name }} + LOCAL_REGISTRY_PORT: ${{ inputs.local_registry_port }} + run: | + echo "image_tag_name=${{ steps.pre-vars.outputs.image_tag_name }}" >> $GITHUB_OUTPUT + echo "image_tag_name_local_base=localhost:${{ env.LOCAL_REGISTRY_PORT }}/${{ steps.pre-vars.outputs.image_tag_name }}-base" >> $GITHUB_OUTPUT + echo "image_file_path=./${{ env.DOCKER_IMAGE_NAME }}-${{ env.DEBIAN_VERSION_NAME }}.tar" >> $GITHUB_OUTPUT + echo "cache_scope=${{ steps.pre-vars.outputs.cache_scope }}" >> $GITHUB_OUTPUT + echo "cache_key=${{ steps.pre-vars.outputs.cache_scope }}-${{ github.sha }}#${{ github.run_attempt }}" >> $GITHUB_OUTPUT + + # Build base image for debian version name. Layers will be cached and image pushes to local registry + - name: Build Image - Base + uses: docker/build-push-action@v5 + with: + context: . + load: false + push: true + file: ./ci/Dockerfile.debian + target: test-code + platforms: ${{ inputs.platform }} + tags: ${{ steps.vars.outputs.image_tag_name_local_base }} + cache-from: type=gha,scope=${{ steps.vars.outputs.cache_scope }} + cache-to: type=gha,mode=max,scope=${{ steps.vars.outputs.cache_scope }} + build-args: | + DEBIAN_VERSION_NAME=${{ inputs.debian_version_name }} + GIT_BRANCH=${{ github.head_ref || github.ref_name }} + GIT_URL=${{ github.server_url }}/${{ github.event.pull_request.head.repo.full_name || github.repository }} + + # Build new image with updates packages based on base image. Layers will NOT be chached. Result is written to file. + - name: Build Image - Update + uses: docker/build-push-action@v5 + with: + context: . + load: false + push: false + file: ./ci/Dockerfile.debian + target: test-update + platforms: ${{ inputs.platform }} + tags: ${{ steps.vars.outputs.image_tag_name }} + cache-from: type=gha,scope=${{ steps.vars.outputs.cache_scope }} + # DON'T use 'cache-to' here as the layer is then cached and this build would be useless + outputs: type=docker,dest=${{ steps.vars.outputs.image_file_path }} + build-args: | + BASE_TEST_IMAGE=${{ steps.vars.outputs.image_tag_name_local_base }} + + # Cache image file for next jobs + - name: Cache Save Docker Image + uses: actions/cache/save@v3 + with: + key: ${{ steps.vars.outputs.cache_key }} + path: ${{ steps.vars.outputs.image_file_path }} + + + # Run tests with build image + test: + needs: [build] + runs-on: ${{ inputs.runs_on }} + + strategy: + fail-fast: false + matrix: + username: ${{ fromJSON(inputs.matrix_usernames) }} + test_script: ${{ fromJSON(inputs.matrix_test_scripts) }} + + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3.0.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.0.0 + + # Load cached image file + - name: Cache Restore Docker Image + uses: actions/cache/restore@v3 + with: + key: ${{ needs.build.outputs.cache_key }} + path: ${{ needs.build.outputs.image_file_path }} + fail-on-cache-miss: true + + - name: Load Docker Image + run: | + docker load --input ${{ needs.build.outputs.image_file_path }} + + # Run test + - name: Run Test ${{ inputs.debian_version_name }}-${{ matrix.username }}-${{ matrix.test_script }} + uses: tj-actions/docker-run@v2 + with: + image: ${{ needs.build.outputs.image_tag_name }} + options: --platform ${{inputs.platform }} --user ${{ matrix.username }} + name: ${{ matrix.test_script }} + args: | + ./${{ matrix.test_script }} diff --git a/ci/Dockerfile.bullseye.test_install.amd64 b/ci/Dockerfile.bullseye.test_install.amd64 deleted file mode 100644 index 971c03537..000000000 --- a/ci/Dockerfile.bullseye.test_install.amd64 +++ /dev/null @@ -1,35 +0,0 @@ -FROM debian:bullseye -ENV DOCKER_RUNNING=true -ENV USER=pi - -COPY . /code -WORKDIR /code - -RUN groupadd --gid 1000 $USER ;\ - useradd -u 1000 -g 1000 -G sudo -d /home/$USER -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER ;\ - chown -R 1000:1000 /code /home/$USER ;\ - chmod +x /code/scripts/installscripts/buster-install-default.sh ;\ - chmod +x /code/scripts/installscripts/tests/run_installation_tests.sh ;\ - chmod +x /code/scripts/installscripts/tests/run_installation_tests2.sh ;\ - chmod +x /code/scripts/installscripts/tests/run_installation_tests3.sh - -RUN export DEBIAN_FRONTEND=noninteractive ;\ - apt-get update ;\ - apt-get -y install curl gnupg sudo nano systemd apt-utils;\ - echo 'deb http://raspbian.raspberrypi.org/raspbian/ bullseye main contrib non-free rpi' > /etc/apt/sources.list.d/raspi.list ;\ - echo 'deb http://archive.raspberrypi.org/debian/ bullseye main' >> /etc/apt/sources.list.d/raspi.list ;\ - curl http://raspbian.raspberrypi.org/raspbian.public.key | apt-key add - ;\ - curl http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | apt-key add - ;\ - echo "$USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER ;\ - apt-get clean ;\ - rm -rf /var/cache/apt/* /var/lib/apt/lists/* - -RUN export DEBIAN_FRONTEND=noninteractive ;\ - apt-get update ;\ - apt-get -y dist-upgrade --auto-remove --purge ;\ - apt-get -y install wget build-essential git iw locales wpasupplicant;\ - apt-get clean ;\ - touch /boot/cmdlinetxt ;\ - rm -rf /var/cache/apt/* /var/lib/apt/lists/* - -USER $USER diff --git a/ci/Dockerfile.bullseye.test_install.armv7 b/ci/Dockerfile.bullseye.test_install.armv7 deleted file mode 100644 index d7fbf9b97..000000000 --- a/ci/Dockerfile.bullseye.test_install.armv7 +++ /dev/null @@ -1,37 +0,0 @@ -FROM --platform=linux/arm/v7 arm32v7/debian:bullseye-slim -ENV DOCKER_RUNNING=true -ENV USER=pi - -COPY . /code -WORKDIR /code - -RUN groupadd --gid 1000 $USER ;\ - useradd -u 1000 -g 1000 -G sudo -d /home/$USER -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER ;\ - chown -R 1000:1000 /code /home/$USER ;\ - chmod +x /code/scripts/installscripts/buster-install-default.sh ;\ - chmod +x /code/scripts/installscripts/tests/run_installation_tests.sh ;\ - chmod +x /code/scripts/installscripts/tests/run_installation_tests2.sh ;\ - chmod +x /code/scripts/installscripts/tests/run_installation_tests3.sh - -RUN export DEBIAN_FRONTEND=noninteractive ;\ - apt-get update ;\ - apt-get -y install curl gnupg sudo nano systemd apt-utils;\ - echo 'deb http://raspbian.raspberrypi.org/raspbian/ bullseye main contrib non-free rpi' > /etc/apt/sources.list.d/raspi.list ;\ - echo 'deb http://archive.raspberrypi.org/debian/ bullseye main' >> /etc/apt/sources.list.d/raspi.list ;\ - curl http://raspbian.raspberrypi.org/raspbian.public.key | apt-key add - ;\ - curl http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | apt-key add - ;\ - echo "$USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER ;\ - apt-get clean ;\ - rm -rf /var/cache/apt/* /var/lib/apt/lists/* - -RUN export DEBIAN_FRONTEND=noninteractive ;\ - apt-get update ;\ - # install here to speed up GitHub Action - apt-get -y install raspberrypi-kernel raspberrypi-kernel-headers;\ - apt-get -y dist-upgrade --auto-remove --purge ;\ - apt-get -y install wget build-essential git iw locales wpasupplicant;\ - apt-get clean ;\ - touch /boot/cmdlinetxt ;\ - rm -rf /var/cache/apt/* /var/lib/apt/lists/* - -USER $USER diff --git a/ci/Dockerfile.bullseye.test_install_altuser.armv7 b/ci/Dockerfile.bullseye.test_install_altuser.armv7 deleted file mode 100644 index 9043d7308..000000000 --- a/ci/Dockerfile.bullseye.test_install_altuser.armv7 +++ /dev/null @@ -1,37 +0,0 @@ -FROM --platform=linux/arm/v7 arm32v7/debian:bullseye-slim -ENV DOCKER_RUNNING=true -ENV USER=hans - -COPY . /code -WORKDIR /code - -RUN groupadd --gid 1000 wurst ;\ - useradd -u 1000 -g 1000 -G sudo -d /home/$USER -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER ;\ - chown -R 1000:1000 /code /home/$USER ;\ - chmod +x /code/scripts/installscripts/buster-install-default.sh ;\ - chmod +x /code/scripts/installscripts/tests/run_installation_tests.sh ;\ - chmod +x /code/scripts/installscripts/tests/run_installation_tests2.sh ;\ - chmod +x /code/scripts/installscripts/tests/run_installation_tests3.sh - -RUN export DEBIAN_FRONTEND=noninteractive ;\ - apt-get update ;\ - apt-get -y install curl gnupg sudo nano systemd apt-utils;\ - # install here to speed up GitHub Action - apt-get -y install raspberrypi-kernel-headers;\ - echo 'deb http://raspbian.raspberrypi.org/raspbian/ bullseye main contrib non-free rpi' > /etc/apt/sources.list.d/raspi.list ;\ - echo 'deb http://archive.raspberrypi.org/debian/ bullseye main' >> /etc/apt/sources.list.d/raspi.list ;\ - curl http://raspbian.raspberrypi.org/raspbian.public.key | apt-key add - ;\ - curl http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | apt-key add - ;\ - echo "$USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER ;\ - apt-get clean ;\ - rm -rf /var/cache/apt/* /var/lib/apt/lists/* - -RUN export DEBIAN_FRONTEND=noninteractive ;\ - apt-get update ;\ - apt-get -y dist-upgrade --auto-remove --purge ;\ - apt-get -y install wget build-essential git iw locales wpasupplicant;\ - apt-get clean ;\ - touch /boot/cmdlinetxt ;\ - rm -rf /var/cache/apt/* /var/lib/apt/lists/* - -USER $USER diff --git a/ci/Dockerfile.buster.amd64 b/ci/Dockerfile.buster.amd64 deleted file mode 100644 index 9c1ff6e09..000000000 --- a/ci/Dockerfile.buster.amd64 +++ /dev/null @@ -1,32 +0,0 @@ -FROM debian:buster -ENV DOCKER_RUNNING=true -ENV USER=pi - -COPY . /code -WORKDIR /code - -RUN groupadd --gid 1000 $USER ;\ - useradd -u 1000 -g 1000 -G sudo -d /home/$USER -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER ;\ - chown -R 1000:1000 /code /home/$USER ;\ - chmod +x /code/scripts/installscripts/buster-install-default.sh - -RUN export DEBIAN_FRONTEND=noninteractive ;\ - apt-get update ;\ - apt-get -y install curl gnupg sudo nano;\ - echo 'deb http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi' > /etc/apt/sources.list.d/raspi.list ;\ - echo 'deb http://archive.raspberrypi.org/debian/ buster main' >> /etc/apt/sources.list.d/raspi.list ;\ - curl http://raspbian.raspberrypi.org/raspbian.public.key | apt-key add - ;\ - curl http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | apt-key add - ;\ - echo "$USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER ;\ - apt-get clean ;\ - rm -rf /var/cache/apt/* /var/lib/apt/lists/* - -RUN export DEBIAN_FRONTEND=noninteractive ;\ - apt-get update ;\ - apt-get -y dist-upgrade --auto-remove --purge ;\ - apt-get -y install wget build-essential git iw locales wpasupplicant;\ - apt-get clean ;\ - touch /boot/cmdline.txt /etc/sysctl.conf ;\ - rm -rf /var/cache/apt/* /var/lib/apt/lists/* - -USER $USER diff --git a/ci/Dockerfile.buster.armv7 b/ci/Dockerfile.buster.armv7 deleted file mode 100644 index 3c6f2841a..000000000 --- a/ci/Dockerfile.buster.armv7 +++ /dev/null @@ -1,32 +0,0 @@ -FROM arm32v7/debian:buster-slim -ENV DOCKER_RUNNING=true -ENV USER=pi - -COPY . /code -WORKDIR /code - -RUN groupadd --gid 1000 $USER ;\ - useradd -u 1000 -g 1000 -G sudo -d /home/$USER -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER ;\ - chown -R 1000:1000 /code /home/$USER ;\ - chmod +x /code/scripts/installscripts/buster-install-default.sh - -RUN export DEBIAN_FRONTEND=noninteractive ;\ - apt-get update ;\ - apt-get -y install curl gnupg sudo nano;\ - echo 'deb http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi' >> /etc/apt/sources.list.d/raspi.list ;\ - echo 'deb http://archive.raspberrypi.org/debian/ buster main' > /etc/apt/sources.list.d/raspi.list ;\ - curl http://raspbian.raspberrypi.org/raspbian.public.key | apt-key add - ;\ - curl http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | apt-key add - ;\ - echo "$USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER ;\ - apt-get clean ;\ - rm -rf /var/cache/apt/* /var/lib/apt/lists/* - -RUN export DEBIAN_FRONTEND=noninteractive ;\ - apt-get update ;\ - apt-get -y dist-upgrade --auto-remove --purge ;\ - apt-get -y install wget build-essential git iw locales wpasupplicant ;\ - apt-get clean ;\ - touch /boot/cmdline.txt /etc/sysctl.conf ;\ - rm -rf /var/cache/apt/* /var/lib/apt/lists/* - -USER $USER diff --git a/ci/Dockerfile.buster.test_install.armv7 b/ci/Dockerfile.buster.test_install.armv7 deleted file mode 100644 index f3cd3f3e6..000000000 --- a/ci/Dockerfile.buster.test_install.armv7 +++ /dev/null @@ -1,37 +0,0 @@ -FROM --platform=linux/arm/v7 arm32v7/debian:buster-slim -ENV DOCKER_RUNNING=true -ENV USER=pi - -COPY . /code -WORKDIR /code - -RUN groupadd --gid 1000 pi ;\ - useradd -u 1000 -g 1000 -G sudo -d /home/$USER -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER ;\ - chown -R 1000:1000 /code /home/$USER ;\ - chmod +x /code/scripts/installscripts/buster-install-default.sh ;\ - chmod +x /code/scripts/installscripts/tests/run_installation_tests.sh ;\ - chmod +x /code/scripts/installscripts/tests/run_installation_tests2.sh ;\ - chmod +x /code/scripts/installscripts/tests/run_installation_tests3.sh - -RUN export DEBIAN_FRONTEND=noninteractive ;\ - apt-get update ;\ - apt-get -y install curl gnupg sudo nano systemd apt-utils;\ - # install here to speed up GitHub Action - apt-get -y install raspberrypi-kernel-headers;\ - echo 'deb http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi' > /etc/apt/sources.list.d/raspi.list ;\ - echo 'deb http://archive.raspberrypi.org/debian/ buster main' >> /etc/apt/sources.list.d/raspi.list ;\ - curl http://raspbian.raspberrypi.org/raspbian.public.key | apt-key add - ;\ - curl http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | apt-key add - ;\ - echo "$USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER ;\ - apt-get clean ;\ - rm -rf /var/cache/apt/* /var/lib/apt/lists/* - -RUN export DEBIAN_FRONTEND=noninteractive ;\ - apt-get update ;\ - apt-get -y dist-upgrade --auto-remove --purge ;\ - apt-get -y install wget build-essential git iw locales wpasupplicant;\ - apt-get clean ;\ - touch /boot/cmdlinetxt ;\ - rm -rf /var/cache/apt/* /var/lib/apt/lists/* - -USER $USER diff --git a/ci/Dockerfile.debian b/ci/Dockerfile.debian new file mode 100644 index 000000000..ecfca4460 --- /dev/null +++ b/ci/Dockerfile.debian @@ -0,0 +1,101 @@ +# Base Target to build and install all needed base configuration and packages. Specifie the needed platform with the docker '--platform XXX' option +ARG DEBIAN_VERSION_NAME=bullseye +ARG BASE_TEST_IMAGE=test-code +FROM debian:${DEBIAN_VERSION_NAME}-slim as base +ARG DEBIAN_VERSION_NAME + +ENV DOCKER_RUNNING=true +RUN touch /boot/cmdlinetxt + +RUN export DEBIAN_FRONTEND=noninteractive \ + && apt-get update \ + && apt-get -y install \ + apt-utils \ + curl \ + gnupg \ + && curl -fsSL http://raspbian.raspberrypi.org/raspbian.public.key | gpg --dearmor > /usr/share/keyrings/raspberrypi-raspbian-keyring.gpg \ + && curl -fsSL http://archive.raspberrypi.org/debian/raspberrypi.gpg.key | gpg --dearmor > /usr/share/keyrings/raspberrypi-archive-debian-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/raspberrypi-raspbian-keyring.gpg] http://raspbian.raspberrypi.org/raspbian/ ${DEBIAN_VERSION_NAME} main contrib non-free rpi" > /etc/apt/sources.list.d/raspi.list \ + && echo "deb [signed-by=/usr/share/keyrings/raspberrypi-archive-debian-keyring.gpg] http://archive.raspberrypi.org/debian/ ${DEBIAN_VERSION_NAME} main" >> /etc/apt/sources.list.d/raspi.list \ + && apt-get update \ + && apt-get -y upgrade \ + && apt-get -y install \ + build-essential \ + git \ + iw \ + locales \ + sudo \ + systemd \ + wget \ + wpasupplicant \ + # install internally used packages here to speed up GitHub Action + raspberrypi-kernel \ + raspberrypi-kernel-headers \ + && rm -rf /var/lib/apt/lists/* +# ------ + +# Base Target for setting up the default user. user can be selected with the docker '--user YYY' option +FROM base as user +ARG USER=pi +ARG USER_GROUP=$USER + +ENV USER=$USER USER_GROUP=$USER_GROUP + +RUN groupadd --gid 1000 $USER_GROUP \ + && useradd -u 1000 -g $USER_GROUP -G sudo -d /home/$USER -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER \ + && echo "$USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER +# ------ + + + +####### Code Targets ####### + +# Target for adding code from the repo. Set Default User +FROM user as code + +COPY --chown=$USER:$USER_GROUP --chmod=770 . /code +WORKDIR /code + +USER $USER +# ------ + + + +####### Test Targets ####### + +# Target for setting up an alternativ user 'hans:wurst'. user can be selected with the docker '--user YYY' option +FROM user as test-user + +ENV TEST_USER_GROUP=test + +RUN groupadd --gid 1002 $TEST_USER_GROUP \ + && usermod -a -G $TEST_USER_GROUP $USER + +RUN export USER_ALT=hans \ + && export USER_ALT_GROUP=wurst \ + && groupadd --gid 1001 $USER_ALT_GROUP \ + && useradd -u 1001 -g $USER_ALT_GROUP -G sudo,$TEST_USER_GROUP -d /home/$USER_ALT -m -s /bin/bash -p '$1$iV7TOwOe$6ojkJQXyEA9bHd/SqNLNj0' $USER_ALT \ + && echo "$USER_ALT ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER_ALT +# ------ + + +# Target for adding envs and scripts from the repo to test installation +FROM test-user as test-code +ARG GIT_BRANCH +ARG GIT_URL + +ENV GIT_BRANCH=$GIT_BRANCH GIT_URL=$GIT_URL + +COPY --chown=root:$TEST_USER_GROUP --chmod=770 scripts/installscripts/buster-install-default.sh . +WORKDIR /tests +COPY --chown=root:$TEST_USER_GROUP --chmod=770 scripts/installscripts/tests/*.sh . +# ------ + + +# Target for applying latest updates (should not be cached!) +FROM $BASE_TEST_IMAGE as test-update +RUN export DEBIAN_FRONTEND=noninteractive \ + && apt-get update \ + && apt-get -y upgrade \ + && rm -rf /var/lib/apt/lists/* +# ------ diff --git a/ci/README.md b/ci/README.md index 867fe1607..4ed2910e7 100644 --- a/ci/README.md +++ b/ci/README.md @@ -19,28 +19,33 @@ This is a work in progress so expect things to fail or being flaky. sudo reboot ``` * login to your RPi -* clone this repo and cd into its local clone: +* clone the repo and cd into its local clone: ```bash + cd /home/pi/ + # optional: change to your fork appropriately git clone https://github.com/MiczFlor/RPi-Jukebox-RFID.git - cd /home/pi/RPi-Jukebox-RFID/ + cd RPi-Jukebox-RFID/ + # optional: switch to another branch + git checkout ``` * build the docker image: - * **on normal PCs:** - ```bash - docker build -t rpi-jukebox-rfid-buster:latest -f ci/Dockerfile.buster.amd64 . - ``` + ```bash + docker build -t rpi-jukebox-rfid:buster-latest -f ci/Dockerfile.debian --platform=linux/arm/v7 --target=code --build-arg="DEBIAN_VERSION_NAME=buster" . + ``` + * additional arguments + * for builds + on normal PCs use `--platform=linux/amd64` + on a raspberry pi use `--platform=linux/arm/v7` + * to use a different debian version change `--build-arg="DEBIAN_VERSION_NAME=buster"` appropriately. - * **on a raspberry pi:** - ```bash - docker build -t rpi-jukebox-rfid-buster:latest -f ci/Dockerfile.buster.armv7 . - ``` * get something to drink or eat * run the freshly built docker image and start testing. For example: ```bash - docker run --rm -ti rpi-jukebox-rfid-buster:latest /bin/bash + docker run --rm -ti rpi-jukebox-rfid:buster-latest /bin/bash cd /home/pi/ cp /code/scripts/installscripts/buster-install-default.sh /home/pi/ - bash buster-install-default.sh + # set GIT_URL and GIT_BRANCH appropriately to your checkout + bash GIT_URL=https://github.com/MiczFlor/RPi-Jukebox-RFID.git GIT_BRANCH=main buster-install-default.sh ``` NOTE: Get familiar with docker and its flags - `--rm` for example will remove the @@ -48,19 +53,7 @@ This is a work in progress so expect things to fail or being flaky. ### mount hosts code as volume -The created image now contains all the code in the directory `/code` - if you do not want to -rebuild the image after each code-change you can 'mount' the RPi's code version into the -container: - -```bash - git clone https://github.com/MiczFlor/RPi-Jukebox-RFID.git - cd /home/pi/RPi-Jukebox-RFID/ - docker build -t rpi-jukebox-rfid-buster:latest -f ci/Dockerfile . - docker run --rm -ti -w /code -v $PWD:/code rpi-jukebox-rfid-buster:latest /bin/bash - - cd /home/pi/ - cp /code/scripts/installscripts/buster-install-default.sh /home/pi/ - bash buster-install-default.sh -``` +The created image now contains all the code in the directory `/code` - if you do not want to rebuild the image after each code-change you can 'mount' the RPi's code version into the container. +Add `-w /code -v $PWD:/code` to the `docker run` parameter. In that way every change to the code in the container will be available on the RPi as well as vice versa. diff --git a/scripts/installscripts/buster-install-default-with-autohotspot.sh b/scripts/installscripts/buster-install-default-with-autohotspot.sh index 917030b35..948b3ebbe 100644 --- a/scripts/installscripts/buster-install-default-with-autohotspot.sh +++ b/scripts/installscripts/buster-install-default-with-autohotspot.sh @@ -28,7 +28,7 @@ DATETIME=$(date +"%Y%m%d_%H%M%S") SCRIPTNAME="$(basename $0)" JOB="${SCRIPTNAME}" -CURRENT_USER="${SUDO_USER:-$USER}" +CURRENT_USER="${SUDO_USER:-$(whoami)}" HOME_DIR=$(getent passwd "$CURRENT_USER" | cut -d: -f6) echo "Current User: $CURRENT_USER" echo "User home dir: $HOME_DIR" @@ -97,7 +97,7 @@ checkPrerequisite() { #currently the user 'pi' is mandatory #https://github.com/MiczFlor/RPi-Jukebox-RFID/issues/1785 if [ "${CURRENT_USER}" != "pi" ]; then - echo + echo echo "ERROR: User must be 'pi'!" echo " Other usernames are currently not supported." echo " Please check the wiki for further information" @@ -105,7 +105,7 @@ checkPrerequisite() { fi if [ "${HOME_DIR}" != "/home/pi" ]; then - echo + echo echo "ERROR: HomeDir must be '/home/pi'!" echo " Other usernames are currently not supported." echo " Please check the wiki for further information" diff --git a/scripts/installscripts/buster-install-default.sh b/scripts/installscripts/buster-install-default.sh index d46e61e7d..b5c9108bc 100644 --- a/scripts/installscripts/buster-install-default.sh +++ b/scripts/installscripts/buster-install-default.sh @@ -28,7 +28,7 @@ DATETIME=$(date +"%Y%m%d_%H%M%S") SCRIPTNAME="$(basename $0)" JOB="${SCRIPTNAME}" -CURRENT_USER="${SUDO_USER:-$USER}" +CURRENT_USER="${SUDO_USER:-$(whoami)}" HOME_DIR=$(getent passwd "$CURRENT_USER" | cut -d: -f6) echo "Current User: $CURRENT_USER" echo "User home dir: $HOME_DIR" @@ -97,7 +97,7 @@ checkPrerequisite() { #currently the user 'pi' is mandatory #https://github.com/MiczFlor/RPi-Jukebox-RFID/issues/1785 if [ "${CURRENT_USER}" != "pi" ]; then - echo + echo echo "ERROR: User must be 'pi'!" echo " Other usernames are currently not supported." echo " Please check the wiki for further information" @@ -105,7 +105,7 @@ checkPrerequisite() { fi if [ "${HOME_DIR}" != "/home/pi" ]; then - echo + echo echo "ERROR: HomeDir must be '/home/pi'!" echo " Other usernames are currently not supported." echo " Please check the wiki for further information" @@ -864,7 +864,7 @@ install_main() { sudo wget -q -O /etc/apt/keyrings/mopidy-archive-keyring.gpg https://apt.mopidy.com/mopidy.gpg sudo wget -q -O /etc/apt/sources.list.d/mopidy.list https://apt.mopidy.com/${OS_CODENAME}.list - + ${apt_get} update ${apt_get} upgrade ${apt_get} install libspotify-dev diff --git a/scripts/installscripts/tests/run_installation_tests.sh b/scripts/installscripts/tests/run_installation_tests.sh index a22e23d99..fb711048b 100644 --- a/scripts/installscripts/tests/run_installation_tests.sh +++ b/scripts/installscripts/tests/run_installation_tests.sh @@ -13,6 +13,7 @@ echo $PWD echo "samba-common samba-common/dhcp boolean false" | sudo debconf-set-selections # No interactive frontend export DEBIAN_FRONTEND=noninteractive +echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections # Run installation (in interactive mode) # y confirm interactive @@ -21,13 +22,12 @@ export DEBIAN_FRONTEND=noninteractive # n no spotify # y configure mpd # y audio default location -# y config gpio +# y config gpio # n no RFID registration # n No reboot -# TODO check, how this behaves on branches other than develop -GIT_BRANCH=develop bash ./scripts/installscripts/buster-install-default.sh <<< $'y\nn\n\ny\n\nn\n\ny\n\ny\n\ny\n\ny\nn\nn\n' +./../buster-install-default.sh <<< $'y\nn\n\ny\n\nn\n\ny\n\ny\n\ny\n\ny\nn\nn\n' INSTALLATION_EXITCODE=$? # Test installation -./scripts/installscripts/tests/test_installation.sh $INSTALLATION_EXITCODE +./test_installation.sh $INSTALLATION_EXITCODE diff --git a/scripts/installscripts/tests/run_installation_tests2.sh b/scripts/installscripts/tests/run_installation_tests2.sh index f907bf29d..58c2a378d 100644 --- a/scripts/installscripts/tests/run_installation_tests2.sh +++ b/scripts/installscripts/tests/run_installation_tests2.sh @@ -13,6 +13,7 @@ echo $PWD echo "samba-common samba-common/dhcp boolean false" | sudo debconf-set-selections # No interactive frontend export DEBIAN_FRONTEND=noninteractive +echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections # Run installation (in interactive mode) # y confirm interactive @@ -27,9 +28,8 @@ export DEBIAN_FRONTEND=noninteractive # yes, reader is connected # n No reboot -# TODO check, how this behaves on branches other than develop -GIT_BRANCH=develop bash ./scripts/installscripts/buster-install-default.sh <<< $'y\nn\n\ny\n\nn\n\ny\n\ny\n\ny\n\ny\ny\n2\ny\nn\n' +./../buster-install-default.sh <<< $'y\nn\n\ny\n\nn\n\ny\n\ny\n\ny\n\ny\ny\n2\ny\nn\n' INSTALLATION_EXITCODE=$? # Test installation -./scripts/installscripts/tests/test_installation.sh $INSTALLATION_EXITCODE +./test_installation.sh $INSTALLATION_EXITCODE diff --git a/scripts/installscripts/tests/run_installation_tests3.sh b/scripts/installscripts/tests/run_installation_tests3.sh index aa75c92b7..09340da8e 100644 --- a/scripts/installscripts/tests/run_installation_tests3.sh +++ b/scripts/installscripts/tests/run_installation_tests3.sh @@ -13,6 +13,7 @@ echo $PWD echo "samba-common samba-common/dhcp boolean false" | sudo debconf-set-selections # No interactive frontend export DEBIAN_FRONTEND=noninteractive +echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections # Run installation (in interactive mode) # y confirm interactive @@ -25,9 +26,8 @@ export DEBIAN_FRONTEND=noninteractive # n no RFID registration # n No reboot -# TODO check, how this behaves on branches other than develop -GIT_BRANCH=develop bash ./scripts/installscripts/buster-install-default.sh <<< $'y\nn\n\ny\n\ny\nmyuser\nmypassword\nmyclient_id\nmyclient_secret\n\ny\n\ny\n\ny\n\ny\nn\nn\n' +./../buster-install-default.sh <<< $'y\nn\n\ny\n\ny\nmyuser\nmypassword\nmyclient_id\nmyclient_secret\n\ny\n\ny\n\ny\n\ny\nn\nn\n' INSTALLATION_EXITCODE=$? # Test installation -./scripts/installscripts/tests/test_installation.sh $INSTALLATION_EXITCODE +./test_installation.sh $INSTALLATION_EXITCODE diff --git a/scripts/installscripts/tests/test_installation.sh b/scripts/installscripts/tests/test_installation.sh index fddeafa2e..332cfb8cb 100755 --- a/scripts/installscripts/tests/test_installation.sh +++ b/scripts/installscripts/tests/test_installation.sh @@ -7,8 +7,8 @@ INSTALLATION_EXITCODE="${1:-0}" PATHDATA="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -USER_NAME="$USER" -HOME_DIR="$HOME" +USER_NAME="$(whoami)" +HOME_DIR=$(getent passwd "$USER_NAME" | cut -d: -f6) tests=0 failed_tests=0