build-push #411
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build-push | |
| on: | |
| schedule: | |
| - cron: "30 */6 * * *" | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| branches: | |
| - master | |
| workflow_dispatch: | |
| inputs: | |
| EDM4EIC_VERSION: | |
| required: false | |
| default: '' | |
| type: string | |
| EICRECON_VERSION: | |
| required: false | |
| default: '' | |
| type: string | |
| JUGGLER_VERSION: | |
| required: false | |
| default: '' | |
| type: string | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: false | |
| env: | |
| ## Default versions are specified in packages.yaml but can be overridden | |
| ## note: nightly builds will always use the master/main branch | |
| EDM4EIC_VERSION: ${{ inputs.EDM4EIC_VERSION }} | |
| EICRECON_VERSION: ${{ inputs.EICRECON_VERSION }} | |
| JUGGLER_VERSION: ${{ inputs.JUGGLER_VERSION }} | |
| ## Dockerhub registry | |
| DH_REGISTRY: docker.io | |
| DH_REGISTRY_USER: eicweb | |
| DH_PUSH: 0 | |
| ## GitHub registry | |
| GH_REGISTRY: ghcr.io | |
| GH_REGISTRY_USER: eic | |
| GH_PUSH: 1 | |
| ## Number of jobs to start during container builds | |
| JOBS: 4 | |
| ## Internal tag used for the CI | |
| INTERNAL_TAG: pipeline-${{ github.run_id }} | |
| jobs: | |
| base: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| include: | |
| - BASE_IMAGE: debian:stable-slim | |
| BUILD_IMAGE: debian_stable_base | |
| PLATFORM: linux/amd64 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| with: | |
| platforms: linux/amd64,linux/arm64 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: | | |
| name=${{ env.DH_REGISTRY }}/${{ env.DH_REGISTRY_USER }}/${{ matrix.BUILD_IMAGE }},enable=${{ env.DH_PUSH != 0 }} | |
| name=${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/${{ matrix.BUILD_IMAGE }},enable=${{ env.GH_PUSH != 0 }} | |
| tags: | | |
| ${{ env.INTERNAL_TAG }} | |
| type=schedule | |
| type=ref,prefix=unstable-pr-,event=pr | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| if: ${{ env.DH_PUSH == '1' }} | |
| with: | |
| registry: ${{ env.DH_REGISTRY }} | |
| username: ${{ env.DH_REGISTRY_USER }} | |
| password: ${{ secrets.DH_EICWEB_TOKEN }} | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| if: ${{ env.GH_PUSH == '1' }} | |
| with: | |
| registry: ${{ env.GH_REGISTRY }} | |
| username: ${{ secrets.GHCR_REGISTRY_USER }} | |
| password: ${{ secrets.GHCR_REGISTRY_TOKEN }} | |
| - name: Build and push | |
| uses: docker/build-push-action@v5 | |
| with: | |
| file: containers/debian/base.Dockerfile | |
| context: containers/debian | |
| platforms: ${{ matrix.PLATFORM }} | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| build-args: | | |
| BASE_IMAGE=${{ matrix.BASE_IMAGE }} | |
| BUILD_IMAGE=${{ matrix.BUILD_IMAGE }} | |
| cache-from: type=gha,scope=${{ github.workflow }} | |
| cache-to: type=gha,mode=max,scope=${{ github.workflow }} | |
| dev: | |
| runs-on: ubuntu-latest | |
| needs: base | |
| strategy: | |
| matrix: | |
| include: | |
| - BUILDER_IMAGE: debian_stable_base | |
| RUNTIME_IMAGE: debian_stable_base | |
| BUILD_IMAGE: dev | |
| PLATFORM: linux/amd64 | |
| ENV: dev | |
| steps: | |
| - name: Free Disk Space (Ubuntu) | |
| uses: jlumbroso/[email protected] | |
| with: | |
| android: true | |
| dotnet: true | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Load spack version and cherry-picks | |
| id: spack | |
| shell: bash | |
| run: | | |
| source spack.sh | |
| echo "orgrepo=${SPACK_ORGREPO}" | tee -a $GITHUB_OUTPUT | |
| echo "version=${SPACK_VERSION}" | tee -a $GITHUB_OUTPUT | |
| echo "cherrypicks=${SPACK_CHERRYPICKS//$'\n'/ }" | tee -a $GITHUB_OUTPUT | |
| echo "cherrypicks_files=${SPACK_CHERRYPICKS_FILES//$'\n'/ }" | tee -a $GITHUB_OUTPUT | |
| - name: Load key4hep-spack version | |
| id: key4hep-spack | |
| run: | | |
| source key4hep-spack.sh | |
| echo "orgrepo=${KEY4HEPSPACK_ORGREPO}" | tee -a $GITHUB_OUTPUT | |
| echo "version=${KEY4HEPSPACK_VERSION}" | tee -a $GITHUB_OUTPUT | |
| - name: Load eic-spack version | |
| id: eic-spack | |
| run: | | |
| source eic-spack.sh | |
| echo "orgrepo=${EICSPACK_ORGREPO}" | tee -a $GITHUB_OUTPUT | |
| echo "version=${EICSPACK_VERSION}" | tee -a $GITHUB_OUTPUT | |
| - name: Load secrets into mirrors.yaml | |
| id: mirrors | |
| run: | | |
| source spack.sh | |
| export SPACK_VERSION | |
| export CI_REGISTRY=ghcr.io | |
| export CI_PROJECT_PATH=eic | |
| export CI_REGISTRY_USER=${{ secrets.GHCR_REGISTRY_USER }} | |
| export CI_REGISTRY_PASSWORD=${{ secrets.GHCR_REGISTRY_TOKEN }} | |
| export GITHUB_REGISTRY_USER=${{ secrets.GHCR_REGISTRY_USER }} | |
| export GITHUB_REGISTRY_TOKEN=${{ secrets.GHCR_REGISTRY_TOKEN }} | |
| export S3RW_ACCESS_KEY=${{ secrets.S3RW_ACCESS_KEY }} | |
| export S3RW_SECRET_KEY=${{ secrets.S3RW_SECRET_KEY }} | |
| cat mirrors.yaml.in | envsubst > mirrors.yaml | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| with: | |
| platforms: linux/amd64 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: | | |
| name=${{ env.DH_REGISTRY }}/${{ env.DH_REGISTRY_USER }}/eic_${{ matrix.BUILD_IMAGE }},enable=${{ env.DH_PUSH != 0 }} | |
| name=${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/eic_${{ matrix.BUILD_IMAGE }},enable=${{ env.GH_PUSH != 0 }} | |
| tags: | | |
| ${{ env.INTERNAL_TAG }} | |
| type=schedule,pattern={{date 'YYYY-MM-DD'}} | |
| type=ref,prefix=unstable-pr-,event=pr | |
| type=match,pattern=^v(\d+\.\d+\.\d+-.*)$,group=1 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| if: ${{ env.DH_PUSH == '1' }} | |
| with: | |
| registry: ${{ env.DH_REGISTRY }} | |
| username: ${{ env.DH_REGISTRY_USER }} | |
| password: ${{ secrets.DH_EICWEB_TOKEN }} | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| if: ${{ env.GH_PUSH == '1' }} | |
| with: | |
| registry: ${{ env.GH_REGISTRY }} | |
| username: ${{ secrets.GHCR_REGISTRY_USER }} | |
| password: ${{ secrets.GHCR_REGISTRY_TOKEN }} | |
| - name: Build and push | |
| uses: docker/build-push-action@v5 | |
| with: | |
| file: containers/jug/dev.Dockerfile | |
| context: containers/jug | |
| build-contexts: | | |
| spack-environment=spack-environment | |
| secret-files: | | |
| mirrors=mirrors.yaml | |
| platforms: ${{ matrix.PLATFORM }} | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| build-args: | | |
| DOCKER_REGISTRY=${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/ | |
| BUILDER_IMAGE=${{ matrix.BUILDER_IMAGE }} | |
| RUNTIME_IMAGE=${{ matrix.RUNTIME_IMAGE }} | |
| BUILD_IMAGE=eic_${{ matrix.BUILD_IMAGE }} | |
| INTERNAL_TAG=${{ env.INTERNAL_TAG }} | |
| SPACK_ORGREPO=${{ steps.spack.outputs.orgrepo }} | |
| SPACK_VERSION=${{ steps.spack.outputs.version }} | |
| SPACK_CHERRYPICKS=${{ steps.spack.outputs.cherrypicks }} | |
| SPACK_CHERRYPICKS_FILES=${{ steps.spack.outputs.cherrypicks_files }} | |
| KEY4HEPSPACK_ORGREPO=${{ steps.eic-spack.outputs.orgrepo }} | |
| KEY4HEPSPACK_VERSION=${{ steps.eic-spack.outputs.version }} | |
| EICSPACK_ORGREPO=${{ steps.eic-spack.outputs.orgrepo }} | |
| EICSPACK_VERSION=${{ steps.eic-spack.outputs.version }} | |
| KEY4HEPSPACK_ORGREPO=${{ steps.key4hep-spack.outputs.orgrepo }} | |
| KEY4HEPSPACK_VERSION=${{ steps.key4hep-spack.outputs.version }} | |
| S3_ACCESS_KEY=${{ secrets.S3_ACCESS_KEY }} | |
| S3_SECRET_KEY=${{ secrets.S3_SECRET_KEY }} | |
| jobs=${{ env.JOBS }} | |
| cache-from: type=gha,scope=${{ github.workflow }} | |
| cache-to: type=gha,mode=max,scope=${{ github.workflow }} | |
| xl: | |
| runs-on: ubuntu-latest | |
| needs: dev | |
| strategy: | |
| matrix: | |
| include: | |
| - BASE_IMAGE: dev | |
| BUILD_IMAGE: xl | |
| PLATFORM: linux/amd64 | |
| steps: | |
| - name: Free Disk Space (Ubuntu) | |
| uses: jlumbroso/[email protected] | |
| with: | |
| android: true | |
| dotnet: true | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| with: | |
| platforms: linux/amd64 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: | | |
| name=${{ env.DH_REGISTRY }}/${{ env.DH_REGISTRY_USER }}/eic_${{ matrix.BUILD_IMAGE }},enable=${{ env.DH_PUSH != 0 }} | |
| name=${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/eic_${{ matrix.BUILD_IMAGE }},enable=${{ env.GH_PUSH != 0 }} | |
| tags: | | |
| ${{ env.INTERNAL_TAG }} | |
| type=schedule,pattern={{date 'YYYY-MM-DD'}} | |
| type=ref,prefix=unstable-pr-,event=pr | |
| type=match,pattern=^v(\d+\.\d+\.\d+-.*)$,group=1 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| if: ${{ env.DH_PUSH == '1' }} | |
| with: | |
| registry: ${{ env.DH_REGISTRY }} | |
| username: ${{ env.DH_REGISTRY_USER }} | |
| password: ${{ secrets.DH_EICWEB_TOKEN }} | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| if: ${{ env.GH_PUSH == '1' }} | |
| with: | |
| registry: ${{ env.GH_REGISTRY }} | |
| username: ${{ secrets.GHCR_REGISTRY_USER }} | |
| password: ${{ secrets.GHCR_REGISTRY_TOKEN }} | |
| - name: Build and push | |
| uses: docker/build-push-action@v5 | |
| with: | |
| file: containers/jug/xl.Dockerfile | |
| context: containers/jug | |
| build-contexts: | | |
| detectors=. | |
| platforms: ${{ matrix.PLATFORM }} | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| build-args: | | |
| DOCKER_REGISTRY=${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/ | |
| BASE_IMAGE=eic_${{ matrix.BASE_IMAGE }} | |
| BUILD_IMAGE=eic_${{ matrix.BUILD_IMAGE }} | |
| INTERNAL_TAG=${{ env.INTERNAL_TAG }} | |
| jobs=${{ env.JOBS }} | |
| cache-from: type=gha,scope=${{ github.workflow }} | |
| cache-to: type=gha,mode=max,scope=${{ github.workflow }} |