diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 96d034bb..218613f2 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -93,20 +93,14 @@ jobs: registry: ${{ env.GH_REGISTRY }} username: ${{ secrets.GHCR_REGISTRY_USER }} password: ${{ secrets.GHCR_REGISTRY_TOKEN }} - - name: Build and push - uses: docker/build-push-action@v5 + - name: Buildx Bake + uses: docker/bake-action@v5 with: - file: containers/debian/base.Dockerfile - context: containers/debian - platforms: ${{ matrix.PLATFORM }} push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - BASE_IMAGE=${{ matrix.BASE_IMAGE }} - BUILD_IMAGE=${{ matrix.BUILD_IMAGE }} - cache-from: type=gha,scope=${{ github.workflow }} - cache-to: type=gha,mode=max,scope=${{ github.workflow }} + provenance: false + files: | + containers/debian/base.hcl + ${{ steps.meta.outputs.bake-file }} dev: runs-on: ubuntu-latest @@ -194,106 +188,11 @@ jobs: registry: ${{ env.GH_REGISTRY }} username: ${{ secrets.GHCR_REGISTRY_USER }} password: ${{ secrets.GHCR_REGISTRY_TOKEN }} - - name: Build and push - uses: docker/build-push-action@v5 + - name: Buildx Bake + uses: docker/bake-action@v5 with: - file: containers/jug/dev.Dockerfile - context: containers/jug - build-contexts: | - spack-environment=spack-environment - secret-files: | - mirrors=mirrors.yaml - platforms: ${{ matrix.PLATFORM }} push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - DOCKER_REGISTRY=${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/ - BUILDER_IMAGE=${{ matrix.BUILDER_IMAGE }} - RUNTIME_IMAGE=${{ matrix.RUNTIME_IMAGE }} - BUILD_IMAGE=eic_${{ matrix.BUILD_IMAGE }} - INTERNAL_TAG=${{ env.INTERNAL_TAG }} - SPACK_ORGREPO=${{ steps.spack.outputs.orgrepo }} - SPACK_VERSION=${{ steps.spack.outputs.version }} - SPACK_CHERRYPICKS=${{ steps.spack.outputs.cherrypicks }} - SPACK_CHERRYPICKS_FILES=${{ steps.spack.outputs.cherrypicks_files }} - KEY4HEPSPACK_ORGREPO=${{ steps.eic-spack.outputs.orgrepo }} - KEY4HEPSPACK_VERSION=${{ steps.eic-spack.outputs.version }} - EICSPACK_ORGREPO=${{ steps.eic-spack.outputs.orgrepo }} - EICSPACK_VERSION=${{ steps.eic-spack.outputs.version }} - KEY4HEPSPACK_ORGREPO=${{ steps.key4hep-spack.outputs.orgrepo }} - KEY4HEPSPACK_VERSION=${{ steps.key4hep-spack.outputs.version }} - S3_ACCESS_KEY=${{ secrets.S3_ACCESS_KEY }} - S3_SECRET_KEY=${{ secrets.S3_SECRET_KEY }} - jobs=${{ env.JOBS }} - cache-from: type=gha,scope=${{ github.workflow }} - cache-to: type=gha,mode=max,scope=${{ github.workflow }} - - xl: - runs-on: ubuntu-latest - needs: dev - strategy: - matrix: - include: - - BASE_IMAGE: dev - BUILD_IMAGE: xl - PLATFORM: linux/amd64 - steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@v1.3.1 - with: - android: true - dotnet: true - - name: Checkout - uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: linux/amd64 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: | - name=${{ env.DH_REGISTRY }}/${{ env.DH_REGISTRY_USER }}/eic_${{ matrix.BUILD_IMAGE }},enable=${{ env.DH_PUSH != 0 }} - name=${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/eic_${{ matrix.BUILD_IMAGE }},enable=${{ env.GH_PUSH != 0 }} - tags: | - ${{ env.INTERNAL_TAG }} - type=schedule,pattern={{date 'YYYY-MM-DD'}} - type=ref,prefix=unstable-pr-,event=pr - type=match,pattern=^v(\d+\.\d+\.\d+-.*)$,group=1 - - name: Login to Docker Hub - uses: docker/login-action@v3 - if: ${{ env.DH_PUSH == '1' }} - with: - registry: ${{ env.DH_REGISTRY }} - username: ${{ env.DH_REGISTRY_USER }} - password: ${{ secrets.DH_EICWEB_TOKEN }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - if: ${{ env.GH_PUSH == '1' }} - with: - registry: ${{ env.GH_REGISTRY }} - username: ${{ secrets.GHCR_REGISTRY_USER }} - password: ${{ secrets.GHCR_REGISTRY_TOKEN }} - - name: Build and push - uses: docker/build-push-action@v5 - with: - file: containers/jug/xl.Dockerfile - context: containers/jug - build-contexts: | - detectors=. - platforms: ${{ matrix.PLATFORM }} - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - DOCKER_REGISTRY=${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/ - BASE_IMAGE=eic_${{ matrix.BASE_IMAGE }} - BUILD_IMAGE=eic_${{ matrix.BUILD_IMAGE }} - INTERNAL_TAG=${{ env.INTERNAL_TAG }} - jobs=${{ env.JOBS }} - cache-from: type=gha,scope=${{ github.workflow }} - cache-to: type=gha,mode=max,scope=${{ github.workflow }} + provenance: false + files: | + containers/jug/dev.hcl + ${{ steps.meta.outputs.bake-file }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 02cc0002..5e1c56fd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -269,21 +269,16 @@ base: needs: - version script: + - docker buildx bake ${BUILD_OPTIONS} + --print + --file containers/variables.hcl + --file containers/debian/base.hcl - while ! - docker buildx build --push ${BUILD_OPTIONS} - --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${INTERNAL_TAG} - ${EXPORT_TAG:+ - ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}:${EXPORT_TAG}} - ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}} - ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}:${EXPORT_TAG}} - } - --file containers/debian/base.Dockerfile - --platform ${PLATFORM} - --build-arg BASE_IMAGE=${BASE_IMAGE} - --build-arg BUILD_IMAGE=${BUILD_IMAGE} - --provenance false - containers/debian - 2>&1 | tee build.log + docker buildx bake ${BUILD_OPTIONS} + --push + --file containers/variables.hcl + --file containers/debian/base.hcl + 2>&1 | tee build.log ; do if grep "unknown blob" build.log ; then BUILD_OPTIONS="--no-cache ${BUILD_OPTIONS}" ; @@ -345,100 +340,26 @@ eic: if [ "$CI_COMMIT_BRANCH" == "master" ]; then PUSH_NIGHTLY_WITH_DATE="1" fi - - if [ "${BUILD_TYPE}" == "nightly" ] ; then IF_BUILD_NIGHTLY= ; fi - - if [ "${BUILD_TYPE}" == "default" ] ; then IF_BUILD_DEFAULT= ; fi - - apk add envsubst git + - apk add envsubst - source spack.sh ; - source key4hep-spack.sh ; - source eic-spack.sh ; export SPACK_VERSION ; cat mirrors.yaml.in | envsubst > mirrors.yaml + - docker buildx bake ${BUILD_OPTIONS} + --print + --file spack.hcl + --file eic-spack.hcl + --file key4hep-spack.hcl + --file containers/variables.hcl + --file containers/jug/dev.hcl - while ! - docker buildx build --push ${BUILD_OPTIONS} - --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${INTERNAL_TAG}-${BUILD_TYPE} - --tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE/eic/jug}${ENV}:${INTERNAL_TAG}-${BUILD_TYPE} - ${EXPORT_TAG:+ - ${IF_BUILD_DEFAULT+ - ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}} - ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}} - ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}} - } - ${IF_BUILD_DEFAULT- - ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}} - ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}} - ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}} - } - } - ${IF_BUILD_NIGHTLY+ - ${NIGHTLY:+ - ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}} - ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}} - ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}} - ${PUSH_NIGHTLY_WITH_DATE:+ - ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)} - ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE}${ENV}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)} - } - } - } - ${EXPORT_TAG:+ - ${IF_BUILD_DEFAULT+ - ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE/eic/jug}${ENV}:${EXPORT_TAG}} - ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE/eic/jug}${ENV}:${EXPORT_TAG}} - ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE/eic/jug}${ENV}:${EXPORT_TAG}} - } - ${IF_BUILD_DEFAULT- - ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE/eic/jug}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}} - ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE/eic/jug}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}} - ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE/eic/jug}${ENV}:${EXPORT_TAG}-${BUILD_TYPE}} - } - } - ${IF_BUILD_NIGHTLY+ - ${NIGHTLY:+ - ${CI_PUSH:+--tag ${CI_REGISTRY}/${CI_PROJECT_PATH}/${BUILD_IMAGE/eic/jug}${ENV}:${NIGHTLY_TAG}} - ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE/eic/jug}${ENV}:${NIGHTLY_TAG}} - ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE/eic/jug}${ENV}:${NIGHTLY_TAG}} - } - ${PUSH_NIGHTLY_WITH_DATE:+ - ${DH_PUSH:+--tag ${DH_REGISTRY}/${DH_REGISTRY_USER}/${BUILD_IMAGE/eic/jug}${ENV}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)} - ${GH_PUSH:+--tag ${GH_REGISTRY}/${GH_REGISTRY_USER}/${BUILD_IMAGE/eic/jug}${ENV}:${NIGHTLY_TAG}-$(date +%Y-%m-%d)} - } - } - --file containers/jug/dev.Dockerfile - --platform ${PLATFORM} - --build-arg DOCKER_REGISTRY=${CI_REGISTRY}/${CI_PROJECT_PATH}/ - --build-arg BUILDER_IMAGE=${BUILDER_IMAGE} - --build-arg RUNTIME_IMAGE=${RUNTIME_IMAGE} - --build-arg INTERNAL_TAG=${INTERNAL_TAG} - --build-arg SPACK_ORGREPO=${SPACK_ORGREPO} - --build-arg SPACK_VERSION=${SPACK_VERSION} - --build-arg SPACK_CHERRYPICKS="${SPACK_CHERRYPICKS}" - --build-arg SPACK_CHERRYPICKS_FILES="${SPACK_CHERRYPICKS_FILES}" - --build-arg KEY4HEPSPACK_ORGREPO=${KEY4HEPSPACK_ORGREPO} - --build-arg KEY4HEPSPACK_VERSION=${KEY4HEPSPACK_VERSION} - --build-arg EICSPACK_ORGREPO=${EICSPACK_ORGREPO} - --build-arg EICSPACK_VERSION=${EICSPACK_VERSION} - --build-arg S3_ACCESS_KEY=${S3_ACCESS_KEY} - --build-arg S3_SECRET_KEY=${S3_SECRET_KEY} - --build-arg JUG_VERSION=${EXPORT_TAG}-${BUILD_TYPE}-$(git rev-parse HEAD) - ${IF_BUILD_DEFAULT+ - ${EDM4EIC_VERSION:+--build-arg EDM4EIC_VERSION=${EDM4EIC_VERSION}} - ${EICRECON_VERSION:+--build-arg EICRECON_VERSION=${EICRECON_VERSION}} - ${EPIC_VERSION:+--build-arg EPIC_VERSION=${EPIC_VERSION}} - ${JUGGLER_VERSION:+--build-arg JUGGLER_VERSION=${JUGGLER_VERSION}} - } - ${IF_BUILD_NIGHTLY+ - --build-arg EDM4EIC_VERSION=main - --build-arg EICRECON_VERSION=main - --build-arg EPIC_VERSION=main - --build-arg JUGGLER_VERSION=main - } - --build-arg ENV=${ENV} - --build-arg jobs=${JOBS} - --build-context spack-environment=spack-environment - --secret id=mirrors,src=mirrors.yaml - --provenance false - containers/jug - 2>&1 | tee build.log + docker buildx bake ${BUILD_OPTIONS} + --push + --file spack.hcl + --file eic-spack.hcl + --file key4hep-spack.hcl + --file containers/variables.hcl + --file containers/jug/dev.hcl + 2>&1 | tee build.log ; do if grep "unknown blob" build.log ; then BUILD_OPTIONS="--no-cache ${BUILD_OPTIONS}" ; diff --git a/containers/debian/base.Dockerfile b/containers/debian/base.Dockerfile index 0eeb0b1a..42f882d4 100644 --- a/containers/debian/base.Dockerfile +++ b/containers/debian/base.Dockerfile @@ -1,6 +1,6 @@ #syntax=docker/dockerfile:1.4 ARG BASE_IMAGE="amd64/debian:stable-slim" -ARG BUILD_IMAGE="debian_stable_base" +ARG BUILD_IMAGE="debian_base" # Minimal container based on Debian base systems for up-to-date packages. FROM ${BASE_IMAGE} diff --git a/containers/debian/base.hcl b/containers/debian/base.hcl new file mode 100644 index 00000000..cb5b5195 --- /dev/null +++ b/containers/debian/base.hcl @@ -0,0 +1,30 @@ +# Variables which are required to be defined +variable "BUILD_IMAGE" { default = "debian_base" } + +# Variables whose defaults can be overridden on build +variable "BASE_IMAGE" { default = null } + +# docker/metadata-action overrides the following target with tags +# but we implement it for use outside docker/metadata-action +target "docker-metadata-action" { + tags = compact(flatten([ + join("/", compact([ CI_REGISTRY, CI_PROJECT_PATH, "${BUILD_IMAGE}:${INTERNAL_TAG}"]) ), + EXPORT_TAG != null && EXPORT_TAG != "" ? [ + for registry in registries: "${registry}/${BUILD_IMAGE}:${EXPORT_TAG}" + ] : [ null ] + ])) +} + +target "default" { + inherits = ["docker-metadata-action"] + attest = [ + "type=provenance,disabled=true" + ] + context = "containers/debian" + dockerfile = "base.Dockerfile" + platforms = [ "linux/amd64" ] + args = { + BASE_IMAGE = BASE_IMAGE + BUILD_IMAGE = BUILD_IMAGE + } +} diff --git a/containers/jug/dev.Dockerfile b/containers/jug/dev.Dockerfile index be4096bf..098c523f 100644 --- a/containers/jug/dev.Dockerfile +++ b/containers/jug/dev.Dockerfile @@ -1,7 +1,7 @@ #syntax=docker/dockerfile:1.4 ARG DOCKER_REGISTRY="eicweb/" -ARG BUILDER_IMAGE="debian_stable_base" -ARG RUNTIME_IMAGE="debian_stable_base" +ARG BUILDER_IMAGE="debian_base" +ARG RUNTIME_IMAGE="debian_base" ARG INTERNAL_TAG="testing" ## ======================================================================================== @@ -32,7 +32,7 @@ EOF ## Setup spack ENV SPACK_ROOT=/opt/spack ARG SPACK_ORGREPO="spack/spack" -ARG SPACK_VERSION="releases/v0.20" +ARG SPACK_VERSION="v0.22.0" ENV SPACK_PYTHON=/usr/bin/python3 ARG SPACK_CHERRYPICKS="" ARG SPACK_CHERRYPICKS_FILES="" @@ -108,7 +108,7 @@ EOF ## Setup eic-spack ENV EICSPACK_ROOT=${SPACK_ROOT}/var/spack/repos/eic-spack ARG EICSPACK_ORGREPO="eic/eic-spack" -ARG EICSPACK_VERSION="$SPACK_VERSION" +ARG EICSPACK_VERSION="develop" ADD https://api.github.com/repos/${EICSPACK_ORGREPO}/commits/${EICSPACK_VERSION} /tmp/eic-spack.json RUN < + "9bcc43c4c158639fa6cb575c6106595a34682081", # protobuf: update hash for patch needed when="@3.4:3.21" + "9f3e45ddbee24aaa7993e575297827e0aed2e6fe", # acts: pass cuda_arch to CMAKE_CUDA_ARCHITECTURES + "85f13442d2a7486daba81fdd9a3b6a1182ba11f6", # Consolidate concretization output for environments + "f73d7d2dce226857cbc774e942454bad2992969e", # dd4hep: cleanup recipe, remove deprecated versions and patches + "cbab451c1a342523ed75e9be1098615a597a9b59", # dd4hep: Add version 1.29 + ] +} +## Optional hash table with comma-separated file list +variable "SPACK_CHERRYPICKS_FILES" { + default = [] +} diff --git a/spack.sh b/spack.sh deleted file mode 100644 index d431e6f9..00000000 --- a/spack.sh +++ /dev/null @@ -1,33 +0,0 @@ -## Spack organization and repository, e.g. spack/spack -SPACK_ORGREPO="spack/spack" - -## Spack github version, e.g. v0.18.1 or commit hash -## note: nightly builds will use e.g. releases/v0.19 -SPACK_VERSION="v0.22.0" - -## Space-separated list of spack cherry-picks -read -r -d '' SPACK_CHERRYPICKS <<- \ ---- || true -09f75ee426a2e05e0543570821582480ff823ba5 -f6d50f790ee8b123f7775429f6ca6394170e6de9 -63f6e6079aacc99078386e5c8ff06173841b9595 -9bcc43c4c158639fa6cb575c6106595a34682081 -9f3e45ddbee24aaa7993e575297827e0aed2e6fe -85f13442d2a7486daba81fdd9a3b6a1182ba11f6 -f73d7d2dce226857cbc774e942454bad2992969e -cbab451c1a342523ed75e9be1098615a597a9b59 ---- -## Optional hash table with comma-separated file list -read -r -d '' SPACK_CHERRYPICKS_FILES <<- \ ---- || true ---- -## Ref: https://github.com/spack/spack/commit/[hash] -## [hash]: [description] -## 09f75ee426a2e05e0543570821582480ff823ba5: setup-env.sh: if exe contains qemu, use /proc/$$/comm instead -## f6d50f790ee8b123f7775429f6ca6394170e6de9: gaudi: Add version 38.1 -## 63f6e6079aacc99078386e5c8ff06173841b9595: gaudi: upstream patch when @38.1 for missing #include -## 9bcc43c4c158639fa6cb575c6106595a34682081: protobuf: update hash for patch needed when="@3.4:3.21" -## 9f3e45ddbee24aaa7993e575297827e0aed2e6fe: acts: pass cuda_arch to CMAKE_CUDA_ARCHITECTURES -## 85f13442d2a7486daba81fdd9a3b6a1182ba11f6: Consolidate concretization output for environments -## f73d7d2dce226857cbc774e942454bad2992969e: dd4hep: cleanup recipe, remove deprecated versions and patches -## cbab451c1a342523ed75e9be1098615a597a9b59: dd4hep: Add version 1.29