Merge branch 'master' into 'wdconinc-master-patch-35288'

# Conflicts:
#   .gitlab-ci.yml

.github/workflows/mirror.yaml

@@ -0,0 +1,41 @@
+name: Mirror and Trigger EICweb
+
+on:
+  pull_request:
+  push:
+  workflow_dispatch:
+
+concurrency:
+  group: mirror
+  cancel-in-progress: false
+
+jobs:
+  build:
+    name: Mirror and Trigger EICweb
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      contents: read
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Push to EICweb
+      uses: eic/gitlab-sync@master
+      with:
+        url: https://eicweb.phy.anl.gov/containers/eic_container.git/
+        token: ${{ secrets.GITLAB_TOKEN }}
+        username: ${{ secrets.GITLAB_USERNAME }}
+        ciskip: true
+    - name: Trigger EICweb
+      uses: eic/trigger-gitlab-ci@v2
+      with:
+        url: https://eicweb.phy.anl.gov
+        project_id: 290
+        token: ${{ secrets.EICWEB_EIC_CONTAINER_TRIGGER }}
+        ref_name: ${{ github.event.pull_request.head.ref || github.ref }}
+        variables:
+          GITHUB_REPOSITORY=${{ github.repository }}
+          GITHUB_SHA=${{ github.event.pull_request.head.sha || github.sha }}
+          GITHUB_PR=${{ github.event.pull_request.number }}

.gitlab-ci.yml

@@ -2,13 +2,13 @@ image: alpine
 
 variables:
   ## External images
-  DOCKER_IMAGE: docker.io/docker:24.0.5
-  SINGULARITY_IMAGE: quay.io/singularity/singularity:v3.11.4
+  DOCKER_IMAGE: docker.io/docker:24.0.7
+  SINGULARITY_IMAGE: quay.io/singularity/singularity:v3.11.5
 
   ## Application versions used for the main release 
   ## note: nightly builds will always use the master/main branch
-  JUGGLER_VERSION: "v11.0.0"
-  EICRECON_VERSION: "v1.9.0"
+  JUGGLER_VERSION: "v13.0.0"
+  EICRECON_VERSION: "v1.9.1"
 
   ## Local registry
   CI_PUSH: 1

containers/jug/dev.Dockerfile

@@ -20,6 +20,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=${TARGETPLATFORM}
 rm -f /etc/apt/apt.conf.d/docker-clean
 apt-get -yqq update
 apt-get -yqq install --no-install-recommends                            \
+        jq                                                              \
         python3                                                         \
         python3-dev                                                     \
         python3-distutils                                               \
@@ -70,14 +71,6 @@ spack compiler find --scope site
 spack config blame compilers
 EOF
 
-## Setup local buildcache mirrors
-RUN --mount=type=cache,target=/var/cache/spack <<EOF
-set -e
-spack mirror add local /var/cache/spack/mirror/${SPACK_VERSION}
-spack buildcache update-index local
-spack mirror list
-EOF
-
 ## Setup eics3 buildcache mirrors
 ## - this always adds the read-only mirror to the container
 ## - the write-enabled mirror is provided later as a secret mount
@@ -134,7 +127,7 @@ FROM spack as builder
 COPY --from=spack-environment . /opt/spack-environment/
 ARG ENV=dev
 ARG JUGGLER_VERSION="main"
-ADD https://eicweb.phy.anl.gov/api/v4/projects/EIC%2Fjuggler/repository/tree?ref=${JUGGLER_VERSION} /tmp/juggler.json
+ADD https://api.github.com/repos/eic/juggler/commits/${JUGGLER_VERSION} /tmp/juggler.json
 ARG EICRECON_VERSION="main"
 ADD https://api.github.com/repos/eic/eicrecon/commits/${EICRECON_VERSION} /tmp/eicrecon.json
 ENV SPACK_ENV=/opt/spack-environment/${ENV}
@@ -144,23 +137,28 @@ RUN --mount=type=cache,target=/ccache,id=${TARGETPLATFORM}              \
     <<EOF
 set -e
 export CCACHE_DIR=/ccache
-spack buildcache update-index local
+find /var/cache/spack/blobs/sha256/ -atime +7 -delete
+JUGGLER_VERSION=$(jq -r .sha /tmp/juggler.json)
+EICRECON_VERSION=$(jq -r .sha /tmp/eicrecon.json)
 spack buildcache update-index eics3rw
 spack env activate --dir ${SPACK_ENV}
 spack add juggler@git.${JUGGLER_VERSION}
 spack add eicrecon@git.${EICRECON_VERSION}
 spack concretize --fresh --force --quiet
-make --jobs ${jobs} --keep-going --directory /opt/spack-environment SPACK_ENV=${SPACK_ENV} BUILDCACHE_MIRROR="local eics3rw"
+make --jobs ${jobs} --keep-going --directory /opt/spack-environment \
+  SPACK_ENV=${SPACK_ENV} \
+  BUILDCACHE_OCI_PROMPT="eicweb" \
+  BUILDCACHE_OCI_FINAL="ghcr" \
+  BUILDCACHE_S3_FINAL="eics3rw"
 ccache --show-stats
 ccache --zero-stats
 EOF
 
 ## Create view at /usr/local
 RUN --mount=type=cache,target=/var/cache/spack <<EOF
 set -e
-spack env activate --dir ${SPACK_ENV}
 rm -r /usr/local
-spack env view enable /usr/local
+spack -e ${SPACK_ENV} env view enable /usr/local
 EOF
 
 ## Optional, nuke the buildcache after install, before (re)caching
@@ -298,10 +296,10 @@ ARG BENCHMARK_DET_VERSION="master"
 ARG BENCHMARK_REC_VERSION="master"
 ARG BENCHMARK_PHY_VERSION="master"
 ## cache bust when updated repositories
-ADD ${EICWEB}/458/repository/tree?ref=${BENCHMARK_COM_VERSION} /tmp/485.json
-ADD ${EICWEB}/399/repository/tree?ref=${BENCHMARK_DET_VERSION} /tmp/399.json
-ADD ${EICWEB}/408/repository/tree?ref=${BENCHMARK_REC_VERSION} /tmp/408.json 
-ADD ${EICWEB}/400/repository/tree?ref=${BENCHMARK_PHY_VERSION} /tmp/400.json
+ADD ${EICWEB}/458/repository/commits/${BENCHMARK_COM_VERSION} /tmp/485.json
+ADD ${EICWEB}/399/repository/commits/${BENCHMARK_DET_VERSION} /tmp/399.json
+ADD ${EICWEB}/408/repository/commits/${BENCHMARK_REC_VERSION} /tmp/408.json 
+ADD ${EICWEB}/400/repository/commits/${BENCHMARK_PHY_VERSION} /tmp/400.json
 RUN <<EOF
 set -ex
 mkdir -p /opt/benchmarks

eic-spack.sh

@@ -3,7 +3,7 @@ EICSPACK_ORGREPO="eic/eic-spack"
 
 ## EIC spack commit hash or github version, e.g. v0.19.7
 ## note: nightly builds could use a branch e.g. releases/v0.19
-EICSPACK_VERSION="4c9c1250d054e587455e5f00df242e376018130a"
+EICSPACK_VERSION="5d8ba97ea6b021a50aa11e924a55a4726c689d65"
 
 ## Space-separated list of eic-spack cherry-picks
 read -r -d '' EICSPACK_CHERRYPICKS <<- \

mirrors.yaml.in

@@ -1,4 +1,16 @@
 mirrors:
+  eicweb:
+    url: oci://${CI_REGISTRY}/${CI_PROJECT_PATH}/spack-${SPACK_VERSION}
+    push:
+      access_pair:
+      - ${CI_REGISTRY_USER}
+      - ${CI_REGISTRY_PASSWORD}
+  ghcr:
+    url: oci://ghcr.io/eic/spack-${SPACK_VERSION}
+    push:
+      access_pair:
+      - ${GITHUB_REGISTRY_USER}
+      - ${GITHUB_REGISTRY_TOKEN}
   eics3rw:
     fetch:
       url: s3://eictest/EPIC/spack/${SPACK_VERSION}

spack-environment/Makefile

@@ -1,13 +1,17 @@
 MAKEFLAGS += -Orecurse
 
 SPACK ?= spack
-SPACK_INSTALL_FLAGS += --no-check-signature
+SPACK_INSTALL_FLAGS += --no-check-signature --show-log-on-error
 
 export SPACK_COLOR = always
 
 SPACK_ENV ?= dev
 
-BUILDCACHE_MIRROR :=
+BUILDCACHE_S3_PROMPT :=
+BUILDCACHE_S3_FINAL :=
+BUILDCACHE_OCI_PROMPT :=
+BUILDCACHE_OCI_FINAL :=
+BUILDCACHE_OCI_BASE_IMAGE := ghcr.io/eic/debian:stable-slim
 
 .PHONY: all clean
 
@@ -18,11 +22,15 @@ include $(SPACK_ENV)/spack.mk
 endif
 
 $(SPACK_ENV)/push: $(addprefix $(SPACK_ENV)/push/,$($(SPACK_ENV)/SPACK_PACKAGE_IDS))
-	$(foreach buildcache, $(BUILDCACHE_MIRROR), $(SPACK) buildcache update-index $(buildcache) ;)
+	$(foreach buildcache, $(BUILDCACHE_S3_FINAL), $(SPACK) buildcache push --unsigned $(buildcache) ;)
+	$(foreach buildcache, $(BUILDCACHE_S3_FINAL), $(SPACK) buildcache update-index $(buildcache) ;)
+	$(foreach buildcache, $(BUILDCACHE_S3_PROMPT), $(SPACK) buildcache update-index $(buildcache) ;)
+	$(foreach buildcache, $(BUILDCACHE_OCI_FINAL), $(SPACK) buildcache push --unsigned --base-image $(BUILDCACHE_OCI_BASE_IMAGE) $(buildcache) ;)
 
 $(SPACK_ENV)/push/%: $(SPACK_ENV)/install/%
 	@mkdir -p $(dir $@)
-	$(foreach buildcache, $(BUILDCACHE_MIRROR), $(SPACK) buildcache push --only=package --unsigned $(buildcache) /$(HASH) ;) # push $(SPEC)
+	$(foreach buildcache, $(BUILDCACHE_S3_PROMPT), $(SPACK) buildcache push --unsigned --only=package $(buildcache) /$(HASH) ;) # push $(SPEC)
+	$(foreach buildcache, $(BUILDCACHE_OCI_PROMPT), $(SPACK) buildcache push --unsigned --base-image $(BUILDCACHE_OCI_BASE_IMAGE) $(buildcache) /$(HASH) | grep -v "Using cached archive" ;) # push $(SPEC)
 	@touch $@
 
 $(SPACK_ENV)/spack.lock: $(SPACK_ENV)/spack.yaml Makefile

spack-environment/dev/spack.yaml

@@ -39,6 +39,8 @@ spack:
   - irt
   - iwyu
   - jana2
+  - k4actstracking
+  - k4fwcore
   - lcov
   - lhapdf
   - madx
@@ -74,6 +76,7 @@ spack:
   - py-scipy
   - py-seaborn
   - py-toml
+  - py-torch
   - py-uproot
   - py-wurlitzer
   - py-yapf

spack-environment/packages.yaml

@@ -17,7 +17,7 @@ packages:
     - any_of: [build_type=Release, '@:']
   acts:
     require:
-    - '@30.3.2'
+    - '@31.2.0'
     - cxxstd=20 +binaries +dd4hep +examples +fatras +geant4 +identification +json +python +tgeo
   actsvg:
     require:
@@ -87,7 +87,7 @@ packages:
     - cxxstd=20
   edm4hep:
     require:
-    - '@0.10'
+    - '@0.10.2'
     - cxxstd=20
   eic-smear:
     require:
@@ -128,6 +128,9 @@ packages:
     - '@11.1.3.east'
     - cxxstd=20 -vecgeom +threads -vtk
     - any_of: [+opengl +qt +x11, -opengl -qt -x11]
+  gloo:
+    require:
+    - '@2023-05-19'
   graphviz:
     require:
     - '@8.0.1'
@@ -158,6 +161,12 @@ packages:
   juggler:
     require:
     - cxxstd=20
+  k4actstracking:
+    require:
+    - '@main'
+  k4fwcore:
+    require:
+    - '@1.0pre17'
   lapack:
     require:
     - openblas
@@ -300,6 +309,10 @@ packages:
   py-toml:
     require:
     - '@0.10.2'
+  py-torch:
+    require:
+    - '@2.1.0'
+    - -mkldnn
   py-uproot:
     require:
     - '@5.0.5'

spack.sh

@@ -16,10 +16,15 @@ c485709f625429a88a184a099373d76c9438f8e3
 6f08daf67020289e6a5ed1df9783ac5b2919e477
 50051b56199992eb4395b8ff22913c1995311a8c
 f01774f1d41781bc4b9e5abb5469e234168da663
-cfa2f19f48699a762af77ece8a9c5bc4db3a75d5
-2433f503b73bb515b2e1a7615ae01fea59d2290e
 16f4c53cd4cfb4bc6c9390f6e65217fc9ccc58c9
 d171f314c77ba61b3cd780f159afe6abced5707d
+b111064e221aae83e62226672cd8bf9a7524423d
+3c54177c5d9032cb36cf154b553d739cbeb2d024
+c07ddf83c32b7129247fe90eed486dd844047087
+00875fbc749a7e4e979c35d59c0a8d60d32d4cd7
+64cf67822471ec2d3df2625a6d713f80cbe7ff43
+7a0c4e8017033430e5f15ed628be6b539e935ba9
+48fcfda1e7c1781cab4cada6d099823b263ab0cc
 ---
 ## Ref: https://github.com/spack/spack/commit/[hash]
 ## [hash]: [description]
@@ -31,7 +36,12 @@ d171f314c77ba61b3cd780f159afe6abced5707d
 ## 6f08daf67020289e6a5ed1df9783ac5b2919e477: root: add a webgui patch
 ## 50051b56199992eb4395b8ff22913c1995311a8c: geant4: new version 11.1.3
 ## f01774f1d41781bc4b9e5abb5469e234168da663: hepmc3: fix from_variant -> self.define
-## cfa2f19f48699a762af77ece8a9c5bc4db3a75d5: py-htgettoken: use os.environ, avoid AttributeError
-## 2433f503b73bb515b2e1a7615ae01fea59d2290e: py-htgettoken: allow for undefined variables
 ## 16f4c53cd4cfb4bc6c9390f6e65217fc9ccc58c9: py-bokeh: new version 3.3.1, and supporting packages
 ## d171f314c77ba61b3cd780f159afe6abced5707d: py-pygithub: new versions, dependencies
+## b111064e221aae83e62226672cd8bf9a7524423d: py-htgettoken: use os.environ, avoid AttributeError
+## 3c54177c5d9032cb36cf154b553d739cbeb2d024: edm4hep: add latest tag for 0.10.2
+## c07ddf83c32b7129247fe90eed486dd844047087: py-torch: set env OpenBLAS_HOME
+## 00875fbc749a7e4e979c35d59c0a8d60d32d4cd7: py-torch: patch for ${OpenBLAS_HOME}/include/openblas
+## 64cf67822471ec2d3df2625a6d713f80cbe7ff43: py-torch: move patch before def patch
+## 7a0c4e8017033430e5f15ed628be6b539e935ba9: acts: new versions 31.*
+## 48fcfda1e7c1781cab4cada6d099823b263ab0cc: setup-env.sh: if exe contains qemu, use /proc/$$/comm instead