Merge branch 'main' of github.com:FNALssi/fermi-spack-tools into main

bin/build-spack-env.sh

@@ -561,7 +561,7 @@ _copy_back_logs() {
   mkdir -p "$tar_tmp/"{spack_env,spack-stage}
   cd "$spack_env_top_dir"
   _cmd $DEBUG_3 spack clean -dmp
-  _cmd $DEBUG_3 $PIPE tar -c *.log *-out.txt *.yaml etc var/spack/environments \
+  _cmd $DEBUG_3 $PIPE tar -c $spack_source_dir/*.log $spack_source_dir/*-out.txt $spack_source_dir/*.yaml $spack_source_dir/etc $spack_source_dir/var/spack/environments \
     | _cmd $DEBUG_3 tar -C "$tar_tmp/spack_env" -x
   _cmd $DEBUG_3 $PIPE tar -C "$(spack location -S)" -c . \
     | _cmd $DEBUG_3 tar -C "$tar_tmp/spack-stage" -x
@@ -717,22 +717,28 @@ _maybe_cache_binaries() {
     for cache in "$working_dir/copyBack/spack-$binary_mirror-cache" \
                  ${extra_sources_write_cache[*]:+"${extra_sources_write_cache[@]}"}; do
       _report $PROGRESS "caching$msg_extra binary packages for environment $env_name to $cache"
-      _cmd $DEBUG_1 $PROGRESS \
+      for hash in "${hashes_to_cache[@]}";do
+        if [  -f "$(spack location -i $hash)/.spack/binary_distribution" ]; then
+	  _report $DEBUG_1 "Skipping package installed from buildcache $hash"
+	else
+          _cmd $DEBUG_1 $PROGRESS \
            spack \
            ${__debug_spack_buildcache:+-d} \
            ${__verbose_spack_buildcache:+-v} \
            ${common_spack_opts[*]:+"${common_spack_opts[@]}"} \
-           buildcache create -a --deptype=all \
+           buildcache create --deptype=all \
            ${buildcache_package_opts[*]:+"${buildcache_package_opts[@]}"} \
            ${buildcache_key_opts[*]:+"${buildcache_key_opts[@]}"} \
            ${buildcache_rel_arg} "$cache" \
-           ${hashes_to_cache[*]:+"${hashes_to_cache[@]}"}
-      _report $PROGRESS "updating build cache index"
-      _cmd $DEBUG_1 $PROGRESS \
+           $hash
+	fi
+      done
+         _report $PROGRESS "updating build cache index"
+         _cmd $DEBUG_1 $PROGRESS \
            spack \
            ${common_spack_opts[*]:+"${common_spack_opts[@]}"} \
            buildcache update-index -k "$cache"
-    done
+   done
   fi
 }
 
@@ -1212,8 +1218,16 @@ local_caches=(
 )
 
 spack_env_top_dir="$working_dir/spack_env"
+case "$ups_opt" in
+    -p) spack_source_dir="./";;
+    -u) spack_source_dir="./spack/$spack_ver/NULL/";;
+    -t) spack_source_dir="./spack/$spack_ver/NULL/";;
+    -*) _die $EXIT_CONFIG_FAILURE "unrecognized --ups option $ups_opt\n$(usage)";;
+    *) break
+esac
 
-####################################
+
+###################################
 # Handle SPACK_PYTHON
 if [ -n "$spack_python" ]; then
   python_type="$(type -t "$spack_python")" \
@@ -1327,13 +1341,19 @@ trap "trap - EXIT; \
 _copy_back_logs; \
 if (( failed )) && (( want_emergency_buildcache )); then \
   tag_text=ALERT _report $ERROR \"emergency buildcache dump...\"; \
-  _cmd $ERROR $PIPE spack \
+  for spec in \$(spack find -L | sed -Ene 's&^([[:alnum:]]+).*\$&/\\1&p');do \
+    if [  -f \"\$(spack location -i \$spec)/.spack/binary_distribution\" ]; then
+      tag_text=ALERT _report $ERROR skipping package installed from buildcache \$spec;\
+      else \
+      _cmd $ERROR $PIPE spack \
       \${common_spack_opts[*]:+\"\${common_spack_opts[@]}\"} \
-      buildcache create -a --deptype=all \
+      buildcache create --deptype=all \
       \${buildcache_key_opts[*]:+\"\${buildcache_key_opts[@]}\"} \
       \$buildcache_rel_arg --rebuild-index \
       \"$working_dir/copyBack/spack-emergency-cache\" \
-      \$(spack find -L | sed -Ene 's&^([[:alnum:]]+).*\$&/\\1&p'); \
+     \$spec; \
+     fi \
+  done;\
   tag_text=ALERT _report $ERROR \"emergency buildcache dump COMPLETE\"; \
 fi; \
 exec $STDOUT>&- $STDERR>&-\
@@ -1352,6 +1372,8 @@ if ! [ "$ups_opt" = "-p" ]; then
   _report $PROGRESS "declaring fermi-spack-tools package to UPS"
   { source /grid/fermiapp/products/common/etc/setups \
       || source /products/setup \
+      || source /cvmfs/mu2e.opensciencegrid.org/artexternals/setups \
+      || source /cvmfs/larsoft.opensciencegrid.org/products/setups \
       || _die $EXIT_UPS_ERROR "unable to set up UPS"
   } >/dev/null 2>&1
   PRODUCTS="$spack_env_top_dir:$PRODUCTS"

templates/packages.yaml.almalinux9

@@ -317,10 +317,6 @@ packages:
     externals:
     - spec: "mesa +glx~osmesa swr=none @22.3.0 %[email protected] os=almalinux9"
       prefix: /usr
-  meson:
-    externals:
-    - spec: "meson @0.63.3 %[email protected] os=almalinux9"
-      prefix: /usr
   mkfontdir:
     externals:
     - spec: "mkfontdir @1.2.1 %[email protected] os=almalinux9"
@@ -363,12 +359,10 @@ packages:
     externals:
     - spec: "pkg-config +internal_glib @1.7.3 %[email protected] os=almalinux9"
       prefix: /usr
-    buildable: False
   pkgconf:
     externals:
     - spec: "pkgconf @1.7.3 %[email protected] os=almalinux9"
       prefix: /usr
-    buildable: False
   python:
     externals:
     - spec: "python @3.9.16 %[email protected] os=almalinux9"

templates/packages.yaml.scientific7

@@ -253,10 +253,6 @@ packages:
     externals:
     - spec: "libxi @1.7.9 %[email protected] os=scientific7"
       prefix: /usr
-  libxml2:
-    externals:
-    - spec: "libxml2 @2.9.1 %[email protected] os=scientific7"
-      prefix: /usr
   libxmu:
     externals:
     - spec: "libxmu @1.1.2 %[email protected] os=scientific7"
@@ -360,7 +356,6 @@ packages:
     externals:
     - spec: "pkg-config +internal_glib @0.27.1 %[email protected] os=scientific7"
       prefix: /usr
-    buildable: False
   python:
     externals:
     - spec: "python @3.6.8 %[email protected] os=scientific7"