From 43d4b07d4a9bc29848d785bf8823182761c9876e Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Thu, 2 Nov 2023 07:47:28 -0500 Subject: [PATCH 01/12] Remove libxml2 from SLF7. Remove meson from SLF7 and AL9. Remove buildable:False from pkgconf and pkg-config --- templates/packages.yaml.almalinux9 | 10 ++-------- templates/packages.yaml.scientific7 | 11 ----------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/templates/packages.yaml.almalinux9 b/templates/packages.yaml.almalinux9 index f7e2d20..80f0340 100644 --- a/templates/packages.yaml.almalinux9 +++ b/templates/packages.yaml.almalinux9 @@ -138,7 +138,7 @@ packages: buildable: False gettext: externals: - - spec: "gettext ~bzip2+curses~git+libunistring+libxml2~tar+xz @0.21 %gcc@11.3.1 os=almalinux9 ^libc" + - spec: "gettext ~bzip2+curses~git+libunistring+libxml2~tar+xz @0.21 %gcc@11.3.1 os=almalinux9" prefix: /usr buildable: False git: @@ -171,7 +171,7 @@ packages: prefix: /usr htgettoken: externals: - - spec: "htgettoken @1.18 %gcc@11.3.1 os=almalinux9" + - spec: "htgettoken @1.20 %gcc@11.3.1 os=almalinux9" prefix: /usr hwloc: require: @@ -312,10 +312,6 @@ packages: externals: - spec: "mesa +glx~osmesa swr=none @22.3.0 %gcc@11.3.1 os=almalinux9" prefix: /usr - meson: - externals: - - spec: "meson @0.63.3 %gcc@11.3.1 os=almalinux9" - prefix: /usr mkfontdir: externals: - spec: "mkfontdir @1.2.1 %gcc@11.3.1 os=almalinux9" @@ -358,12 +354,10 @@ packages: externals: - spec: "pkg-config +internal_glib @1.7.3 %gcc@11.3.1 os=almalinux9" prefix: /usr - buildable: False pkgconf: externals: - spec: "pkgconf @1.7.3 %gcc@11.3.1 os=almalinux9" prefix: /usr - buildable: False python: externals: - spec: "python @3.9.16 %gcc@11.3.1 os=almalinux9" diff --git a/templates/packages.yaml.scientific7 b/templates/packages.yaml.scientific7 index e5e2452..d348fed 100644 --- a/templates/packages.yaml.scientific7 +++ b/templates/packages.yaml.scientific7 @@ -248,10 +248,6 @@ packages: externals: - spec: "libxi @1.7.9 %gcc@4.8.5 os=scientific7" prefix: /usr - libxml2: - externals: - - spec: "libxml2 @2.9.1 %gcc@4.8.5 os=scientific7" - prefix: /usr libxmu: externals: - spec: "libxmu @1.1.2 %gcc@4.8.5 os=scientific7" @@ -309,10 +305,6 @@ packages: externals: - spec: "mesa +glx~osmesa swr=none @18.3.4 %gcc@4.8.5 os=scientific7" prefix: /usr - meson: - externals: - - spec: "meson @0.55.1 %gcc@4.8.5 os=scientific7" - prefix: /usr mkfontdir: externals: - spec: "mkfontdir @7.5 %gcc@4.8.5 os=scientific7" @@ -359,13 +351,10 @@ packages: externals: - spec: "pkg-config +internal_glib @0.27.1 %gcc@4.8.5 os=scientific7" prefix: /usr - buildable: False python: externals: - spec: "python @3.6.8 %gcc@4.8.5 os=scientific7" prefix: /usr - - spec: "python @3.6.8 %gcc@4.8.5 os=scientific7" - prefix: /usr readline: externals: - spec: "readline @6.2 %gcc@4.8.5 os=scientific7" From 256a574d932687aa269e4757ab497591e5e1e19f Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Thu, 2 Nov 2023 07:59:12 -0500 Subject: [PATCH 02/12] Skip creation of empty ups_to_spack repo --- bin/make_spack | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/make_spack b/bin/make_spack index 7f01a7a..66edb5e 100755 --- a/bin/make_spack +++ b/bin/make_spack @@ -104,8 +104,8 @@ add_recipe_repos() { git clone https://github.com/marcmengel/scd_recipes.git $SPACK_ROOT/var/spack/repos/scd_recipes spack repo add --scope=site $SPACK_ROOT/var/spack/repos/scd_recipes - spack repo create $SPACK_ROOT/var/spack/repos/ups_to_spack - spack repo add --scope=site $SPACK_ROOT/var/spack/repos/ups_to_spack +# spack repo create $SPACK_ROOT/var/spack/repos/ups_to_spack +# spack repo add --scope=site $SPACK_ROOT/var/spack/repos/ups_to_spack } create_spack() { From 773aecc3dea04cad55659401f2c56f8b52f38a5f Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Thu, 2 Nov 2023 10:55:50 -0500 Subject: [PATCH 03/12] Add more ups setups locations --- bin/build-spack-env.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/build-spack-env.sh b/bin/build-spack-env.sh index 93198cd..af96575 100755 --- a/bin/build-spack-env.sh +++ b/bin/build-spack-env.sh @@ -1352,6 +1352,8 @@ if ! [ "$ups_opt" = "-p" ]; then _report $PROGRESS "declaring fermi-spack-tools package to UPS" { source /grid/fermiapp/products/common/etc/setups \ || source /products/setup \ + || source /cvmfs/mu2e.opensciencegrid.org/artexternals/setups \ + || source /cvmfs/larsoft.opensciencegrid.org/products/setups \ || _die $EXIT_UPS_ERROR "unable to set up UPS" } >/dev/null 2>&1 PRODUCTS="$spack_env_top_dir:$PRODUCTS" From 085129750e6668ed82a4aadcd7d11ec32400976f Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Thu, 2 Nov 2023 11:38:07 -0500 Subject: [PATCH 04/12] Looks to spack directory in unified layout --- bin/build-spack-env.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/bin/build-spack-env.sh b/bin/build-spack-env.sh index af96575..90ebdb4 100755 --- a/bin/build-spack-env.sh +++ b/bin/build-spack-env.sh @@ -1212,8 +1212,16 @@ local_caches=( ) spack_env_top_dir="$working_dir/spack_env" +case "$ups_opt" in + -p) spack_env_top_dir="$working_dir/spack_env";; + -u) spack_env_top_dir="$working_dir/spack_env/spack/$spack_ver/NULL/";; + -t) spack_env_top_dir="$working_dir/spack_env/spack/$spack_ver/NULL/";; + -*) _die $EXIT_CONFIG_FAILURE "unrecognized --ups option $ups_opt\n$(usage)";; + *) break +esac -#################################### + +################################### # Handle SPACK_PYTHON if [ -n "$spack_python" ]; then python_type="$(type -t "$spack_python")" \ From fc4d02fcafe7ed38698c3074367319043e94c694 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Thu, 2 Nov 2023 12:11:50 -0500 Subject: [PATCH 05/12] Don't know why layout is duplicated. --- bin/build-spack-env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/build-spack-env.sh b/bin/build-spack-env.sh index 90ebdb4..b300393 100755 --- a/bin/build-spack-env.sh +++ b/bin/build-spack-env.sh @@ -1214,8 +1214,8 @@ local_caches=( spack_env_top_dir="$working_dir/spack_env" case "$ups_opt" in -p) spack_env_top_dir="$working_dir/spack_env";; - -u) spack_env_top_dir="$working_dir/spack_env/spack/$spack_ver/NULL/";; - -t) spack_env_top_dir="$working_dir/spack_env/spack/$spack_ver/NULL/";; + -u) spack_env_top_dir="$working_dir/spack_env/spack/$spack_ver/NULL/spack/$spack_ver/NULL/";; + -t) spack_env_top_dir="$working_dir/spack_env/spack/$spack_ver/NULL/spack/$spack_ver/NULL/";; -*) _die $EXIT_CONFIG_FAILURE "unrecognized --ups option $ups_opt\n$(usage)";; *) break esac From 9e891b1624e73382f502d6d5b26798c7b887bd02 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Thu, 2 Nov 2023 12:26:40 -0500 Subject: [PATCH 06/12] Use spack_source_dir to indicate the relative path of the spack installation to spack_env_top_dir --- bin/build-spack-env.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/build-spack-env.sh b/bin/build-spack-env.sh index b300393..6a117c7 100755 --- a/bin/build-spack-env.sh +++ b/bin/build-spack-env.sh @@ -561,7 +561,7 @@ _copy_back_logs() { mkdir -p "$tar_tmp/"{spack_env,spack-stage} cd "$spack_env_top_dir" _cmd $DEBUG_3 spack clean -dmp - _cmd $DEBUG_3 $PIPE tar -c *.log *-out.txt *.yaml etc var/spack/environments \ + _cmd $DEBUG_3 $PIPE tar -c $spack_source_dir/*.log $spack_source_dir/*-out.txt $spack_source_dir/*.yaml $spack_source_dir/etc $spack_source_dir/spack/environments \ | _cmd $DEBUG_3 tar -C "$tar_tmp/spack_env" -x _cmd $DEBUG_3 $PIPE tar -C "$(spack location -S)" -c . \ | _cmd $DEBUG_3 tar -C "$tar_tmp/spack-stage" -x @@ -1213,9 +1213,9 @@ local_caches=( spack_env_top_dir="$working_dir/spack_env" case "$ups_opt" in - -p) spack_env_top_dir="$working_dir/spack_env";; - -u) spack_env_top_dir="$working_dir/spack_env/spack/$spack_ver/NULL/spack/$spack_ver/NULL/";; - -t) spack_env_top_dir="$working_dir/spack_env/spack/$spack_ver/NULL/spack/$spack_ver/NULL/";; + -p) spack_source_dir="./";; + -u) spack_source_dir="./spack/$spack_ver/NULL/";; + -t) spack_source_dir="./spack/$spack_ver/NULL/";; -*) _die $EXIT_CONFIG_FAILURE "unrecognized --ups option $ups_opt\n$(usage)";; *) break esac From 299ecd6f831c4eca28f47276ea1e9eae8c1d1d5c Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Thu, 2 Nov 2023 08:00:43 -0500 Subject: [PATCH 07/12] Check for .spack/binary_distribution and skip buildcache creation if it exists --- bin/build-spack-env.sh | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/bin/build-spack-env.sh b/bin/build-spack-env.sh index 6a117c7..f3ee63e 100755 --- a/bin/build-spack-env.sh +++ b/bin/build-spack-env.sh @@ -717,22 +717,26 @@ _maybe_cache_binaries() { for cache in "$working_dir/copyBack/spack-$binary_mirror-cache" \ ${extra_sources_write_cache[*]:+"${extra_sources_write_cache[@]}"}; do _report $PROGRESS "caching$msg_extra binary packages for environment $env_name to $cache" - _cmd $DEBUG_1 $PROGRESS \ + for hash in "${hashes_to_cache[@]}";do + if [ ! -f $(spack location -i $hash)/.spack/binary_distribution ]; then + _cmd $DEBUG_1 $PROGRESS \ spack \ ${__debug_spack_buildcache:+-d} \ ${__verbose_spack_buildcache:+-v} \ ${common_spack_opts[*]:+"${common_spack_opts[@]}"} \ - buildcache create -a --deptype=all \ + buildcache create --deptype=all \ ${buildcache_package_opts[*]:+"${buildcache_package_opts[@]}"} \ ${buildcache_key_opts[*]:+"${buildcache_key_opts[@]}"} \ ${buildcache_rel_arg} "$cache" \ - ${hashes_to_cache[*]:+"${hashes_to_cache[@]}"} - _report $PROGRESS "updating build cache index" - _cmd $DEBUG_1 $PROGRESS \ + $hash + fi + done + _report $PROGRESS "updating build cache index" + _cmd $DEBUG_1 $PROGRESS \ spack \ ${common_spack_opts[*]:+"${common_spack_opts[@]}"} \ buildcache update-index -k "$cache" - done + done fi } @@ -1335,13 +1339,17 @@ trap "trap - EXIT; \ _copy_back_logs; \ if (( failed )) && (( want_emergency_buildcache )); then \ tag_text=ALERT _report $ERROR \"emergency buildcache dump...\"; \ - _cmd $ERROR $PIPE spack \ + for spec in \$(spack find -L | sed -Ene 's&^([[:alnum:]]+).*\$&/\\1&p');do \ + if [ ! -f \$(spack location -i \$spec)/.spack/binary_distribution ]; then + _cmd $ERROR $PIPE spack \ \${common_spack_opts[*]:+\"\${common_spack_opts[@]}\"} \ - buildcache create -a --deptype=all \ + buildcache create --deptype=all --only=package \ \${buildcache_key_opts[*]:+\"\${buildcache_key_opts[@]}\"} \ \$buildcache_rel_arg --rebuild-index \ \"$working_dir/copyBack/spack-emergency-cache\" \ - \$(spack find -L | sed -Ene 's&^([[:alnum:]]+).*\$&/\\1&p'); \ + \$spec; \ + fi \ + done;\ tag_text=ALERT _report $ERROR \"emergency buildcache dump COMPLETE\"; \ fi; \ exec $STDOUT>&- $STDERR>&-\ From 2258f025089556488ba1f28dc5f12d5edf2e5cc2 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Thu, 2 Nov 2023 12:09:32 -0500 Subject: [PATCH 08/12] Put quotes around path to binary_distribution --- bin/build-spack-env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/build-spack-env.sh b/bin/build-spack-env.sh index f3ee63e..6760252 100755 --- a/bin/build-spack-env.sh +++ b/bin/build-spack-env.sh @@ -718,7 +718,7 @@ _maybe_cache_binaries() { ${extra_sources_write_cache[*]:+"${extra_sources_write_cache[@]}"}; do _report $PROGRESS "caching$msg_extra binary packages for environment $env_name to $cache" for hash in "${hashes_to_cache[@]}";do - if [ ! -f $(spack location -i $hash)/.spack/binary_distribution ]; then + if [ ! -f "$(spack location -i $hash)/.spack/binary_distribution" ]; then _cmd $DEBUG_1 $PROGRESS \ spack \ ${__debug_spack_buildcache:+-d} \ @@ -1340,7 +1340,7 @@ _copy_back_logs; \ if (( failed )) && (( want_emergency_buildcache )); then \ tag_text=ALERT _report $ERROR \"emergency buildcache dump...\"; \ for spec in \$(spack find -L | sed -Ene 's&^([[:alnum:]]+).*\$&/\\1&p');do \ - if [ ! -f \$(spack location -i \$spec)/.spack/binary_distribution ]; then + if [ ! -f \"\$(spack location -i \$spec)/.spack/binary_distribution\" ]; then _cmd $ERROR $PIPE spack \ \${common_spack_opts[*]:+\"\${common_spack_opts[@]}\"} \ buildcache create --deptype=all --only=package \ From a745fa8ddcc5289c6dd6bdd267fe87913f08067c Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Thu, 2 Nov 2023 12:39:00 -0500 Subject: [PATCH 09/12] Drop duplicate --only=package --- bin/build-spack-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/build-spack-env.sh b/bin/build-spack-env.sh index 6760252..aca2513 100755 --- a/bin/build-spack-env.sh +++ b/bin/build-spack-env.sh @@ -1343,7 +1343,7 @@ if (( failed )) && (( want_emergency_buildcache )); then \ if [ ! -f \"\$(spack location -i \$spec)/.spack/binary_distribution\" ]; then _cmd $ERROR $PIPE spack \ \${common_spack_opts[*]:+\"\${common_spack_opts[@]}\"} \ - buildcache create --deptype=all --only=package \ + buildcache create --deptype=all \ \${buildcache_key_opts[*]:+\"\${buildcache_key_opts[@]}\"} \ \$buildcache_rel_arg --rebuild-index \ \"$working_dir/copyBack/spack-emergency-cache\" \ From 9e566862f6a353a3ed2be79184ea45dd4d272a6f Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Thu, 2 Nov 2023 13:28:30 -0500 Subject: [PATCH 10/12] Reverse the logic and output message when skipping --- bin/build-spack-env.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bin/build-spack-env.sh b/bin/build-spack-env.sh index aca2513..40f1483 100755 --- a/bin/build-spack-env.sh +++ b/bin/build-spack-env.sh @@ -718,7 +718,9 @@ _maybe_cache_binaries() { ${extra_sources_write_cache[*]:+"${extra_sources_write_cache[@]}"}; do _report $PROGRESS "caching$msg_extra binary packages for environment $env_name to $cache" for hash in "${hashes_to_cache[@]}";do - if [ ! -f "$(spack location -i $hash)/.spack/binary_distribution" ]; then + if [ -f "$(spack location -i $hash)/.spack/binary_distribution" ]; then + _report $DEBUG_1 "Skipping package installed from buildcache $hash" + else _cmd $DEBUG_1 $PROGRESS \ spack \ ${__debug_spack_buildcache:+-d} \ @@ -1340,7 +1342,9 @@ _copy_back_logs; \ if (( failed )) && (( want_emergency_buildcache )); then \ tag_text=ALERT _report $ERROR \"emergency buildcache dump...\"; \ for spec in \$(spack find -L | sed -Ene 's&^([[:alnum:]]+).*\$&/\\1&p');do \ - if [ ! -f \"\$(spack location -i \$spec)/.spack/binary_distribution\" ]; then + if [ -f \"\$(spack location -i \$spec)/.spack/binary_distribution\" ]; then + _report $ERROR skipping package installed from buildcache \$spec;\ + else \ _cmd $ERROR $PIPE spack \ \${common_spack_opts[*]:+\"\${common_spack_opts[@]}\"} \ buildcache create --deptype=all \ From 14e8d65c854e5f17091bdb312925dbe7e5c95903 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Thu, 2 Nov 2023 14:40:20 -0500 Subject: [PATCH 11/12] add var to environments relative path --- bin/build-spack-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/build-spack-env.sh b/bin/build-spack-env.sh index 40f1483..693590f 100755 --- a/bin/build-spack-env.sh +++ b/bin/build-spack-env.sh @@ -561,7 +561,7 @@ _copy_back_logs() { mkdir -p "$tar_tmp/"{spack_env,spack-stage} cd "$spack_env_top_dir" _cmd $DEBUG_3 spack clean -dmp - _cmd $DEBUG_3 $PIPE tar -c $spack_source_dir/*.log $spack_source_dir/*-out.txt $spack_source_dir/*.yaml $spack_source_dir/etc $spack_source_dir/spack/environments \ + _cmd $DEBUG_3 $PIPE tar -c $spack_source_dir/*.log $spack_source_dir/*-out.txt $spack_source_dir/*.yaml $spack_source_dir/etc $spack_source_dir/var/spack/environments \ | _cmd $DEBUG_3 tar -C "$tar_tmp/spack_env" -x _cmd $DEBUG_3 $PIPE tar -C "$(spack location -S)" -c . \ | _cmd $DEBUG_3 tar -C "$tar_tmp/spack-stage" -x From fed2e5dd1b1b5be14eca005a62f32e1ba6aad87f Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Wed, 8 Nov 2023 08:48:33 -0600 Subject: [PATCH 12/12] Make skipping package installed from buildache an ALERT not a FATAL ERROR --- bin/build-spack-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/build-spack-env.sh b/bin/build-spack-env.sh index 693590f..5c50c4b 100755 --- a/bin/build-spack-env.sh +++ b/bin/build-spack-env.sh @@ -1343,7 +1343,7 @@ if (( failed )) && (( want_emergency_buildcache )); then \ tag_text=ALERT _report $ERROR \"emergency buildcache dump...\"; \ for spec in \$(spack find -L | sed -Ene 's&^([[:alnum:]]+).*\$&/\\1&p');do \ if [ -f \"\$(spack location -i \$spec)/.spack/binary_distribution\" ]; then - _report $ERROR skipping package installed from buildcache \$spec;\ + tag_text=ALERT _report $ERROR skipping package installed from buildcache \$spec;\ else \ _cmd $ERROR $PIPE spack \ \${common_spack_opts[*]:+\"\${common_spack_opts[@]}\"} \