Merge pull request #55 from greenc-FNAL/feature/upgrade-spack-bits

Targeted upgrade of existing installation components

bin/build-spack-env.sh

@@ -78,10 +78,12 @@ BRIEF OPTIONS
   --debug-tmp
   --fermi-spack-tools-root[= ]<repo>
   --fermi-spack-tools-version[= ]<version>
+  --no-auto-recipe-repos
   --(no-)?emergency-buildcache
   --(no-)?fail-fast
   --(no-)?query-packages
   --(no-)?safe-concretize
+  --(no-)?upgrade-(etc|extensions|recipes|spack)
   --no-view
   --spack-python[= ]<python-exec>
   --spack-config-cmd[= ]<config-cmd-string>+
@@ -194,6 +196,11 @@ SPACK CONFIGURATION OPTIONS
 
  Other Spack Configuration
 
+  --no-auto-recipe-repos
+
+    Do not define a default set of recipe repositories to obtain and
+    configure (handled by `make_spack`).
+
   --(no-)?emergency-buildcache
 
     Control whether to dump successfully installed binaries to an
@@ -215,6 +222,16 @@ SPACK CONFIGURATION OPTIONS
     Control whether to concretize environments with only a minimal set
     of mirrors configured to improve reproducibility (default yes).
 
+  --(no-)?upgrade-(etc|extensions|recipes|spack)
+
+    Control whether to upgrade the corresponding component(s) of an
+    existing Spack installation:
+
+    * `etc/` (default yes)
+    * Spack extensions (default yes)
+    * recipe repositories including those specified with --spack-repo (default no)
+    * the Spack installation itself (default no).
+
   --no-view
 
     Disable views in created environments. If specified, any view
@@ -447,23 +464,27 @@ _configure_recipe_repos() {
       path="$SPACK_ROOT/${path##*/}"
       local configure_namespace=1
       if [ -d "$path" ]; then # We already have a repo here.
-        orig_namespace="$(_cmd $DEBUG_3 $PIPE sed -Ene 's&^[[:space:]]*namespace[[:space:]]*:[[:space:]]*'"'([^']+)'"'$&\1&p' "$path/repo.yaml")"
+        # Deactivate existing namespace.
+        _deactivate_repo_at "$path"
         if [ "$(_cmd $DEBUG_3 $PIPE git -C "$path" remote)" = "origin" ] &&
              [ "$(_cmd $DEBUG_3 $PIPE git -C "$path" remote get-url origin)" = "$url" ]; then
           if [ "$(_cmd $DEBUG_3 $PIPE git -C "$path" branch --show-current)" = "$branch_etc" ]; then
-            # Nothing to do, existing repo is what we want.
-            configure_namespace=0
+            :
           elif (( $(_cmd $DEBUG_3 $PIPE git -C "$path" status -s | wc -l) == 0 )) &&
                  [ -n "$branch_etc" ]; then
-            # Deactivate existing namespace.
-            _deactivate_repo $orig_namespace
             # Switch to desired branch.
             _report $INFO "switching to branch $branch_etc in $path"
             _cmd $DEBUG_1 git -C "$path" switch "$branch_etc"
           fi
+          if (( upgrade_recipes )); then
+            _report $INFO "upgrading recipe repository on branch $branch_etc at $path"
+            _cmd $DEBUG_1 git -C "$path" pull ||
+              _die "unable to update recipe repository on branch $branch_etc at $path"
+          fi
         else
           local bnum=0 rpath= orig_path="$path"
           while read -r rpath < <(_cmd $DEBUG_3 $PIPE ls -1 "$orig_path"-*); do
+            _deactivate_repo_at "$rpath"
             [[ "$rpath" =~ -([0-9]+)$ ]] &&
               (( "${BASH_REMATCH[1]}" > bnum )) &&
               (( bnum = "${BASH_REMATCH[1]}" ))
@@ -484,14 +505,12 @@ _configure_recipe_repos() {
     else
       path="$repo_element"
     fi
-    if (( configure_namespace )); then
-      local namespace="$(_cmd $DEBUG_3 $PIPE sed -Ene 's&^[[:space:]]*namespace[[:space:]]*:[[:space:]]*'"'([^']+)'"'$&\1&p' "$path/repo.yaml")"
-      # Deactivate namespace if it is configured.
-      _deactivate_repo $namespace
-      _report $INFO "configuring Spack recipe repo $namespace${scope:+ in scope $scope} at $path"
-      _cmd $DEBUG_1 spack repo add${scope:+ --scope $scope} "$path" ||
-        _die "unable to add repo $namespace${scope:+ in scope $scope} at $path"
-    fi
+    local new_namespace="$(_cmd $DEBUG_3 $PIPE sed -Ene 's&^[[:space:]]*namespace[[:space:]]*:[[:space:]]*'"'([^']+)'"'$&\1&p' "$path/repo.yaml")"
+    # Deactivate namespace if it is configured.
+    _deactivate_repo $new_namespace
+    _report $INFO "configuring Spack recipe repo $new_namespace${scope:+ in scope $scope} at $path"
+    _cmd $DEBUG_1 spack repo add${scope:+ --scope $scope} "$path" ||
+      _die "unable to add repo $new_namespace${scope:+ in scope $scope} at $path"
   done
 }
 
@@ -667,6 +686,12 @@ _deactivate_repo() {
   done
 }
 
+_deactivate_repo_at() {
+  local path="$1"
+  local orig_namespace="$(_cmd $DEBUG_3 $PIPE sed -Ene 's&^[[:space:]]*namespace[[:space:]]*:[[:space:]]*'"'([^']+)'"'$&\1&p' "$path/repo.yaml")"
+  _deactivate_repo $orig_namespace
+}
+
 # Print a message and exit with the specifed numeric first argument or 1
 # as status code.
 _die() {
@@ -1198,13 +1223,18 @@ concretizing_cache_specs=()
 extra_binaries_write_cache=()
 extra_sources_write_cache=()
 fail_fast=1
+no_auto_recipe_repos=
 recipe_repos=()
 si_root=https://github.com/FNALssi/fermi-spack-tools.git
 si_ver=main
 spack_config_cmds=()
 spack_config_files=()
 spack_source_dir="./"
 spack_ver=v0.22.0-fermi
+upgrade_etc=1
+upgrade_extensions=1
+upgrade_recipes=
+upgrade_spack=
 ups_opt=-p
 want_emergency_buildcache=1
 
@@ -1226,13 +1256,18 @@ while (( $# )); do
     --emergency-buildcache) want_emergency_buildcache=1;;
     --fail-fast) fail_fast=1;;
     --help|-h|-\?) usage 2; exit 1;;
+    --no-auto-recipe-repos) no_auto_recipe_repos=1;;
     --no-cache-write-binaries) _set_cache_write_binaries "none";;
     --no-cache-write-sources) unset cache_write_sources;;
     --no-emergency-buildcache) unset want_emergency_buildcache;;
     --no-fail-fast) unset fail_fast;;
     --no-view) view_opt="--without-view";;
     --no-query-packages) unset query_packages;;
     --no-safe-concretize) unset concretize_safely;;
+    --no-upgrade-etc) unset upgrade_etc;;
+    --no-upgrade-extensions) unset upgrade_extensions;;
+    --no-upgrade-recipes) unset upgrade_recipes;;
+    --no-upgrade-spack) unset upgrade_spack;;
     --no-ups) ups_opt=-p;;
     --query-packages) query_packages=1;;
     --quiet|-q) (( VERBOSITY = WARNING ));;
@@ -1255,6 +1290,10 @@ while (( $# )); do
     --spack-version=*) spack_ver="${1#*=}";;
     --test) tests_type="$2"; shift;;
     --test=*) tests_type="${1#*=}";;
+    --upgrade-etc) upgrade_etc=1;;
+    --upgrade-extensions) upgrade_extensions=1;;
+    --upgrade-recipes) upgrade_recipes=1;;
+    --upgrade-spack) upgrade_spack=1;;
     --ups) ups_opt="$(_ups_string_to_opt "$2")" || exit; shift;;
     --ups=*) ups_opt="$(_ups_string_to_opt "${1#*=}")" || exit;;
     +v) (( --VERBOSITY ));;
@@ -1392,8 +1431,11 @@ make_spack_cmd=(
   ${spack_root:+--spack_repo "$spack_root"}
   --spack_release $spack_ver
   --minimal
-  --upgrade-etc
-  --upgrade-extensions
+  ${no_auto_recipe_repos:+"--no-recipe-repos"}
+  ${upgrade_etc:+"--upgrade-etc"}
+  ${upgrade_extensions:+"--upgrade-extensions"}
+  ${upgrade_recipes:+"--upgrade-recipes"}
+  ${upgrade_spack:+"--upgrade-spack"}
   $ups_opt
 )
 (( VERBOSITY < DEBUG_1 )) || make_spack_cmd+=(-v)

bin/make_spack

@@ -14,16 +14,22 @@ find_ourselves() {
 clone_repos() {
     while read ddir branch repo
     do
-      if [ -e "$ddir/.git" ]; then
-        if [ "$repo" = "$spack_repo" ] ; then
-          repo_type=spack
-        elif [[ "${ddir#$dir/}" == etc/* ]]; then
-          repo_type=etc
-        elif [[ "${ddir#$dir/var/spack/}" == extensions/* ]]; then
-          repo_type=extensions
-        else
-          unset repo_type
+      if [ "$repo" = "$spack_repo" ] ; then
+        repo_type=spack
+      elif [[ "${ddir#$dir/}" == etc/* ]]; then
+        repo_type=etc
+      elif [[ "${ddir#$dir/var/spack/}" == extensions/* ]]; then
+        repo_type=extensions
+      elif [[ "${ddir#$dir/var/spack/}" == repos/* ]] ||
+             [ -f "$ddir/repo.yaml" ]; then
+        if $no_recipes; then
+          continue
         fi
+        repo_type=recipes
+      else
+        unset repo_type
+      fi
+      if [ -e "$ddir/.git" ]; then
         if [ -n "$repo_type" ]; then
           eval "upgrade_repo=\$upgrade_$repo_type"
         else
@@ -44,12 +50,11 @@ clone_repos() {
             echo "$(date --iso-8601=seconds) Refusing to update Git repository from branch $current_branch to $branch at" 1>&2
             echo "$(date --iso-8601=seconds)   $ddir" 1>&2
             echo "$(date --iso-8601=seconds) Fix manually" 1>&2
-            exit 2
           fi
         else
           echo "$(date --iso-8601=seconds) Using existing repository on branch $current_branch at "
           echo "$(date --iso-8601=seconds)   $ddir"
-          echo "$(date --iso-8601=seconds) Use --upgrade-$repo_type or upgrade manually"
+          echo "$(date --iso-8601=seconds) U${repo_type:+se --upgrade-$repo_type or u}pgrade manually"
         fi
       else
         if $verbose; then
@@ -60,7 +65,6 @@ clone_repos() {
           echo "$(date --iso-8601=seconds) Unable to clone Git repository on branch $branch at" 1>&2
           echo "$(date --iso-8601=seconds)   $ddir" 1>&2
           echo "$(date --iso-8601=seconds) Fix manually" 1>&2
-          exit 2
         fi
       fi
     done <<EOF
@@ -127,6 +131,7 @@ usage() {
         echo "  --upgrade (deprecated, ignored)"
         echo "  --upgrade-etc"
         echo "  --upgrade-extensions"
+        echo "  --upgrade-recipes"
         echo "  --upgrade-spack"
         echo "  --no-buildcache"
         echo "  --query-packages"
@@ -163,9 +168,10 @@ parse_args() {
     repovers=""
     verbose=false
     depth=1
+    no_recipes=false
 
     origargs="$*"
-    if x=$(getopt --longoptions help,depth,with_padding,upgrade,upgrade-etc,upgrade-extensions,upgrade-spack,spack_release:,minimal,no_buildcache,repover,spack_repo:,query-packages,verbose --options mptuv -- "$@")
+    if x=$(getopt --longoptions help,depth,with_padding,upgrade,upgrade-etc,upgrade-extensions,upgrade-recipes,upgrade-spack,spack_release:,minimal,no-recipe-repos,no_buildcache,repover,spack_repo:,query-packages,verbose --options mptuv -- "$@")
     then
         eval set : $x
         shift
@@ -178,14 +184,15 @@ parse_args() {
         case "x$1" in
         x--depth)               depth=$2; shift;;
         x--with_padding)        padding=true; shift ;;
-        x--upgrade)             echo "Deprecated option --upgrade ignored: use --upgrade-{etc,extensions,spack} ";  shift;;
+        x--upgrade)             echo "Deprecated option --upgrade ignored: use --upgrade-{etc,extensions,recipes,spack} ";  shift;;
         x--upgrade-*)           eval upgrade_${1##*-}=1; shift;;
         x--spack_release)       spack_release=$2; shift; shift ;;
         x--spack_repo)          spack_repo=$2; shift; shift;;
         x--minimal)             minimal=true; shift ;;
         x--query-packages)      query_packages=true; shift ;;
         x--help)                usage; exit;;
         x--no_buildcache)       use_buildcache=false; shift;;
+        x--no-recipe-repos)     no_recipes=true; shift;;
         x--repover)             repovers="$repovers $2"; shift; shift;;
         x-v|x--verbose)         verbose=true; shift;;
         x-u)                    echo "Deprecated option -u ignored"; shift;;