From a8f084e25c5f8e5affd125ea26ce0e2539418186 Mon Sep 17 00:00:00 2001
From: Xylar Asay-Davis <xylarstorm@gmail.com>
Date: Mon, 24 Jul 2023 11:14:59 +0200
Subject: [PATCH 01/17] Remove --with_netlib_lapack flag

This is tedious to type and the suffix causes environment names
that are too long (spack can't create a valid shebang).
---
 deploy/albany_supported.txt | 1 +
 deploy/bootstrap.py         | 7 +------
 deploy/petsc_supported.txt  | 1 -
 deploy/shared.py            | 8 ++------
 4 files changed, 4 insertions(+), 13 deletions(-)

diff --git a/deploy/albany_supported.txt b/deploy/albany_supported.txt
index 778bca2c4..e09445797 100644
--- a/deploy/albany_supported.txt
+++ b/deploy/albany_supported.txt
@@ -5,3 +5,4 @@ chicoma-cpu, gnu, mpich
 chrysalis, gnu, openmpi
 compy, gnu, openmpi
 pm-cpu, gnu, mpich
+morpheus, gnu, openmpi
diff --git a/deploy/bootstrap.py b/deploy/bootstrap.py
index 956e976c8..b5cec958e 100755
--- a/deploy/bootstrap.py
+++ b/deploy/bootstrap.py
@@ -186,11 +186,6 @@ def get_env_setup(args, config, machine, compiler, mpi, env_type, source_path,
     else:
         config.set('deploy', 'albany', 'None')
 
-    if args.with_netlib_lapack:
-        lib_suffix = f'{lib_suffix}_netlib_lapack'
-    else:
-        config.set('deploy', 'lapack', 'None')
-
     if args.with_petsc:
         lib_suffix = f'{lib_suffix}_petsc'
         logger.info('Turning off OpenMP because it doesn\'t work well '
@@ -198,6 +193,7 @@ def get_env_setup(args, config, machine, compiler, mpi, env_type, source_path,
         args.without_openmp = True
     else:
         config.set('deploy', 'petsc', 'None')
+        config.set('deploy', 'lapack', 'None')
 
     activ_suffix = f'{activ_suffix}{lib_suffix}'
 
@@ -969,7 +965,6 @@ def main():  # noqa: C901
             check_env(script_filename, conda_env_name, logger)
 
         if env_type == 'release' and not (args.with_albany or
-                                          args.with_netlib_lapack or
                                           args.with_petsc):
             # make a symlink to the activation script
             link = os.path.join(activ_path,
diff --git a/deploy/petsc_supported.txt b/deploy/petsc_supported.txt
index d47a438c9..5bc50ab6a 100644
--- a/deploy/petsc_supported.txt
+++ b/deploy/petsc_supported.txt
@@ -7,5 +7,4 @@ chicoma-cpu, gnu, mpich
 chrysalis, intel, openmpi
 chrysalis, gnu, openmpi
 compy, intel, impi
-compy, gnu, openmpi
 pm-cpu, gnu, mpich
diff --git a/deploy/shared.py b/deploy/shared.py
index 9fe560895..bf705ad0a 100644
--- a/deploy/shared.py
+++ b/deploy/shared.py
@@ -58,14 +58,10 @@ def parse_args(bootstrap):
                         action='store_true',
                         help="Whether to include albany in the spack "
                              "environment.")
-    parser.add_argument("--with_netlib_lapack", dest="with_netlib_lapack",
-                        action='store_true',
-                        help="Whether to include Netlib-LAPACK in the spack "
-                             "environment.")
     parser.add_argument("--with_petsc", dest="with_petsc",
                         action='store_true',
-                        help="Whether to include PETSc in the spack "
-                             "environment.")
+                        help="Whether to include PETSc and Netlib-LAPACK in "
+                             "the spack environment.")
     parser.add_argument("--without_openmp", dest="without_openmp",
                         action='store_true',
                         help="If this flag is included, OPENMP=false will "

From 239f5842ca9368a808d1c5a48b04c9000ccde9a2 Mon Sep 17 00:00:00 2001
From: Xylar Asay-Davis <xylarstorm@gmail.com>
Date: Wed, 17 May 2023 17:11:45 -0600
Subject: [PATCH 02/17] Name the spack clone the same as the spack env

This way, different users can create different spack clones.
Multiple users cannot modify the same clone.
---
 deploy/bootstrap.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/deploy/bootstrap.py b/deploy/bootstrap.py
index b5cec958e..2240a8468 100755
--- a/deploy/bootstrap.py
+++ b/deploy/bootstrap.py
@@ -15,7 +15,6 @@
 from jinja2 import Template
 from mache import MachineInfo, discover_machine
 from mache.spack import get_spack_script, make_spack_env
-from mache.version import __version__ as mache_version
 from shared import (
     check_call,
     get_conda_base,
@@ -398,7 +397,7 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env,
     petsc = config.get('deploy', 'petsc')
     scorpio = config.get('deploy', 'scorpio')
 
-    spack_branch_base = f'{spack_base}/spack_for_mache_{mache_version}'
+    spack_branch_base = f'{spack_base}/{spack_env}'
 
     specs = list()
 

From a7797b16d4df5737a9973772dd7fc1d99b9dea81 Mon Sep 17 00:00:00 2001
From: Xylar Asay-Davis <xylarstorm@gmail.com>
Date: Sun, 30 Jul 2023 11:26:22 +0200
Subject: [PATCH 03/17] Update to v0.2.0

---
 polaris/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/polaris/version.py b/polaris/version.py
index e1b4fa56b..7fd229a32 100644
--- a/polaris/version.py
+++ b/polaris/version.py
@@ -1 +1 @@
-__version__ = '0.1.0-alpha.3'
+__version__ = '0.2.0'

From e78fbf264c6b1c93f5cbcf9f86e11e67721af836 Mon Sep 17 00:00:00 2001
From: Xylar Asay-Davis <xylarstorm@gmail.com>
Date: Thu, 25 May 2023 19:43:17 -0500
Subject: [PATCH 04/17] Fix quotes in spack env. yaml

---
 deploy/bootstrap.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/deploy/bootstrap.py b/deploy/bootstrap.py
index 2240a8468..fc3101e65 100755
--- a/deploy/bootstrap.py
+++ b/deploy/bootstrap.py
@@ -402,7 +402,7 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env,
     specs = list()
 
     if cmake != 'None':
-        specs.append(f'cmake "@{cmake}"')
+        specs.append(f'"cmake@{cmake}"')
 
     e3sm_hdf5_netcdf = config.getboolean('deploy', 'use_e3sm_hdf5_netcdf')
     if not e3sm_hdf5_netcdf:
@@ -411,27 +411,28 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env,
         netcdf_fortran = config.get('deploy', 'netcdf_fortran')
         pnetcdf = config.get('deploy', 'pnetcdf')
         specs.extend([
-            f'hdf5@{hdf5}+cxx+fortran+hl+mpi+shared',
-            f'netcdf-c@{netcdf_c}+mpi~parallel-netcdf',
-            f'netcdf-fortran@{netcdf_fortran}',
-            f'parallel-netcdf@{pnetcdf}+cxx+fortran'])
+            f'"hdf5@{hdf5}+cxx+fortran+hl+mpi+shared"',
+            f'"netcdf-c@{netcdf_c}+mpi~parallel-netcdf"',
+            f'"netcdf-fortran@{netcdf_fortran}"',
+            f'"parallel-netcdf@{pnetcdf}+cxx+fortran"'])
 
     if esmf != 'None':
-        specs.append(f'esmf@{esmf}+mpi+netcdf~pio+pnetcdf')
+        specs.append(f'"esmf@{esmf}+mpi+netcdf+pnetcdf"')
     if lapack != 'None':
-        specs.append(f'netlib-lapack@{lapack}')
+        specs.append(f'"netlib-lapack@{lapack}"')
         include_e3sm_lapack = False
     else:
         include_e3sm_lapack = True
     if petsc != 'None':
-        specs.append(f'petsc@{petsc}+mpi+batch')
+        specs.append(f'"petsc@{petsc}+mpi+batch"')
 
     if scorpio != 'None':
         specs.append(
-            f'scorpio@{scorpio}+pnetcdf~timing+internal-timing~tools+malloc')
+            f'"scorpio'
+            f'@{scorpio}+pnetcdf~timing+internal-timing~tools+malloc"')
 
     if albany != 'None':
-        specs.append(f'albany@{albany}+mpas')
+        specs.append(f'"albany@{albany}+mpas"')
 
     yaml_template: str | None = None
     template_path = f'{spack_template_path}/{machine}_{compiler}_{mpi}.yaml'

From 8ff9eb4fd4514d79bcab8463d4bd4193586741ef Mon Sep 17 00:00:00 2001
From: Xylar Asay-Davis <xylarstorm@gmail.com>
Date: Sun, 30 Jul 2023 11:33:24 +0200
Subject: [PATCH 05/17] Update esmf spack variant

---
 deploy/bootstrap.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/bootstrap.py b/deploy/bootstrap.py
index fc3101e65..d30d20f6d 100755
--- a/deploy/bootstrap.py
+++ b/deploy/bootstrap.py
@@ -417,7 +417,7 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env,
             f'"parallel-netcdf@{pnetcdf}+cxx+fortran"'])
 
     if esmf != 'None':
-        specs.append(f'"esmf@{esmf}+mpi+netcdf+pnetcdf"')
+        specs.append(f'"esmf@{esmf}+mpi+netcdf~pnetcdf~external-parallelio"')
     if lapack != 'None':
         specs.append(f'"netlib-lapack@{lapack}"')
         include_e3sm_lapack = False

From 6da13e714d777e07cfc6b97d8a35676acc836bef Mon Sep 17 00:00:00 2001
From: Xylar Asay-Davis <xylarstorm@gmail.com>
Date: Sun, 30 Jul 2023 11:54:03 +0200
Subject: [PATCH 06/17] Update dependency versions

---
 deploy/conda-dev-spec.template |  5 ++++-
 deploy/default.cfg             | 23 ++++++++++++-----------
 2 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/deploy/conda-dev-spec.template b/deploy/conda-dev-spec.template
index 0fc019b4b..60cd16871 100644
--- a/deploy/conda-dev-spec.template
+++ b/deploy/conda-dev-spec.template
@@ -31,7 +31,7 @@ otps={{ otps }}
 progressbar2
 pyamg>=4.2.2
 pyproj
-pyremap>=0.0.16,<0.1.0
+pyremap>=1.0.0,<2.0.0
 ruamel.yaml
 requests
 scipy>=1.8.0
@@ -76,3 +76,6 @@ sphinx_rtd_theme
 myst-parser
 sphinx-multiversion
 rst-to-myst
+
+# Visualization
+ncview
diff --git a/deploy/default.cfg b/deploy/default.cfg
index 73427a0d9..4ecf7cacd 100644
--- a/deploy/default.cfg
+++ b/deploy/default.cfg
@@ -14,30 +14,31 @@ recreate = False
 suffix =
 
 # the python version
-python = 3.10
+python = 3.11
 
 # the MPI version (nompi, mpich or openmpi)
 mpi = nompi
 
 # versions of conda packages
-geometric_features = 1.0.1
+geometric_features = 1.2.0
 jigsaw = 0.9.14
 jigsawpy = 0.3.3
-mache = 1.14.0
-mpas_tools = 0.18.0
+mache = 1.16.0
+mpas_tools = 0.21.0
 otps = 2021.10
+parallelio = 2.6.0
 
 # versions of conda or spack packages (depending on machine type)
-esmf = 8.2.0
-netcdf_c = 4.8.1
-netcdf_fortran = 4.6
-pnetcdf = 1.12.2
-scorpio = 1.3.2
+esmf = 8.4.2
+netcdf_c = 4.9.2
+netcdf_fortran = 4.6.1
+pnetcdf = 1.12.3
 
 # versions of spack packages
 albany = develop
 # cmake newer than 3.23.0 needed for Trilinos
 cmake = 3.23.0:
-hdf5 = 1.12.1
+hdf5 = 1.14.1
 lapack = 3.9.1
-petsc = 3.16.1
+petsc = 3.19.1
+scorpio = 1.4.1

From f0f1ba46821a5618fe8c11a61896ad2f3c4ae461 Mon Sep 17 00:00:00 2001
From: Xylar Asay-Davis <xylarstorm@gmail.com>
Date: Sun, 30 Jul 2023 11:33:57 +0200
Subject: [PATCH 07/17] Switch conda env to use conda-forge pnetcdf and pio

---
 deploy/bootstrap.py            | 13 ++++++++++---
 deploy/conda-dev-spec.template |  2 +-
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/deploy/bootstrap.py b/deploy/bootstrap.py
index d30d20f6d..da79d48d5 100755
--- a/deploy/bootstrap.py
+++ b/deploy/bootstrap.py
@@ -285,7 +285,7 @@ def build_conda_env(config, env_type, recreate, mpi, conda_mpi, version,
 
         for package in ['esmf', 'geometric_features', 'jigsaw', 'jigsawpy',
                         'mache', 'mpas_tools', 'netcdf_c', 'netcdf_fortran',
-                        'otps', 'pnetcdf', 'scorpio']:
+                        'otps', 'parallelio', 'pnetcdf']:
             replacements[package] = config.get('deploy', package)
 
         spec_file = template.render(**replacements)
@@ -361,6 +361,12 @@ def get_env_vars(machine, compiler, mpilib):
                    f'export I_MPI_F77=ifort\n' \
                    f'export I_MPI_F90=ifort\n'
 
+    if machine.startswith('conda'):
+        # we're using parallelio so we don't have ADIOS support
+        env_vars = \
+            f'{env_vars}' \
+            f'export HAVE_ADIOS=false\n'
+
     if platform.system() == 'Linux' and machine.startswith('conda'):
         env_vars = \
             f'{env_vars}' \
@@ -387,8 +393,9 @@ def get_env_vars(machine, compiler, mpilib):
     return env_vars
 
 
-def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env,
-                    spack_base, spack_template_path, env_vars, tmpdir, logger):
+def build_spack_env(config, update_spack, machine, compiler, mpi,  # noqa: C901
+                    spack_env, spack_base, spack_template_path, env_vars,
+                    tmpdir, logger):
 
     albany = config.get('deploy', 'albany')
     cmake = config.get('deploy', 'cmake')
diff --git a/deploy/conda-dev-spec.template b/deploy/conda-dev-spec.template
index 60cd16871..df397d635 100644
--- a/deploy/conda-dev-spec.template
+++ b/deploy/conda-dev-spec.template
@@ -59,7 +59,7 @@ fortran-compiler
 libnetcdf={{ netcdf_c }}={{ mpi_prefix }}_*
 netcdf-fortran={{ netcdf_fortran }}={{ mpi_prefix }}_*
 libpnetcdf={{ pnetcdf }}={{ mpi_prefix }}_*
-scorpio={{ scorpio }}={{ mpi_prefix }}_*
+parallelio={{ parallelio }}={{ mpi_prefix }}_*
 m4
 make
 {{ mpi }}

From c1dde502e0407e1a5fdddc55acf25236c57b0597 Mon Sep 17 00:00:00 2001
From: Xylar Asay-Davis <xylarstorm@gmail.com>
Date: Sun, 30 Jul 2023 12:25:04 +0200
Subject: [PATCH 08/17] Update E3SM-Project submodule to 4fb8eddcf4

---
 e3sm_submodules/E3SM-Project | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/e3sm_submodules/E3SM-Project b/e3sm_submodules/E3SM-Project
index b9c1a8c5c..4fb8eddcf 160000
--- a/e3sm_submodules/E3SM-Project
+++ b/e3sm_submodules/E3SM-Project
@@ -1 +1 @@
-Subproject commit b9c1a8c5cfdb5045995c4af05d9ebb7c6f736d32
+Subproject commit 4fb8eddcf457d18d5619dd4a1c4785ed7c45f725

From 6ed4ebbb4d61b8e2a3954f95c3d2a8852cd2333e Mon Sep 17 00:00:00 2001
From: Xylar Asay-Davis <xylarstorm@gmail.com>
Date: Mon, 15 May 2023 21:30:52 -0600
Subject: [PATCH 09/17] For known machines, use config file in spack builds

This will allow support for machines that are unknown to mache.
---
 deploy/bootstrap.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/deploy/bootstrap.py b/deploy/bootstrap.py
index da79d48d5..abdb7df67 100755
--- a/deploy/bootstrap.py
+++ b/deploy/bootstrap.py
@@ -445,10 +445,18 @@ def build_spack_env(config, update_spack, machine, compiler, mpi,  # noqa: C901
     template_path = f'{spack_template_path}/{machine}_{compiler}_{mpi}.yaml'
     if os.path.exists(template_path):
         yaml_template = template_path
+
+    if machine is not None:
+        here = os.path.abspath(os.path.dirname(__file__))
+        machine_config = os.path.join(here, '..', 'polaris', 'machines',
+                                      f'{machine}.cfg')
+    else:
+        machine_config = None
+
     if update_spack:
         make_spack_env(spack_path=spack_branch_base, env_name=spack_env,
                        spack_specs=specs, compiler=compiler, mpi=mpi,
-                       machine=machine,
+                       machine=machine, config_file=machine_config,
                        include_e3sm_lapack=include_e3sm_lapack,
                        include_e3sm_hdf5_netcdf=e3sm_hdf5_netcdf,
                        yaml_template=yaml_template, tmpdir=tmpdir)
@@ -464,7 +472,7 @@ def build_spack_env(config, update_spack, machine, compiler, mpi,  # noqa: C901
 
     spack_script = get_spack_script(
         spack_path=spack_branch_base, env_name=spack_env, compiler=compiler,
-        mpi=mpi, shell='sh', machine=machine,
+        mpi=mpi, shell='sh', machine=machine, config_file=machine_config,
         include_e3sm_lapack=include_e3sm_lapack,
         include_e3sm_hdf5_netcdf=e3sm_hdf5_netcdf,
         yaml_template=yaml_template)

From 2c87502d4be9a6e2a694ed0f29c7f72238faa1ff Mon Sep 17 00:00:00 2001
From: Xylar Asay-Davis <xylarstorm@gmail.com>
Date: Mon, 15 May 2023 21:54:06 -0600
Subject: [PATCH 10/17] Skip exception when getting config from mache

To support machines that aren't known to mache, we need to suppress
exceptions when trying to read mache config options for
a machine.
---
 polaris/setup.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/polaris/setup.py b/polaris/setup.py
index 897f64b52..a80ad53e8 100644
--- a/polaris/setup.py
+++ b/polaris/setup.py
@@ -417,7 +417,8 @@ def _get_basic_config(config_file, machine, component_path, component):
 
     # add the E3SM config options from mache
     if machine is not None:
-        config.add_from_package('mache.machines', f'{machine}.cfg')
+        config.add_from_package('mache.machines', f'{machine}.cfg',
+                                exception=False)
 
     # add the polaris machine config file
     if machine is None:

From e32e481ff2225cade24f52863f2be9d91a65633c Mon Sep 17 00:00:00 2001
From: Xylar Asay-Davis <xylarstorm@gmail.com>
Date: Sun, 4 Jun 2023 23:17:38 -0600
Subject: [PATCH 11/17] Support config option for identifying known machines

This merge adds a config option `hostname_contains` to a new
config section `discovery` that can be used to identify the
machine by hostname if `mache` is unable to identify it.

This capability has been added both when calling the configure
script and when setting up test cases.
---
 deploy/bootstrap.py          | 59 ++++++++++++++++++++++++++++++++++--
 polaris/machines/__init__.py | 56 ++++++++++++++++++++++++++++++++++
 polaris/setup.py             |  3 +-
 3 files changed, 113 insertions(+), 5 deletions(-)

diff --git a/deploy/bootstrap.py b/deploy/bootstrap.py
index abdb7df67..20fec0ed7 100755
--- a/deploy/bootstrap.py
+++ b/deploy/bootstrap.py
@@ -6,6 +6,7 @@
 import os
 import platform
 import shutil
+import socket
 import stat
 import subprocess
 from configparser import ConfigParser
@@ -13,7 +14,8 @@
 
 import progressbar
 from jinja2 import Template
-from mache import MachineInfo, discover_machine
+from mache import MachineInfo
+from mache import discover_machine as mache_discover_machine
 from mache.spack import get_spack_script, make_spack_env
 from shared import (
     check_call,
@@ -815,6 +817,57 @@ def check_supported(library, machine, compiler, mpi, source_path):
                      f'on {machine}')
 
 
+def safe_rmtree(path):
+    try:
+        shutil.rmtree(path)
+    except OSError:
+        pass
+
+
+def discover_machine(quiet=False):
+    """
+    Figure out the machine from the host name
+
+    Parameters
+    ----------
+    quiet : bool, optional
+        Whether to print warnings if the machine name is ambiguous
+
+    Returns
+    -------
+    machine : str
+        The name of the current machine
+    """
+    machine = mache_discover_machine(quiet=quiet)
+    if machine is None:
+        possible_hosts = get_possible_hosts()
+        hostname = socket.gethostname()
+        for possible_machine, hostname_contains in possible_hosts.items():
+            if hostname_contains in hostname:
+                machine = possible_machine
+                break
+    return machine
+
+
+def get_possible_hosts():
+    here = os.path.abspath(os.path.dirname(__file__))
+    files = sorted(glob.glob(os.path.join(
+        here, '..', 'polaris', 'machines', '*.cfg')))
+
+    possible_hosts = dict()
+    for filename in files:
+        machine = os.path.splitext(os.path.split(filename)[1])[0]
+        config = ConfigParser()
+        config.read(filename)
+        if config.has_section('discovery') and \
+                config.has_option('discovery', 'hostname_contains'):
+            hostname_contains = config.get('discovery',
+                                           'hostname_contains')
+            possible_hosts[machine] = hostname_contains
+
+    return possible_hosts
+
+
 def main():  # noqa: C901
     args = parse_args(bootstrap=True)
 
@@ -839,7 +892,7 @@ def main():  # noqa: C901
         else:
             machine = args.machine
 
-    e3sm_machine = machine is not None
+    known_machine = machine is not None
 
     if machine is None and not args.conda_env_only:
         if platform.system() == 'Linux':
@@ -910,7 +963,7 @@ def main():  # noqa: C901
 
         if args.spack_base is not None:
             spack_base = args.spack_base
-        elif e3sm_machine and compiler is not None:
+        elif known_machine and compiler is not None:
             spack_base = get_spack_base(args.spack_base, config)
         else:
             spack_base = None
diff --git a/polaris/machines/__init__.py b/polaris/machines/__init__.py
index e69de29bb..03d057181 100644
--- a/polaris/machines/__init__.py
+++ b/polaris/machines/__init__.py
@@ -0,0 +1,56 @@
+import os
+import socket
+import sys
+from typing import TYPE_CHECKING  # noqa: F401
+
+if TYPE_CHECKING or sys.version_info >= (3, 9, 0):
+    import importlib.resources as imp_res  # noqa: F401
+else:
+    # python <= 3.8
+    import importlib_resources as imp_res  # noqa: F401
+
+from mache import discover_machine as mache_discover_machine
+
+from polaris.config import PolarisConfigParser
+
+
+def discover_machine(quiet=False):
+    """
+    Figure out the machine from the host name
+
+    Parameters
+    ----------
+    quiet : bool, optional
+        Whether to print warnings if the machine name is ambiguous
+
+    Returns
+    -------
+    machine : str
+        The name of the current machine
+    """
+    machine = mache_discover_machine(quiet=quiet)
+    if machine is None:
+        possible_hosts = _get_possible_hosts()
+        hostname = socket.gethostname()
+        for possible_machine, hostname_contains in possible_hosts.items():
+            if hostname_contains in hostname:
+                machine = possible_machine
+                break
+    return machine
+
+
+def _get_possible_hosts():
+    machine_contents = imp_res.contents('polaris.machines')
+    possible_hosts = dict()
+    for filename in machine_contents:
+        if filename.endswith('.cfg'):
+            machine = os.path.split(filename)[0]
+            config = PolarisConfigParser()
+            config.add_from_package('polaris.machines', filename)
+            if config.has_section('discovery') and \
+                    config.has_option('discovery', 'hostname_contains'):
+                hostname_contains = config.get('discovery',
+                                               'hostname_contains')
+                possible_hosts[machine] = hostname_contains
+
+    return possible_hosts
diff --git a/polaris/setup.py b/polaris/setup.py
index a80ad53e8..953f6f867 100644
--- a/polaris/setup.py
+++ b/polaris/setup.py
@@ -5,13 +5,12 @@
 import warnings
 from typing import Dict, List
 
-from mache import discover_machine
-
 from polaris import TestCase, provenance
 from polaris.components import get_components
 from polaris.config import PolarisConfigParser
 from polaris.io import symlink
 from polaris.job import write_job_script
+from polaris.machines import discover_machine
 
 
 def setup_cases(work_dir, tests=None, numbers=None, config_file=None,

From 8883dd80d01f7075eac098063391b25bdf8c98cc Mon Sep 17 00:00:00 2001
From: Xylar Asay-Davis <xylarstorm@gmail.com>
Date: Mon, 15 May 2023 21:32:42 -0600
Subject: [PATCH 12/17] Add support for new machine `morpheus`

This is my laptop as an example of a machine not supported by
mache.
---
 deploy/spack/morpheus_gnu_openmpi.yaml | 38 ++++++++++++++++
 polaris/machines/morpheus.cfg          | 60 ++++++++++++++++++++++++++
 2 files changed, 98 insertions(+)
 create mode 100644 deploy/spack/morpheus_gnu_openmpi.yaml
 create mode 100644 polaris/machines/morpheus.cfg

diff --git a/deploy/spack/morpheus_gnu_openmpi.yaml b/deploy/spack/morpheus_gnu_openmpi.yaml
new file mode 100644
index 000000000..6e5a4aa8d
--- /dev/null
+++ b/deploy/spack/morpheus_gnu_openmpi.yaml
@@ -0,0 +1,38 @@
+spack:
+  specs:
+  - gcc
+  - openmpi
+{{ specs }}
+  concretizer:
+    unify: true
+  packages:
+    all:
+      compiler: [gcc@11.3.0]
+      providers:
+        mpi: [openmpi]
+    curl:
+      externals:
+      - spec:  curl@7.81.0
+        prefix: /usr
+      buildable: false
+    gcc:
+      externals:
+      - spec: gcc@11.3.0
+        prefix: /usr
+      buildable: false
+  config:
+    install_missing_compilers: false
+  compilers:
+  - compiler:
+      spec: gcc@11.3.0
+      paths:
+        cc: /usr/bin/gcc
+        cxx: /usr/bin/g++
+        f77: /usr/bin/gfortran
+        fc: /usr/bin/gfortran
+      flags: {}
+      operating_system: ubuntu22.04
+      target: x86_64
+      modules: []
+      environment: {}
+      extra_rpaths: []
diff --git a/polaris/machines/morpheus.cfg b/polaris/machines/morpheus.cfg
new file mode 100644
index 000000000..d4641ab0d
--- /dev/null
+++ b/polaris/machines/morpheus.cfg
@@ -0,0 +1,60 @@
+# The parallel section describes options related to running jobs in parallel
+[parallel]
+
+# parallel system of execution: slurm, cobalt or single_node
+system = single_node
+
+# whether to use mpirun or srun to run a task
+parallel_executable = mpirun
+
+# cores per node on the machine
+cores_per_node = 8
+
+
+# Config options related to spack environments
+[spack]
+
+# whether to load modules from the spack yaml file before loading the spack
+# environment
+modules_before = False
+
+# whether to load modules from the spack yaml file after loading the spack
+# environment
+modules_after = False
+
+
+# The paths section describes paths that are used within the ocean core test
+# cases.
+[paths]
+
+# A shared root directory where MPAS standalone data can be found
+database_root = /home/xylar/data/mpas/mpas_standalonedata
+
+# the path to the base conda environment where polaris environments have
+# been created
+polaris_envs = /home/xylar/data/mpas/polaris_envs
+
+
+# Options related to deploying a polaris conda environment on supported
+# machines
+[deploy]
+
+# the compiler set to use for system libraries and MPAS builds
+compiler = gnu
+
+# the system MPI library to use for gnu compiler
+mpi_gnu = openmpi
+
+# the base path for spack environments used by polaris
+spack = /home/xylar/data/mpas/spack
+
+# whether to use the same modules for hdf5, netcdf-c, netcdf-fortran and
+# pnetcdf as E3SM (spack modules are used otherwise)
+use_e3sm_hdf5_netcdf = False
+
+
+# Options related to machine discovery
+[discovery]
+
+# a substring used to identify this machine from its hostname
+hostname_contains = morpheus

From cc4069a9e01129f157d9bf4dd18414afa0274797 Mon Sep 17 00:00:00 2001
From: Xylar Asay-Davis <xylarstorm@gmail.com>
Date: Fri, 21 Jul 2023 10:46:47 +0200
Subject: [PATCH 13/17] Add hack to make sure scorpio, not esmf, provides
 libpio*.a

---
 deploy/bootstrap.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/deploy/bootstrap.py b/deploy/bootstrap.py
index 20fec0ed7..e10d940a4 100755
--- a/deploy/bootstrap.py
+++ b/deploy/bootstrap.py
@@ -435,10 +435,19 @@ def build_spack_env(config, update_spack, machine, compiler, mpi,  # noqa: C901
     if petsc != 'None':
         specs.append(f'"petsc@{petsc}+mpi+batch"')
 
+    custom_spack = ''
     if scorpio != 'None':
         specs.append(
             f'"scorpio'
             f'@{scorpio}+pnetcdf~timing+internal-timing~tools+malloc"')
+        # make sure scorpio, not esmf, libraries are linked
+        lib_path = f'{spack_branch_base}/var/spack/environments/' \
+                   f'{spack_env}/.spack-env/view/lib'
+        scorpio_lib_path = '$(spack find --format "{prefix}" scorpio)'
+        custom_spack = \
+            f'{custom_spack}' \
+            f'ln -sfn {scorpio_lib_path}/lib/libpioc.a {lib_path}\n' \
+            f'ln -sfn {scorpio_lib_path}/lib/libpiof.a {lib_path}\n'
 
     if albany != 'None':
         specs.append(f'"albany@{albany}+mpas"')
@@ -461,7 +470,8 @@ def build_spack_env(config, update_spack, machine, compiler, mpi,  # noqa: C901
                        machine=machine, config_file=machine_config,
                        include_e3sm_lapack=include_e3sm_lapack,
                        include_e3sm_hdf5_netcdf=e3sm_hdf5_netcdf,
-                       yaml_template=yaml_template, tmpdir=tmpdir)
+                       yaml_template=yaml_template, tmpdir=tmpdir,
+                       custom_spack=custom_spack)
 
         # remove ESMC/ESMF include files that interfere with MPAS time keeping
         include_path = f'{spack_branch_base}/var/spack/environments/' \

From 7c18126211c3c42f3a15277e0d1d27219969ee61 Mon Sep 17 00:00:00 2001
From: Xylar Asay-Davis <xylarstorm@gmail.com>
Date: Sun, 30 Jul 2023 14:46:40 +0200
Subject: [PATCH 14/17] Fix mamba init

We should no longer need the workaround where be move aside the
user's .bashrc, etc.
---
 deploy/shared.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/deploy/shared.py b/deploy/shared.py
index bf705ad0a..44036e847 100644
--- a/deploy/shared.py
+++ b/deploy/shared.py
@@ -173,19 +173,16 @@ def install_mambaforge(conda_base, activate_base, logger):
         check_call(command, logger=logger)
         os.remove(mambaforge)
 
-    backup_bashrc()
-
     print('Doing initial setup\n')
     commands = f'{activate_base} && ' \
                f'conda config --add channels conda-forge && ' \
                f'conda config --set channel_priority strict && ' \
+               f'mamba update -y "mamba>1.3.1" "conda>=23.1.0" && ' \
                f'mamba update -y --all && ' \
-               f'mamba init'
+               f'mamba init --no-user'
 
     check_call(commands, logger=logger)
 
-    restore_bashrc()
-
 
 def backup_bashrc():
     home_dir = os.path.expanduser('~')

From 9385c53b17133b8194dd07c235d6a62f7cbc432d Mon Sep 17 00:00:00 2001
From: Xylar Asay-Davis <xylarstorm@gmail.com>
Date: Mon, 31 Jul 2023 10:57:56 +0200
Subject: [PATCH 15/17] Add documentation on adding a supported machine

---
 docs/developers_guide/machines/index.md | 271 ++++++++++++++++++++++++
 1 file changed, 271 insertions(+)

diff --git a/docs/developers_guide/machines/index.md b/docs/developers_guide/machines/index.md
index b23668f3c..a2df3b427 100644
--- a/docs/developers_guide/machines/index.md
+++ b/docs/developers_guide/machines/index.md
@@ -114,3 +114,274 @@ Under OSX, you can build the MPAS model with
 ```bash
 make gfortran-clang
 ```
+
+(dev-add-supported-machine)=
+
+## Adding a New Supported Machine
+
+If you want to add a new supported machine, you
+
+### Adding a Machine Config File
+
+The first step in adding a new supported machine to add a config file in
+`polaris/machines`.  The config file needs to describe the parallel
+environment and some paths where shared Spack environments will be installed
+and shared data will be downloaded.  The easiest place to start is one of the
+examples provided (machine `morpheus` for now, but more will be added soon.)
+
+```cfg
+# The parallel section describes options related to running jobs in parallel
+[parallel]
+
+# parallel system of execution: slurm, cobalt or single_node
+system = single_node
+
+# whether to use mpirun or srun to run a task
+parallel_executable = mpirun
+
+# cores per node on the machine
+cores_per_node = 8
+
+
+# Config options related to spack environments
+[spack]
+
+# whether to load modules from the spack yaml file before loading the spack
+# environment
+modules_before = False
+
+# whether to load modules from the spack yaml file after loading the spack
+# environment
+modules_after = False
+
+
+# The paths section describes paths that are used within the ocean core test
+# cases.
+[paths]
+
+# A shared root directory where MPAS standalone data can be found
+database_root = /home/xylar/data/mpas/mpas_standalonedata
+
+# the path to the base conda environment where polaris environments have
+# been created
+polaris_envs = /home/xylar/data/mpas/polaris_envs
+
+
+# Options related to deploying a polaris conda environment on supported
+# machines
+[deploy]
+
+# the compiler set to use for system libraries and MPAS builds
+compiler = gnu
+
+# the system MPI library to use for gnu compiler
+mpi_gnu = openmpi
+
+# the base path for spack environments used by polaris
+spack = /home/xylar/data/mpas/spack
+
+# whether to use the same modules for hdf5, netcdf-c, netcdf-fortran and
+# pnetcdf as E3SM (spack modules are used otherwise)
+use_e3sm_hdf5_netcdf = False
+
+
+# Options related to machine discovery
+[discovery]
+
+# a substring used to identify this machine from its hostname
+hostname_contains = morpheus
+```
+
+The `[parallel]` section should describe the type of parallel queuing
+system (currently only `slurm` or `single_node` are supported), the number
+of cores per node and the command for running an MPI executable (typically
+`srun` for Slurm and `mpirun` for a "single node" machine like a laptop or
+workstation.
+
+The `[spack]` section has some config options to do with loading system
+modules before or after loading a Spack environment.  On a "single node"
+machine, you typically don't have modules so both `modules_before` and
+`modules_after` can be set to `False`.  On a high-performance computing
+(HPC) machine, you may find it is safest to load modules after the Spack
+environment to ensure that certain paths and environment variables are set the
+way the modules have them, rather than the way that Spack would have them.
+The recommended starting point would be `modules_before = False` and
+`modules_after = True`, but could be adjusted as needed if the right shared
+libraries aren't being found when you try to build an MPAS component.
+
+In the `[paths]` section, you will first give a path where you would like
+to store shared data files used in polaris test cases in `database_root`.
+Polaris will create this directory if it doesn't exist.  Then, you can specify
+`polaris_envs` as a path where shared conda environments will be installed
+for polaris releases.  If developers always create their own conda
+environments, this path will never be used.
+
+In `[deploy]`, you will specify config options used in setting up conda
+and Spack environments for developers.  The `compiler` is the default
+compiler to use for your system.  You must supply a corresponding
+`mpi_<compiler>` for each supported compiler (not just the default compiler)
+that specifies the default MPI library for that compiler.  If you only support
+one compiler and MPI library, that's pretty simple: `compiler` is the name
+of the compiler (e.g. `intel` or `gnu`) and `mpi_<compiler>` is the
+MPI library (e.g. `compiler_gnu = mpich` or `compiler_intel = openmpi`).
+The `spack` option specifies a path where Spack environment will be created.
+The option `use_e3sm_hdf5_netcdf = False` indicates that you will not use
+the E3SM default modules for HDF5 and NetCDF libraries (which are not available
+for machines installed in the way described here).
+
+Finally, `[discovery]` allows you to add a `hostname_contains` that is used
+to automatically identify your machine based on its hostname.  If your machine
+has multiple login nodes with different hostnames, hopefully, a string common
+to all login nodes can be used here.  If your machine has a unique hostname,
+simply give that.  This option saves developers from having to specify
+`--machine <machine>` each time they setup polaris environments or test
+cases.
+
+
+### Describing a Spack Environment
+
+The next step is to create a template YAML file that can be used to create
+[Spack environments](https://spack-tutorial.readthedocs.io/en/latest/tutorial_environments.html)
+for your machine. Polaris uses Spack environments to build packages that need
+MPI support or which should be build for some other reason with system
+compilers rather than coming from pre-built conda packages. Using a Spack
+environment allows these packages to be built together in a consistent way that
+is not guaranteed if you try to install dependencies one-by-one.  In Spack
+parlance, this is known as
+[unified concretization](https://spack.readthedocs.io/en/latest/environments.html#spec-concretization).
+
+To do this, you will create a file `deploy/spack/<machine>_<compiler>_<mpi>.yaml`
+similar to the following example for an Ubuntu laptop:
+
+``` yaml
+spack:
+  specs:
+  - gcc
+  - openmpi
+{{ specs }}
+  concretizer:
+    unify: true
+  packages:
+    all:
+      compiler: [gcc@11.3.0]
+      providers:
+        mpi: [openmpi]
+    curl:
+      externals:
+      - spec:  curl@7.81.0
+        prefix: /usr
+      buildable: false
+    gcc:
+      externals:
+      - spec: gcc@11.3.0
+        prefix: /usr
+      buildable: false
+  config:
+    install_missing_compilers: false
+  compilers:
+  - compiler:
+      spec: gcc@11.3.0
+      paths:
+        cc: /usr/bin/gcc
+        cxx: /usr/bin/g++
+        f77: /usr/bin/gfortran
+        fc: /usr/bin/gfortran
+      flags: {}
+      operating_system: ubuntu22.04
+      target: x86_64
+      modules: []
+      environment: {}
+      extra_rpaths: []
+```          
+          
+Typically your system will already have compilers if nothing else, and this is
+what we assume here.  Give the appropriate path (replace `/usr` with the
+appropriate path on your system).  We have had better luck with `gcc` than
+other compilers like Intel so far so for new supported machines so that's our
+recommendation.  Use `gcc --version` to determine the version and replace
+`11.3.0` with this number.
+
+Finally, you might need to update the `target` and `operating_system`.
+This is a bit of a "catch 22" in that you can use Spack to find this out but
+polaris is designed to clone and set up Spack for you so we assume you don't
+have it yet.  For now, make your best guess using the info on
+[this page](https://spack.readthedocs.io/en/latest/basic_usage.html#architecture-specifiers)
+and correct it later if necessary.
+
+You may need to load a system module to get the compilers and potentially other
+libraries such as MPI, HDF5, and NetCDF-C if you prefer to use system modules
+rather than having Spack build them.  If this is the case, the best way to do
+this is to add a file
+`conda/spack/<machine>_<compiler>_<mpi>.sh` along these lines:
+
+``` bash
+module purge
+module load perl/5.32.0-bsnc6lt
+module load gcc/9.2.0-ugetvbp
+module load openmpi/4.1.3-sxfyy4k
+module load intel-mkl/2020.4.304-n3b5fye
+module load hdf5/1.10.7-j3zxncu
+module load netcdf-c/4.4.1-7ohuiwq
+module load netcdf-fortran/4.4.4-k2zu3y5
+module load parallel-netcdf/1.11.0-mirrcz7
+```
+
+These modules will be loaded either before or after the spack environment,
+depending on the `modules_before` and `modules_after` config options above.
+You can also add modules in your YAML file but this shouldn't be necessary.
+
+For examples from various supported machines, compilers and MPI libraries, see the
+[mache spack directory](https://github.com/E3SM-Project/mache/tree/main/mache/spack).
+
+### Building the Spack Environment
+
+The next step is to try setting up polaris and asking it to build the Spack
+environment with a command something like:
+
+``` bash
+  ./configure_polaris_envs.py --verbose --update_spack --conda <conda_path> -c gnu -i openmpi ...
+```
+
+The `--update_spack` flag tells polaris to create (or update) a Spack
+environment.  You can specify a directory for testing Spack with the
+`--spack` flag.  You can specify a temporary directory for building spack
+packages with `--tmpdir` (this directory must already exist).  This is useful
+if your `/tmp` space is small (Spack will use several GB of temporary space).
+
+
+Creating the Spack environment may take anywhere from minutes to hours,
+depending on your system.
+
+If dependencies don't build as expected, you may get an error message
+suggesting that your `operating_system` or `target` aren't right. Here's
+an example:
+
+```
+==> Error: concretization failed for the following reasons:
+   1. zlib compiler '%gcc@9.4.0' incompatible with 'os=ubuntu20.04'
+   2. readline compiler '%gcc@9.4.0' incompatible with 'os=ubuntu20.04'
+   3. pkgconf compiler '%gcc@9.4.0' incompatible with 'os=ubuntu20.04'
+```
+In this example, I had specified `operating_system: ubuntu22.04` in the YAML
+file but in fact my operating system is `ubuntu20.04` as shown in the error
+message.
+
+You can run:
+
+``` bash
+source $SPACKDIR/share/spack/setup-env.sh
+spack arch -o
+spack arch -g
+```
+where `$SPACKDIR` is the directory where the Spack repository was cloned
+by polaris (you should see `Cloning into <$SPACKDIR>` in the terminal, which
+will hopefully help you find the right directory).  This should hopefully give
+you something close to what Spack wants.  If you get something like
+`x86_64_v4` for the target, use `x86_64` instead.
+
+If you are getting other error messages, do your best to debug them but also
+feel free to get in touch with the polaris development team and we'll help if
+we can.
+
+If you get everything working well, please feel free to make a pull request
+into the polaris main repo to add your supported machine.

From ec14b6f623c5e9d720f8da9882897ea345955a06 Mon Sep 17 00:00:00 2001
From: Xylar Asay-Davis <xylarstorm@gmail.com>
Date: Mon, 31 Jul 2023 11:10:43 +0200
Subject: [PATCH 16/17] Add instructions for deploying spack envs.

---
 docs/developers_guide/deploying_spack.md | 444 +++++++++++++++++++++++
 docs/index.md                            |   1 +
 2 files changed, 445 insertions(+)
 create mode 100644 docs/developers_guide/deploying_spack.md

diff --git a/docs/developers_guide/deploying_spack.md b/docs/developers_guide/deploying_spack.md
new file mode 100644
index 000000000..45b365f5a
--- /dev/null
+++ b/docs/developers_guide/deploying_spack.md
@@ -0,0 +1,444 @@
+(dev-deploying-spack)=
+
+# Deploying a new spack environment
+
+## Where do we update polaris dependencies?
+
+### Mache
+
+If system modules change in E3SM, we try to stay in sync:
+
+- compilers
+- MPI libraries
+- netcdf-C
+- netcdf-fortran
+- pnetcdf
+- mkl (or other linear algebra libs)
+
+### Spack
+
+Spack is for libraries used by MPAS and tools that need system MPI:
+
+- ESMF
+- SCORPIO
+- Albany
+- PETSc
+- Netlib LAPACK
+
+### Conda
+
+Conda (via conda-forge) is used for python packages and related dependencies
+that don’t need system MPI. Conda environments aren’t shared between
+developers because the polaris you’re developing is part of the conda
+environment.
+
+## Mache
+
+A brief tour of mache.
+
+### Identifying E3SM machines
+
+Polaris and other packages use mache to identify what machine they’re on (when
+you’re on a login node).  This is used when configuring polaris and creating a
+conda environment.  Because of this, there is a “bootstrapping” process where
+a conda environment is created with mache installed in it, then the machine is
+identified and the rest of the polaris setup can continue.
+
+### Config options describing E3SM machines
+
+Mache has config files for each E3SM machine that tell us where E3SM-Unified
+is, where to find diagnostics, and much more, see
+[machines](https://github.com/E3SM-Project/mache/tree/main/mache/machines).
+These config options are shared across packages including:
+
+- MPAS-Analysis
+- E3SM_Diags
+- zppy
+- polaris
+- E3SM-Unified
+
+Polaris uses these config options to know how to make a job script, where to
+find locally cached files in its “databases” and much more.
+
+### Modules, env. variables, etc. for  E3SM machines
+
+Mache keeps its own copy of the E3SM file
+[config_machines.xml](https://github.com/E3SM-Project/E3SM/blob/master/cime_config/machines/config_machines.xml)
+in the package [here](https://github.com/E3SM-Project/mache/blob/main/mache/cime_machine_config/config_machines.xml).
+We try to keep a close eye on E3SM master and update mache when system modules
+for machines that mache knows about get updated.  When this happens, we update
+mache’s copy of `config_machines.xml` and that tells me which modules to
+update in spack, see below.dev_quick_start
+
+### Creating spack environments
+
+Mache has templates for making spack environments on some of the E3SM supported
+machines.  See [spack](https://github.com/E3SM-Project/mache/tree/main/mache/spack).
+It also has functions for building the spack environments with these templates
+using E3SM’s fork of spack (see below).
+
+## Updating spack from polaris with mache from a remote branch
+
+If you haven’t cloned polaris and added Xylar's fork, here’s the process:
+
+```bash
+mkdir polaris
+cd polaris/
+git clone git@github.com:E3SM-Project/polaris.git main
+cd main/
+git remote add xylar/polaris git@github.com:xylar/polaris.git
+```
+
+Now, we need to set up polaris and build spack packages making use of the
+updated mache.  This involves changing the mache version in a couple of places
+in polaris and updating the version of polaris itself to a new alpha, beta or
+rc.  As an example, we will use the branch
+[simplify_local_mache](https://github.com/xylar/polaris/tree/simplify_local_mache).
+
+Often, we will need to test with a `mache` branch that has changes needed
+by polaris.  Here, we will use `<fork>` as a stand-in for the fork of mache
+to use (e.g. `E3SM-Project/mache`) and `<branch>` as the stand-in for a branch on
+that fork (e.g. `main`).
+
+We also need to make sure there is a spack branch for the version of mache.
+This is a branch off of the develop branch on
+[E3SM’s spack repo](https://github.com/E3SM-Project/spack) that has any
+updates to packages required for this version of mache and polaris. The
+spack branch will omit any alpha, beta or rc suffix on the mache version
+because it is intended to be the spack branch we will use once the `mache`
+release happens.  In this example, we will work with the branch
+[spack_for_mache_1.12.0](https://github.com/E3SM-Project/spack/tree/spack_for_mache_1.12.0).
+
+Here's how to get a branch of polaris we're testing (`simplify_local_mache`
+in this case) as a local worktree:
+
+```bash
+# get my branch
+git fetch --all -p
+# make a worktree for checking out my branch
+git worktree add ../simplify_local_mache -b simplify_local_mache \
+    --checkout xylar/polaris/simplify_local_mache
+cd ../simplify_local_mache/
+```
+
+You will also need a local installation of
+[Mambaforge](https://github.com/conda-forge/miniforge#mambaforge).
+Polaris can do this for you if you haven't already installed it.  If you want
+to download it manually, use the Linux x86_64 version for all our supported
+machines.
+
+:::{note}
+We have found that an existing Miniconda3 installation **does not** always
+work well for polaris, so please start with Mambaforge instead.
+:::
+
+:::{note}
+You definitely need your own local Mambaforge installation -- you can’t use
+a system version or a shared one like E3SM-Unified.
+:::
+
+Define a location where Mambaforge is installed or where you want to install
+it:
+
+```bash
+# change to your conda installation
+export CONDA_BASE=${HOME}/mambaforge
+```
+
+Okay, we're finally ready to do a test spack build for polaris.
+To do this, we call the `configure_polaris_env.py` script using
+`--mache_fork`, `--mache_branch`, `--update_spack`, `--spack` and
+`--tmpdir`. Here is an example appropriate for Anvil or Chrysalis:
+
+```bash
+export TMPDIR=/lcrc/group/e3sm/${USER}/spack_temp
+./configure_polaris_envs.py \
+    --conda ${CONDA_BASE} \
+    --mache_fork <fork> \
+    --mache_branch <branch> \
+    --update_spack \
+    --spack /lcrc/group/e3sm/${USER}/spack_test \
+    --tmpdir ${TMPDIR} \
+    --compiler intel intel gnu \
+    --mpi openmpi impi openmpi \
+    --recreate \
+    --verbose
+```
+
+The directory you point to with `--conda` either doesn't exist or contains
+your existing installation of Mambaforge.
+
+When you supply `--mache_fork` and `--mache_branch`, polaris will clone
+a fork of the `mache` repo and check out the requested branch, then install
+that version of mache into both the polaris installation conda environment and
+the final polaris environment.
+
+`mache` gets installed twice because the deployment tools need `mache` to
+even know how to install polaris and build the spack environment on supported
+machines.  The "prebootstrap" step in deployment is creating the installation
+conda environment.  The "bootstrap" step is creating the conda environment that
+polaris will actually use and (in this case with `--update_spack`) building
+spack packages, then creating the "load" or "activation" script that you will
+need to build MPAS components and run polaris.
+
+For testing, you want to point to a different location for installing spack
+using `--spack`.
+
+On many machines, the `/tmp` directory is not a safe place to build spack
+packages.  Use `--tmpdir` to point to another place, e.g., your scratch
+space.
+
+The `--recreate` flag may not be strictly necessary but it’s a good idea.
+This will make sure both the bootstrapping conda environment (the one that
+installs mache to identify the machine) and the polaris conda environment are
+created fresh.
+
+The `--compiler` flag is a list of one or more compilers to build for and the
+`--mpi` flag is the corresponding list of MPI libraries.  To see what is
+supported on each machine, take a look at {ref}`dev-supported-machines`.
+
+Be aware that not all compilers and MPI libraries support Albany and PETSc, as
+discussed below.
+
+### Testing spack with PETSc (and Netlib LAPACK)
+
+If you want to build PETSc (and Netlib LAPACK), use the `--with_petsc` flag.
+Currently, this only works with some
+compilers, but that may be more that I was trying to limit the amount of work
+for the polaris support team.  There is a file,
+[petsc_supported.txt](https://github.com/E3SM-Project/polaris/blob/main/deploy/petsc_supported.txt),
+that lists supported compilers and MPI libraries on each machine.
+
+Here is an example:
+
+```bash
+export TMPDIR=/lcrc/group/e3sm/${USER}/spack_temp
+./configure_polaris_envs.py \
+    --conda ${CONDA_BASE} \
+    --mache_fork <fork> \
+    --mache_branch <branch> \
+    --update_spack \
+    --spack /lcrc/group/e3sm/${USER}/spack_test \
+    --tmpdir ${TMPDIR} \
+    --compiler intel gnu \
+    --mpi openmpi \
+    --with_petsc \
+    --recreate \
+    --verbose
+```
+
+### Testing spack with Albany
+
+If you also want to build Albany, use the `--with_albany` flag.  Currently,
+this only works with Gnu compilers.  There is a file,
+[albany_support.txt](https://github.com/E3SM-Project/polaris/blob/main/deploy/albany_supported.txt),
+that lists supported compilers and MPI libraries on each machine.
+
+Here is an example:
+
+```bash
+export TMPDIR=/lcrc/group/e3sm/${USER}/spack_temp
+./configure_polaris_envs.py \
+    --conda ${CONDA_BASE} \
+    --mache_fork <fork> \
+    --mache_branch <branch> \
+    --update_spack \
+    --spack /lcrc/group/e3sm/${USER}/spack_test \
+    --tmpdir ${TMPDIR} \
+    --compiler gnu \
+    --mpi openmpi \
+    --with_albany \
+    --recreate \
+    --verbose
+```
+
+### Troubleshooting spack
+
+If you encounter an error like:
+```
+==>   spack env activate dev_polaris_0_2_0_gnu_mpich
+==> Error: Package 'armpl' not found.
+You may need to run 'spack clean -m'.
+```
+during the attempt to build spack, you will first need to find the path to
+`setup-env.sh` (see `polaris/build_*/build*.sh`) and source that script to
+get the `spack` command, e.g.:
+
+```bash
+source ${PSCRATCH}/spack_test/dev_polaris_0_2_0_gnu_mpich/share/spack/setup-env.sh
+```
+
+Then run the suggested command:
+
+```bash
+spack clean -m
+```
+
+After that, re-running `./configure_polaris_envs.py` should work correctly.
+
+This issue seems to be related to switching between spack v0.18 and v0.19 (used by different versions of polaris).
+
+## Testing polaris
+
+### Testing MPAS-Ocean without PETSc
+
+Please use the E3SM-Project submodule in polaris for testing, rather than
+E3SM’s master branch.  The submodule is the version we know works with polaris
+and serves as kind of a baseline for other testing.
+
+```bash
+# source whichever load script is appropriate
+source load_dev_polaris_0.2.0_chrysalis_intel_openmpi.sh
+git submodule update --init --recursive
+cd E3SM-Project/components/mpas-ocean
+# this will build with PIO and OpenMP
+make ifort
+polaris suite -s -c ocean -t pr -p . \
+    -w /lcrc/group/e3sm/ac.xylar/polaris/test_20230202/ocean_pr_chrys_intel_openmpi
+cd /lcrc/group/e3sm/ac.xylar/polaris/test_20230202/ocean_pr_chrys_intel_openmpi
+sbatch job_script.pr.bash
+```
+
+You can make other worktrees of E3SM-Project for testing other compilers if
+that’s helpful.  It also might be good to open a fresh terminal to source a
+new load script.  This isn’t required but you’ll get some warnings.
+
+```bash
+source load_dev_polaris_0.2.0_chrysalis_gnu_openmpi.sh
+cd E3SM-Project
+git worktree add ../e3sm_chrys_gnu_openmpi
+cd ../e3sm_chrys_gnu_openmpi
+git submodule update --init --recursive
+cd components/mpas-ocean
+make gfortran
+polaris suite -s -c ocean -t pr -p . \
+    -w /lcrc/group/e3sm/ac.xylar/polaris/test_20230202/ocean_pr_chrys_gnu_openmpi
+cd /lcrc/group/e3sm/ac.xylar/polaris/test_20230202/ocean_pr_chrys_gnu_openmpi
+sbatch job_script.pr.bash
+```
+
+You can also explore the utility in
+[utils/matrix](https://github.com/E3SM-Project/polaris/tree/main/utils/matrix) to
+test on several compilers automatically.
+
+### Testing MALI with Albany
+
+Please use the MALI-Dev submodule in polaris for testing, rather than MALI-Dev
+develop branch.  The submodule is the version we know works with polaris and
+serves as kind of a baseline for other testing.
+
+```bash
+# source whichever load script is appropriate
+source load_dev_polaris_0.2.0_chrysalis_gnu_openmpi_albany.sh
+git submodule update --init --recursive
+cd MALI-Dev/components/mpas-albany-landice
+# you need to tell it to build with Albany
+make ALBANY=true gfortran
+polaris suite -s -c landice -t full_integration -p . \
+    -w /lcrc/group/e3sm/ac.xylar/polaris/test_20230202/landice_full_chrys_gnu_openmpi
+cd /lcrc/group/e3sm/ac.xylar/polaris/test_20230202/landice_full_chrys_gnu_openmpi
+sbatch job_script.full_integration.bash
+```
+
+### Testing MPAS-Ocean with PETSc
+
+The tests for PETSc use nonhydrostatic capabilities not yet integrated into
+E3SM.  So you can’t use the E3SM-Project submodule.  You need to use Sara
+Calandrini’s [nonhydro](https://github.com/scalandr/E3SM/tree/ocean/nonhydro)
+branch.
+
+```bash
+# source whichever load script is appropriate
+source load_dev_polaris_0.2.0_chrysalis_intel_openmpi_petsc.sh
+git submodule update --init
+cd E3SM-Project
+git remote add scalandr/E3SM git@github.com:scalandr/E3SM.git
+git worktree add ../nonhydro_chrys_intel_openmpi -b nonhydro_chrys_intel_openmpi \
+    --checkout scalandr/E3SM/ocean/nonhydro
+cd ../nonhydro_chrys_intel_openmpi
+git submodule update --init --recursive
+cd components/mpas-ocean
+# this will build with PIO, Netlib LAPACK and PETSc
+make ifort
+polaris list | grep nonhydro
+# update these numbers for the 2 nonhydro test cases
+polaris setup -n 245 246 -p . \
+    -w /lcrc/group/e3sm/ac.xylar/polaris/test_20230202/nonhydro_chrys_intel_openmpi
+cd /lcrc/group/e3sm/ac.xylar/polaris/test_20230202/nonhydro_chrys_intel_openmpi
+sbatch job_script.custom.bash
+```
+
+As with non-PETSc MPAS-Ocean and MALI, you can have different worktrees with
+Sara’s nonhydro branch for building with different compilers or use
+[utils/matrix](https://github.com/E3SM-Project/polaris/tree/main/utils/matrix) to
+build (and run).
+
+## Deploying shared spack environments
+
+:::{note}
+Be careful about deploying shared spack environments, as changes you make
+can affect other polaris users.
+:::
+
+Once polaris has been tested with the spack builds in a temporary location, it
+is time to deploy the shared spack environments for all developers to use.
+A `mache` developer will make a `mache` release (if needed) before this
+step begins.  So there is no need to build mache from a remote branch anymore.
+
+Polaris knows where to deploy spack on each machine because of the `spack`
+config option specified in the `[deploy]` section of each machine's config
+file, see the [machine configs](https://github.com/E3SM-Project/polaris/tree/main/polaris/machines).
+
+It is best to update the remote polaris branch in case of changes:
+
+```bash
+cd simplify_local_mache
+# get any changes
+git fetch --all -p
+# hard reset if there are changes
+git reset –hard xylar/polaris/simplify_local_mache
+```
+
+### Deploy spack for polaris without Albany or PETSc
+
+```bash
+export TMPDIR=/lcrc/group/e3sm/${USER}/spack_temp
+./configure_polaris_envs.py \
+    --conda ${CONDA_BASE} \
+    --update_spack \
+    --tmpdir ${TMPDIR} \
+    --compiler intel intel gnu \
+    --mpi openmpi impi openmpi \
+    --recreate \
+    --verbose
+```
+
+### Deploying spack with Albany
+
+```bash
+export TMPDIR=/lcrc/group/e3sm/${USER}/spack_temp
+./configure_polaris_envs.py \
+    --conda ${CONDA_BASE} \
+    --update_spack \
+    --tmpdir ${TMPDIR} \
+    --compiler gnu \
+    --mpi openmpi \
+    --with_albany \
+    --recreate \
+    --verbose
+```
+
+### Deploying spack with PETSc (and Netlib LAPACK)
+
+```bash
+export TMPDIR=/lcrc/group/e3sm/${USER}/spack_temp
+./configure_polaris_envs.py \
+    --conda ${CONDA_BASE} \
+    --update_spack \
+    --tmpdir ${TMPDIR} \
+    --compiler intel gnu \
+    --mpi openmpi \
+    --with_petsc \
+    --recreate \
+    --verbose
+```
diff --git a/docs/index.md b/docs/index.md
index 01352c260..5f17a89e0 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -63,6 +63,7 @@ developers_guide/machines/index
 developers_guide/troubleshooting
 developers_guide/docs
 developers_guide/building_docs
+developers_guide/deploying_spack
 developers_guide/api
 
 design_docs/index

From 844896ae489a44d2df33034db1a973b72ba92dd8 Mon Sep 17 00:00:00 2001
From: Xylar Asay-Davis <xylarstorm@gmail.com>
Date: Mon, 31 Jul 2023 18:48:58 +0200
Subject: [PATCH 17/17] Remove 2 configurations we no longer support

---
 deploy/unsupported.txt                  | 2 ++
 docs/developers_guide/machines/index.md | 4 ----
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/deploy/unsupported.txt b/deploy/unsupported.txt
index 4e72cffc0..4490368e8 100644
--- a/deploy/unsupported.txt
+++ b/deploy/unsupported.txt
@@ -5,10 +5,12 @@ anvil, gnu, impi
 chicoma-cpu, nvidia, mpich
 chicoma-cpu, aocc, mpich
 chicoma-cpu, amdclang, mpich
+chrysalis, intel, impi
 chrysalis, gnu, impi
 chrysalis, oneapi-ifx, openmpi
 chrysalis, oneapi-ifx, impi
 compy, intel, mvapich2
+compy, gnu, openmpi
 compy, pgi, impi
 compy, pgi, mvapich2
 pm-cpu, nvidia, mpich
diff --git a/docs/developers_guide/machines/index.md b/docs/developers_guide/machines/index.md
index a2df3b427..04979ca3e 100644
--- a/docs/developers_guide/machines/index.md
+++ b/docs/developers_guide/machines/index.md
@@ -51,14 +51,10 @@ supported for those configurations with `gnu` compilers.
 | chicoma-cpu  | gnu        | mpich     | gnu-cray          |
 +--------------+------------+-----------+-------------------+
 | chrysalis    | intel      | openmpi   | ifort             |
-|              |            +-----------+-------------------+
-|              |            | impi      | intel-mpi         |
 |              +------------+-----------+-------------------+
 |              | gnu        | openmpi   | gfortran          |
 +--------------+------------+-----------+-------------------+
 | compy        | intel      | impi      | intel-mpi         |
-|              +------------+-----------+-------------------+
-|              | gnu        | openmpi   | gfortran          |
 +--------------+------------+-----------+-------------------+
 | pm-cpu       | gnu        | mpich     | gnu-cray          |
 +--------------+------------+-----------+-------------------+