From 41817270b5059d9e204506e795aa693c815f9dba Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 8 Sep 2023 08:19:02 -0600 Subject: [PATCH 01/13] Replace spack submodule branch name jcsda_emc_spack_stack with release/1.5.0 in .gitmodules and container yamls --- .gitmodules | 2 +- configs/containers/docker-ubuntu-clang-mpich.yaml | 4 ++-- configs/containers/docker-ubuntu-gcc-openmpi.yaml | 2 +- configs/containers/docker-ubuntu-intel-impi.yaml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index b1f475b64..31d7a8707 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,7 +3,7 @@ #url = https://github.com/spack/spack #branch = develop url = https://github.com/jcsda/spack - branch = jcsda_emc_spack_stack + branch = release/1.5.0 [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/configs/containers/docker-ubuntu-clang-mpich.yaml b/configs/containers/docker-ubuntu-clang-mpich.yaml index bcf04c337..b18237130 100644 --- a/configs/containers/docker-ubuntu-clang-mpich.yaml +++ b/configs/containers/docker-ubuntu-clang-mpich.yaml @@ -104,8 +104,8 @@ spack: images: os: ubuntu:20.04 spack: - url: https://github.com/jcsda/spack - ref: jcsda_emc_spack_stack + url: https://github.com/jcrelease/1.5.0sda/spack + ref: release/1.5.0 resolve_sha: false # Whether or not to strip binaries diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index 45f16d4b4..8105c0804 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -92,7 +92,7 @@ spack: os: ubuntu:20.04 spack: url: https://github.com/jcsda/spack - ref: jcsda_emc_spack_stack + ref: release/1.5.0 resolve_sha: false # Whether or not to strip binaries diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index fe3f03efd..964d100fc 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -110,7 +110,7 @@ spack: os: ubuntu:20.04 spack: url: https://github.com/jcsda/spack - ref: jcsda_emc_spack_stack + ref: release/1.5.0 resolve_sha: false # Whether or not to strip binaries From f254169856e0b92138742fefe656c646053de020 Mon Sep 17 00:00:00 2001 From: Alex Richert <82525672+AlexanderRichert-NOAA@users.noreply.github.com> Date: Fri, 8 Sep 2023 18:22:28 -0400 Subject: [PATCH 02/13] (release/1.5.0) Update grib-util, w3emc, ip defaults (#767) * point submodule to Alex's fork * Update default ip and grib-util versions * Update w3emc version to satisfy grib-util * revert submodule --- configs/common/packages.yaml | 6 +++--- spack | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index fddf1f87d..5e81ed879 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -103,7 +103,7 @@ # the container builds. #version: ['2.11.0'] grib-util: - version: ['1.2.3'] + version: ['1.3.0'] gsibec: version: ['1.1.3'] gsi-ncdiag: @@ -117,7 +117,7 @@ version: ['1.14.0'] variants: +hl +fortran +mpi ~threadsafe +szip ip: - version: ['3.3.3'] + version: ['4.1.0'] ip2: version: ['1.1.2'] jasper: @@ -298,7 +298,7 @@ upp: version: ['10.0.10'] w3emc: - version: ['2.9.2'] + version: ['2.10.0'] w3nco: version: ['2.4.1'] wget: diff --git a/spack b/spack index e8037bcaf..872c4a113 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit e8037bcaf26fe80a24e72a14bfcdb5495e506c6f +Subproject commit 872c4a1136c32321895199e074e46be5f1571d8c From d2d065e801b23fbb5d2700839779ae2a1b8eaf24 Mon Sep 17 00:00:00 2001 From: Alex Richert <82525672+AlexanderRichert-NOAA@users.noreply.github.com> Date: Mon, 11 Sep 2023 17:58:44 -0400 Subject: [PATCH 03/13] UFS and acorn updates (#772) * Update ufs-weather-model-static template * Add intel 22 for acorn --- configs/sites/acorn/compilers.yaml | 21 ++++++++++++++++++ configs/sites/acorn/packages.yaml | 2 +- .../ufs-weather-model-static/spack.yaml | 22 +++++++++---------- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/configs/sites/acorn/compilers.yaml b/configs/sites/acorn/compilers.yaml index 8daf3571e..447fc44bb 100644 --- a/configs/sites/acorn/compilers.yaml +++ b/configs/sites/acorn/compilers.yaml @@ -20,6 +20,27 @@ compilers: # which confuses some packages. CONFIG_SITE: '' extra_rpaths: [] +- compiler: + spec: intel@2022.0.2.262 + paths: + cc: cc + cxx: CC + f77: ftn + fc: ftn + flags: {} + operating_system: sles15 + modules: + - PrgEnv-intel/8.3.3 + - craype/2.7.13 + - intel-classic/2022.2.0.262 + - libfabric + environment: + set: + # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so + # Automake-based builds are installed in lib64 + # which confuses some packages. + CONFIG_SITE: '' + extra_rpaths: [] - compiler: spec: gcc@11.2.0 paths: diff --git a/configs/sites/acorn/packages.yaml b/configs/sites/acorn/packages.yaml index 7f831c14c..be0a06d19 100644 --- a/configs/sites/acorn/packages.yaml +++ b/configs/sites/acorn/packages.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@19.1.3.304,gcc@11.2.0] + compiler:: [intel@19.1.3.304,intel@2022.0.2.262,gcc@11.2.0] providers: mpi:: [cray-mpich@8.1.9] cray-mpich: diff --git a/configs/templates/ufs-weather-model-static/spack.yaml b/configs/templates/ufs-weather-model-static/spack.yaml index 60d60995f..565c85e12 100644 --- a/configs/templates/ufs-weather-model-static/spack.yaml +++ b/configs/templates/ufs-weather-model-static/spack.yaml @@ -14,26 +14,24 @@ spack: - site - common specs: - - jasper@2.0.25~shared - - libjpeg-turbo~shared - - libpng@1.6.37 libs=static - - zlib@1.2.11~shared - bacio@2.4.1 - crtm@2.4.0~fix - - esmf@8.4.1+debug~shared+external-parallelio - - esmf@8.4.1~debug~shared+external-parallelio - - fms@2022.04 constants=GFS + - esmf@8.4.2~shared+external-parallelio + - fms@2023.01 constants=GFS - g2@3.4.5 - - g2tmpl@1.10.0 + - g2tmpl@1.10.2 - gftl-shared@1.5.0 - hdf5@1.14.0+hl+mpi~shared~szip - ip@3.3.3 - - mapl@2.35.2~pnetcdf+debug~shared - - mapl@2.35.2~pnetcdf~debug~shared + - jasper@2.0.32~shared + - libjpeg-turbo~shared + - libpng@1.6.37 libs=static + - mapl@2.35.2~pnetcdf~shared - netcdf-c@4.9.2~parallel-netcdf+mpi~shared~dap - netcdf-fortran@4.6.0~shared - parallel-netcdf@1.12.2~shared - - parallelio@2.5.9+fortran~pnetcdf~shared + - parallelio@2.5.10+fortran~pnetcdf~shared + - scotch@7.0.3 - sp@2.3.3 - w3emc@2.9.2 - - scotch@7.0.3 + - zlib@1.2.13~shared From 1b2c22c2956c7c0df5813e21b593f5ebb45b092f Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 12 Sep 2023 13:24:34 -0600 Subject: [PATCH 04/13] release/1.5.0: site updates part 1 (Orion, Hercules, Narwhal) (#770) * Update Orion site config * Update Narwhal site config * Update submodule pointer for spack * Turn off variants ~extdata2g and ~pflogger for mapl to avoid compilation errors with intel@2021.7.0 and later and to avoid segmentation faults with ufs-weather-model * Update Hercules site config * First updates of doc/source/MaintainersSection.rst doc/source/PreConfiguredSites.rst * Pin awscli to 1.27.84 and py-pyyaml to 5.4.1 --------- Co-authored-by: Mark A Potts --- .github/workflows/macos-ci-aarch64.yaml | 6 +++ configs/common/packages.yaml | 16 ++++--- configs/sites/hercules/modules.yaml | 2 + configs/sites/hercules/packages.yaml | 37 ---------------- configs/sites/narwhal/mirrors.yaml | 4 +- configs/sites/narwhal/modules.yaml | 2 + configs/sites/narwhal/packages.yaml | 18 -------- configs/sites/orion/modules.yaml | 2 + configs/sites/orion/packages.yaml | 35 +--------------- doc/source/MaintainersSection.rst | 8 +--- doc/source/PreConfiguredSites.rst | 56 +++++++++---------------- spack | 2 +- 12 files changed, 50 insertions(+), 138 deletions(-) diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml index 47659d1f2..27a4c9525 100644 --- a/.github/workflows/macos-ci-aarch64.yaml +++ b/.github/workflows/macos-ci-aarch64.yaml @@ -111,6 +111,12 @@ jobs: echo "Packages in combined spack build caches:" spack buildcache list + # Workaround for limited disk space on macOS arm instance + spack config add "config:build_stage:/Users/ec2-user/spack-stack/spack-cache/build_stage" + spack config add "config:test_stage:/Users/ec2-user/spack-stack/spack-cache/test_stage" + spack config add "config:source_cache:/Users/ec2-user/spack-stack/spack-cache/source_cache" + spack config add "config:misc_cache:/Users/ec2-user/spack-stack/spack-cache/misc_cache" + # Break installation up in pieces and create build caches in between # This allows us to "spin up" builds that altogether take longer than # six hours, and/or fail later in the build process. diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 5e81ed879..52e1e37d3 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -11,13 +11,14 @@ lapack: [openblas] yacc: [bison] # + # This version of awscli goes with py-pyyaml@5.4.1 awscli: - version: ['1.29.41'] - bacio: + version: ['1.27.84'] + bacio: version: ['2.4.1'] bison: version: ['3.8.2'] - # Attention - when updating also check orion site config + # Attention - when updating also check S4 site config boost: version: ['1.78.0'] variants: ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden @@ -137,8 +138,12 @@ version: ['0.2.5'] mapl: # 2.35.2 goes with esmf@8.4.2, 2.40.3 goes with esmf@8.5.0 + # turn off ~pflogger and extdata2g to avoid compilation + # errors with intel@2021.7.0+, see + # https://github.com/JCSDA/spack-stack/issues/769 + # also: ... extdata2g segfault UFS? version: ['2.35.2'] - variants: ~shared + variants: ~shared ~extdata2g ~pflogger # If making changes here, also check the Discover site config and the CI workflows met: version: ['11.1.0'] @@ -260,8 +265,9 @@ # Versions earlier than 0.11.0 don't compile on macOS with llvm-clang/13.0.0 and Python/3.9, # and 0.11.0 leads to downstream errors in py-scipy with the Intel compilers version: ['0.12.2'] + # This version of py-pyyaml goes with awscli@1.27.84 py-pyyaml: - version: ['6.0'] + version: ['5.4.1'] py-scipy: version: ['1.9.3'] # Pin the py-setuptools version to avoid duplicate Python packages diff --git a/configs/sites/hercules/modules.yaml b/configs/sites/hercules/modules.yaml index 8264ecf47..4c530d42d 100644 --- a/configs/sites/hercules/modules.yaml +++ b/configs/sites/hercules/modules.yaml @@ -5,3 +5,5 @@ modules: lmod: exclude: - ecflow + include: + - python diff --git a/configs/sites/hercules/packages.yaml b/configs/sites/hercules/packages.yaml index 26df869b3..6a328485b 100644 --- a/configs/sites/hercules/packages.yaml +++ b/configs/sites/hercules/packages.yaml @@ -20,11 +20,6 @@ packages: prefix: /work/noaa/epic/role-epic/spack-stack/hercules/openmpi-4.1.5/gcc-11.3.1 modules: - openmpi/4.1.5 - python: - buildable: False - externals: - - spec: python@3.9.14 #+bz2+crypt+ctypes+dbm+lzma~nis+pyexpat~pythoncmd+readline+sqlite3+ssl~tix~tkinter+uuid+zlib - prefix: /usr ### Modifications of common packages # Version 2.0.8 doesn't compile on Hercules @@ -44,15 +39,6 @@ packages: externals: - spec: binutils@2.35.2 prefix: /usr - bison: - externals: - - spec: bison@3.7.4 - prefix: /usr - # Do not use! - #cmake: - # externals: - # - spec: cmake@3.20.2 - # prefix: /usr coreutils: externals: - spec: coreutils@8.32 @@ -121,17 +107,6 @@ packages: externals: - spec: openssh@8.7p1 prefix: /usr - # Do not use, can lead to duplicate packages being built - #openssl: - # buildable: False - # externals: - # - spec: openssl@3.0.1 - # prefix: /usr - # Do not use, incomplete package (missing FindBin for example) - #perl: - # externals: - # - spec: perl@5.32.1~cpanm+shared+threads - # prefix: /usr pkgconf: externals: - spec: pkgconf@1.7.3 @@ -146,11 +121,6 @@ packages: externals: - spec: subversion@1.14.1 prefix: /usr - # Do not use, problems on compute nodes with Intel - #tar: - # externals: - # - spec: tar@1.34 - # prefix: /usr texinfo: externals: - spec: texinfo@6.7 @@ -159,10 +129,3 @@ packages: externals: - spec: wget@1.21.1 prefix: /usr - # Do not use external zlib; causes issues with tar due to qt/zlib dependencies - # zlib: - # externals: - # - spec: zlib@1.2.13 - # prefix: /apps/spack-managed/gcc-11.3.1/zlib-1.2.13-ltp4c3zzde3zi3gf7x4b7c7nj5ww4i4g - # modules: - # - zlib/1.2.13 diff --git a/configs/sites/narwhal/mirrors.yaml b/configs/sites/narwhal/mirrors.yaml index 8f845ff68..709f9096c 100644 --- a/configs/sites/narwhal/mirrors.yaml +++ b/configs/sites/narwhal/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: local-source: fetch: - url: file:///p/app/projects/NEPTUNE/spack-stack/spack-stack-v1/source-cache + url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///p/app/projects/NEPTUNE/spack-stack/spack-stack-v1/source-cache + url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache access_pair: - null - null diff --git a/configs/sites/narwhal/modules.yaml b/configs/sites/narwhal/modules.yaml index 9724c405d..f83e5dc2c 100644 --- a/configs/sites/narwhal/modules.yaml +++ b/configs/sites/narwhal/modules.yaml @@ -5,3 +5,5 @@ modules: tcl: exclude: - ecflow + include: + - python diff --git a/configs/sites/narwhal/packages.yaml b/configs/sites/narwhal/packages.yaml index 5cf51e0ad..987b5ad0e 100644 --- a/configs/sites/narwhal/packages.yaml +++ b/configs/sites/narwhal/packages.yaml @@ -27,20 +27,6 @@ packages: # prefix: /opt/intel/oneapi_2021.3.0.3219 # modules: # - intel/2021.3.0 - python: - buildable: False - externals: - - spec: python@3.9.7+bz2+ctypes+dbm+lzma+nis+pyexpat+pythoncmd+readline+sqlite3+ssl+tix+tkinter+uuid+zlib - prefix: /opt/cray/pe/python/3.9.7.1 - modules: - - cray-python/3.9.7.1 - py-pip: - buildable: False - externals: - - spec: py-pip@21.2.3 - prefix: /opt/cray/pe/python/3.9.7.1 - modules: - - cray-python/3.9.7.1 ### All other external packages listed alphabetically autoconf: @@ -55,10 +41,6 @@ packages: externals: - spec: binutils@2.37.20211103 prefix: /usr - bison: - externals: - - spec: bison@3.0.4 - prefix: /usr curl: externals: - spec: curl@7.66.0+gssapi+ldap+nghttp2 diff --git a/configs/sites/orion/modules.yaml b/configs/sites/orion/modules.yaml index 8264ecf47..4c530d42d 100644 --- a/configs/sites/orion/modules.yaml +++ b/configs/sites/orion/modules.yaml @@ -5,3 +5,5 @@ modules: lmod: exclude: - ecflow + include: + - python diff --git a/configs/sites/orion/packages.yaml b/configs/sites/orion/packages.yaml index 707826850..47726af43 100644 --- a/configs/sites/orion/packages.yaml +++ b/configs/sites/orion/packages.yaml @@ -28,25 +28,8 @@ packages: prefix: /apps/gcc-10.2.0/openmpi-4.0.4/openmpi-4.0.4 modules: - openmpi/4.0.4 - python: - buildable: False - externals: - - spec: python@3.9.7 - prefix: /work/noaa/da/jedipara/spack-stack/miniconda-3.9.7 - modules: - - miniconda/3.9.7 - py-pip: - buildable: False - externals: - - spec: py-pip@21.2.4 - prefix: /work/noaa/da/jedipara/spack-stack/miniconda-3.9.7 - modules: - - miniconda/3.9.7 ### Modifications of common packages - # Versions 1.73+ don't compile on orion (bootstrap.sh fails) - boost: - version:: ['1.72.0'] ### All other external packages listed alphabetically autoconf: @@ -65,11 +48,6 @@ packages: externals: - spec: berkeley-db@5.3.21 prefix: /usr - # Do not use, can lead to duplicate versions of nco being installed - #bison: - # externals: - # - spec: bison@3.0.4 - # prefix: /usr bzip2: externals: - spec: bzip2@1.0.6 @@ -78,7 +56,6 @@ packages: externals: - spec: cpio@2.11 prefix: /usr - # Don't use OS curl, doesn't support secure transfers diffutils: externals: - spec: diffutils@3.3 @@ -91,7 +68,7 @@ packages: buildable: False externals: - spec: ecflow@5.8.4+ui+static_boost - prefix: /work/noaa/da/role-da/spack-stack/ecflow-5.8.4 + prefix: /work/noaa/epic/role-epic/spack-stack/orion/ecflow-5.8.4 modules: - ecflow/5.8.4 file: @@ -168,7 +145,7 @@ packages: buildable: False externals: - spec: mysql@8.0.31 - prefix: /work/noaa/da/role-da/spack-stack/mysql-8.0.31 + prefix: /work/noaa/epic/role-epic/spack-stack/orion/mysql-8.0.31 modules: - mysql/8.0.31 ncurses: @@ -179,10 +156,6 @@ packages: externals: - spec: openssh@7.4p1 prefix: /usr - openssl: - externals: - - spec: openssl@1.0.2k-fips - prefix: /usr perl: externals: - spec: perl@5.16.3+cpanm+shared+threads @@ -207,10 +180,6 @@ packages: externals: - spec: sed@4.2.2 prefix: /usr - sqlite: - externals: - - spec: sqlite@3.7.17~fts~functions+rtree - prefix: /usr tar: externals: - spec: tar@1.26 diff --git a/doc/source/MaintainersSection.rst b/doc/source/MaintainersSection.rst index b3970130b..e62f7b43b 100644 --- a/doc/source/MaintainersSection.rst +++ b/doc/source/MaintainersSection.rst @@ -227,22 +227,18 @@ MSU Orion On Orion, it is necessary to change the default ``umask`` from ``0027`` to ``0022`` so that users not in the group of the role account can still see and use the software stack. This can be done by running ``umask 022`` after logging into the role account. -miniconda - Follow the instructions in :numref:`Section %s ` to create a basic ``miniconda`` installation and associated modulefile for working with spack. Don't forget to log off and back on to forget about the conda environment. - ecflow ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library. After installing `miniconda`, and loading the following modules, follow the instructions in :numref:`Section %s `. Note that the default/system ``qt@5`` can be used on Orion. .. code-block:: console module purge - module use /work/noaa/da/jedipara/spack-stack/modulefiles - module load miniconda/3.9.7 + module load python/3.9.2 module load cmake/3.22.1 module load gcc/10.2.0 mysql - ``mysql`` must be installed separately from ``spack`` using a binary tarball provided by the MySQL community. Follow the instructions in :numref:`Section %s ` to install ``mysql`` in ``/work/noaa/da/role-da/spack-stack/mysql-8.0.31``. + ``mysql`` must be installed separately from ``spack`` using a binary tarball provided by the MySQL community. Follow the instructions in :numref:`Section %s `. .. _MaintainersSection_Hercules: diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 8844451ce..35577bb64 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -8,19 +8,19 @@ Directory ``configs/sites`` contains site configurations for several HPC systems Pre-configured sites are split into two categories: Tier 1 with officially supported spack-stack installations (see :numref:`Section %s `), and Tier 2 (sites with configuration files that were tested or contributed by others in the past, but that are not officially supported by the spack-stack team; see :numref:`Section %s `). ============================================================= -Officially supported spack-stack 1.4.1 installations (tier 1) +Officially supported spack-stack 1.5.0 installations (tier 1) ============================================================= -Ready-to-use spack-stack 1.4.1 installations are available on the following, fully supported platforms. This version supports the JEDI Skylab release 5 of June 2023, and the UFS Weather Model of July 2023. It can also be used for testing spack-stack with other UFS applications (e.g. the UFS Short Range Weather Application, and the EMC Global Workflow). Note that some platforms have not received the 1.4.1 installations, for these the previous 1.4.0 installations are listed below. Amazon Web Services AMI are available in the US East 1 or 2 regions for the previous 1.4.0 release (1.4.1 is not provided on AWS AMIs). +Ready-to-use spack-stack 1.5.0 installations are available on the following, fully supported platforms. This version supports the JEDI Skylab release 5 of June 2023, and the UFS Weather Model of July 2023. It can also be used for testing spack-stack with other UFS applications (e.g. the UFS Short Range Weather Application, and the EMC Global Workflow). Amazon Web Services AMI are available in the US East 1 or 2 regions. +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | Organization | System | Compilers | Location | Maintainers | +=====================+==================================+=================+=========================================================================================================+===============================+ | **HPC platforms** | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Hercules^** | GCC^*, Intel | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-dev-20230814/envs/unified-env`` | Cam Book / Dom Heinzeller | +| | Hercules^* | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.0/envs/unified-env`` | Cam Book / Dom Heinzeller | | MSU +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/spack-stack-1.4.1/envs/unified-env`` | Cam Book / Dom Heinzeller | +| | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.0/envs/unified-env`` | Cam Book / Dom Heinzeller | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | NASA | Discover | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/spack-stack-1.4.1/envs/unified-env`` | Dom Heinzeller / ??? | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -40,11 +40,9 @@ Ready-to-use spack-stack 1.4.1 installations are available on the following, ful | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Jet | GCC, Intel | ``/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.4.1/envs/unified-env`` | Cam Book / Dom Heinzeller | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Narwhal | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.4.0/envs/unified-env-intel-2021.4.0-hdf5-1.14.0`` | Dom Heinzeller / Sarah King | +| | Narwhal | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.0/envs/unified-env-intel-2021.4.0`` | Dom Heinzeller / Sarah King | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Narwhal^** | Intel (test) | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-dev-20230628/envs/unified-env-intel-2021.4.0`` | Dom Heinzeller / Sarah King | -| +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Narwhal | GCC | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.4.0/envs/unified-env-gcc-10.3.0`` | Dom Heinzeller / Sarah King | +| | Narwhal | GCC | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.0/envs/unified-env-gcc-10.3.0`` | Dom Heinzeller / Sarah King | | U.S. Navy (HPCMP) +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Nautilus | Intel^* | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.4.0/envs/unified-env-intel-2021.5.0-openmpi-4.1.5`` | Dom Heinzeller / Sarah King | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -67,8 +65,6 @@ Ready-to-use spack-stack 1.4.1 installations are available on the following, ful ^* This system uses a different wgrib2 version 3.1.1 than the default 2.0.8. -^** This system uses a different version of spack-stack (mostly newer), which is usually the case when we onboard a new system in between spack-stack releases. - For questions or problems, please consult the known issues in :numref:`Section %s `, the currently open GitHub `issues `_ and `discussions `_ first. .. _Preconfigured_Sites_Tier1: @@ -88,26 +84,26 @@ The following is required for building new spack environments and for using spac .. code-block:: console module purge - module use /work/noaa/da/role-da/spack-stack/modulefiles - module load miniconda/3.9.7 + module use /work/noaa/epic/role-epic/spack-stack/orion/modulefiles + module load python/3.9.2 module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.4.1`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.5.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /work/noaa/epic/role-epic/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core + module use /work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2022.0.2 module load stack-intel-oneapi-mpi/2021.5.1 module load stack-python/3.9.7 module available -For ``spack-stack-1.4.1`` with GNU, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.5.0`` with GNU, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /work/noaa/epic/role-epic/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core + module use /work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/10.2.0 module load stack-openmpi/4.0.4 module load stack-python/3.9.7 @@ -126,29 +122,26 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-dev-20230814`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.5.0`` with Intel, load the following modules after loading mysql and ecflow: .. code-block:: console - module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-dev-20230814/envs/unified-env/install/modulefiles/Core + module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.9.0 module load stack-intel-oneapi-mpi/2021.9.0 module load stack-python/3.9.14 module available -For ``spack-stack-dev-20230814`` with GNU, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.5.0`` with GNU, load the following modules after loading mysql and ecflow: .. code-block:: console - module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-dev-20230814/envs/unified-env/install/modulefiles/Core + module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/11.3.1 module load stack-openmpi/4.1.5 module load stack-python/3.9.14 module available -.. note:: - The recent update to ``spack-stack-dev-20230814`` was required on Hercules due to a bug in the Intel compilers used in ``spack-stack-1.4.1``. - .. _Preconfigured_Sites_Discover: ------------------------------ @@ -211,20 +204,11 @@ With Intel, the following is required for building new spack environments and fo module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.4.0`` with Intel, load the following modules after loading the above modules. - -.. code-block:: console - - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.4.0/envs/unified-env-intel-2021.4.0-hdf5-1.14.0/install/modulefiles/Core - module load stack-intel/2021.4.0 - module load stack-cray-mpich/8.1.14 - module load stack-python/3.9.7 - -For a more recent version of spack-stack based on develop as of June 26, 2023, load the following modules after loading the basic modules above. +For ``spack-stack-1.5.0`` with Intel, load the following modules after loading the above modules. .. code-block:: console - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-dev-20230628/envs/unified-env-intel-2021.4.0/install/modulefiles/Core + module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.0/envs/unified-env-intel-2021.4.0/install/modulefiles/Core module load stack-intel/2021.4.0 module load stack-cray-mpich/8.1.14 module load stack-python/3.9.7 @@ -249,11 +233,11 @@ With GNU, the following is required for building new spack environments and for module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.4.0`` with GNU, load the following modules after loading the above modules. +For ``spack-stack-1.5.0`` with GNU, load the following modules after loading the above modules. .. code-block:: console - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.4.0/envs/unified-env-gcc-10.3.0/install/modulefiles/Core + module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.0/envs/unified-env-gcc-10.3.0/install/modulefiles/Core module load stack-gcc/10.3.0 module load stack-cray-mpich/8.1.14 module load stack-python/3.9.7 diff --git a/spack b/spack index 872c4a113..cce83b11d 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 872c4a1136c32321895199e074e46be5f1571d8c +Subproject commit cce83b11d25f230d44ff27367ffefd22bf7c9e39 From 642e3a2464e22de28871c885f888c09b5145f9da Mon Sep 17 00:00:00 2001 From: Alex Richert <82525672+AlexanderRichert-NOAA@users.noreply.github.com> Date: Tue, 12 Sep 2023 22:32:20 -0400 Subject: [PATCH 05/13] Restrict eckit version for intel 19 for acorn; remove acorn ext python (#773) Co-authored-by: Dom Heinzeller --- configs/sites/acorn/packages.yaml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/configs/sites/acorn/packages.yaml b/configs/sites/acorn/packages.yaml index be0a06d19..515799866 100644 --- a/configs/sites/acorn/packages.yaml +++ b/configs/sites/acorn/packages.yaml @@ -22,13 +22,6 @@ externals: - spec: git-lfs@2.11.0 modules: [git-lfs/2.11.0] - python: - buildable: false - externals: - - spec: python@3.8.6%intel - prefix: /apps/spack/python/3.8.6/intel/19.1.3.304/pjn2nzkjvqgmjw4hmyz43v5x4jbxjzpk - - spec: python@3.8.6%gcc - prefix: /apps/spack/python/3.8.6/gcc/11.2.0/gr4zqejk5oxwsrmxvkenmye4nni4zt6e perl: buildable: false externals: @@ -61,3 +54,7 @@ externals: - spec: flex@2.6.4+lex prefix: /usr + eckit: + require: + - any_of: ["@1.23.1"] + when: "%intel@19.1.3.304" From 907ed2d4e5835b66e4f325ff43bbdeaf9817e5e6 Mon Sep 17 00:00:00 2001 From: Alex Richert <82525672+AlexanderRichert-NOAA@users.noreply.github.com> Date: Wed, 13 Sep 2023 22:56:11 -0400 Subject: [PATCH 06/13] Update to ip 4.3.0 (#776) * Update to ip 4.3.0 * Update submodule pointer for spack --------- Co-authored-by: Dom Heinzeller --- configs/common/packages.yaml | 2 +- spack | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 52e1e37d3..6884fe0e9 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -118,7 +118,7 @@ version: ['1.14.0'] variants: +hl +fortran +mpi ~threadsafe +szip ip: - version: ['4.1.0'] + version: ['4.3.0'] ip2: version: ['1.1.2'] jasper: diff --git a/spack b/spack index cce83b11d..610a6c4fe 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit cce83b11d25f230d44ff27367ffefd22bf7c9e39 +Subproject commit 610a6c4fee272a9d06e83b3cd962542fe8280219 From 5d76fdda0a2b64775bc3bedfecd32a25e02c4470 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 14 Sep 2023 12:57:43 -0600 Subject: [PATCH 07/13] release/1.5.0: site config and doc updates round 2 (S4, aws-pcluster, Discover, Nautilus, JCSDA CI containers) (#775) * Update S4 site config * Update site config for aws-pcluster * Update versions in configs/containers/README.md, cleanup unused parts * Update discover site config * Update nautilus site config * Update to ip 4.3.0 * Update documentation * [skip ci] Add python module to module includes on acorn --------- Co-authored-by: Alex Richert Co-authored-by: Alex Richert <82525672+AlexanderRichert-NOAA@users.noreply.github.com> --- configs/common/packages.yaml | 1 - configs/containers/README.md | 30 ++----- .../containers/docker-ubuntu-clang-mpich.yaml | 2 - .../containers/docker-ubuntu-gcc-openmpi.yaml | 2 - .../containers/docker-ubuntu-intel-impi.yaml | 2 - configs/sites/acorn/modules.yaml | 3 + configs/sites/aws-pcluster/modules.yaml | 1 - configs/sites/aws-pcluster/packages.yaml | 17 ---- configs/sites/discover/modules.yaml | 2 + configs/sites/discover/packages.yaml | 38 --------- configs/sites/nautilus/compilers.yaml | 3 - configs/sites/nautilus/packages.yaml | 12 --- configs/sites/s4/modules.yaml | 1 + configs/sites/s4/packages.yaml | 64 +++------------ doc/source/PreConfiguredSites.rst | 81 +++++++++++-------- 15 files changed, 72 insertions(+), 187 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 6884fe0e9..cd28b2a1a 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -18,7 +18,6 @@ version: ['2.4.1'] bison: version: ['3.8.2'] - # Attention - when updating also check S4 site config boost: version: ['1.78.0'] variants: ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden diff --git a/configs/containers/README.md b/configs/containers/README.md index bf6f79b8f..9bfa6c57c 100644 --- a/configs/containers/README.md +++ b/configs/containers/README.md @@ -7,15 +7,16 @@ To avoid hardcoding specs in the generic container recipes, we keep the specs li specs: [base-env@1.0.0, jedi-base-env@1.0.0 ~fftw, ewok-env@1.0.0, jedi-fv3-env@1.0.0, jedi-mpas-env@1.0.0, bacio@2.4.1, bison@3.8.2, bufr@12.0.0, ecbuild@3.7.2, eccodes@2.27.0, ecflow@5, eckit@1.24.4, ecmwf-atlas@0.34.0 +trans ~fftw, fiat@1.2.0, ectrans@1.2.0 ~fftw, eigen@3.4.0, - fckit@0.11.0, fms@release-jcsda, g2@3.4.5, g2tmpl@1.10.0, gftl-shared@1.5.0, - gsibec@1.1.3, hdf@4.2.15, hdf5@1.14.1-2, ip@3.3.3, jasper@2.0.32, jedi-cmake@1.4.0, + fckit@0.11.0, fms@release-jcsda, g2@3.4.5, g2tmpl@1.10.2, gftl-shared@1.5.0, + gsibec@1.1.3, hdf@4.2.15, hdf5@1.14.0, ip@4.3.0, jasper@2.0.32, jedi-cmake@1.4.0, libpng@1.6.37, nccmp@1.9.0.1, netcdf-c@4.9.2, netcdf-cxx4@4.3.1, netcdf-fortran@4.6.0, nlohmann-json@3.10.5, nlohmann-json-schema-validator@2.1.0, parallelio@2.5.10, parallel-netcdf@1.12.2, py-eccodes@1.4.2, py-f90nml@1.4.3, py-gitpython@3.1.27, py-h5py@3.7.0, py-numpy@1.22.3, - py-pandas@1.4.0, py-pip, py-pyyaml@6.0, py-scipy@1.9.3, py-shapely@1.8.0, py-xarray@2022.3.0, - sp@2.3.3, udunits@2.2.28, w3nco@2.4.1, nco@5.0.6, esmf@8.4.2, mapl@2.35.2, - yafyaml@0.5.1, zlib@1.2.13, odc@1.4.6, shumlib@macos_clang_linux_intel_port] + py-pandas@1.5.3, py-pip, py-pyyaml@5.4.1, py-scipy@1.9.3, py-shapely@1.8.0, py-xarray@2022.3.0, + sp@2.3.3, udunits@2.2.28, w3nco@2.4.1, w3emc@2.10.0, nco@5.0.6, esmf@8.4.2, mapl@2.35.2, + yafyaml@0.5.1, zlib@1.2.13, zstd@1.5.2, odc@1.4.6, shumlib@macos_clang_linux_intel_port, + awscli@1.27.84, py-globus-cli@3.16.0] # Don't build CRTM by default so that it gets built in the JEDI bundles: # crtm@v2.4.1-jedi # Comment out for now until build problems are solved @@ -23,25 +24,6 @@ To avoid hardcoding specs in the generic container recipes, we keep the specs li # py-mysql-connector-python@8.0.32 ``` -### spack-stack-1.5.0 / ufs-weather-model-x.y.z containers for ufs-weather-model as of July 5, 2023 - -**Note. This is not yet working correctly, some libraries are missing. Please do not use yet! Also, if using the clang-mpich container, need to disable openmp for fms, not clear how to do this cleanly.** - -``` - specs: [base-env@1.0.0, - bacio@2.4.1, bison@3.8.2, bufr@12.0.0, ecbuild@3.7.2, eccodes@2.27.0, ecflow@5, - eckit@1.24.4, ecmwf-atlas@0.34.0 +trans ~fftw, fiat@1.2.0, ectrans@1.2.0 ~fftw, eigen@3.4.0, - fckit@0.11.0, fms@2023.01, g2@3.4.5, g2tmpl@1.10.0, gftl-shared@1.5.0, - gsibec@1.1.3, hdf@4.2.15, hdf5@1.14.1-2, ip@3.3.3, jasper@2.0.32, jedi-cmake@1.4.0, - libpng@1.6.37, nccmp@1.9.0.1, netcdf-c@4.9.2, netcdf-cxx4@4.3.1, - netcdf-fortran@4.6.0, nlohmann-json@3.10.5, nlohmann-json-schema-validator@2.1.0, - parallelio@2.5.10, parallel-netcdf@1.12.2, py-eccodes@1.4.2, py-f90nml@1.4.3, - py-gitpython@3.1.27, py-h5py@3.7.0, py-numpy@1.22.3, py-pandas@1.4.0, - py-pip, py-pyyaml@6.0, py-scipy@1.9.3, py-shapely@1.8.0, py-xarray@2022.3.0, - sp@2.3.3, udunits@2.2.28, w3emc@2.9.2, w3nco@2.4.1, nco@5.0.6, esmf@8.4.2, mapl@2.35.2, - yafyaml@0.5.1, zlib@1.2.13, odc@1.4.6, crtm@2.4.0, crtm-fix@2.4.0_emc] -``` - ### Create an AMI on AWS EC2 to build docker containers AMI ami-0a934b7133c2b7102 (dom-docker-builder-full-backup-20230217; JCSDA-NOAA account, us-east-1) was created following the instructions below diff --git a/configs/containers/docker-ubuntu-clang-mpich.yaml b/configs/containers/docker-ubuntu-clang-mpich.yaml index b18237130..8ecf5554a 100644 --- a/configs/containers/docker-ubuntu-clang-mpich.yaml +++ b/configs/containers/docker-ubuntu-clang-mpich.yaml @@ -235,8 +235,6 @@ spack: echo "export CXX=clang++" >> /root/.bashrc && \ echo "export FC=gfortran" >> /root/.bashrc && \ printf "[credential]\n helper = cache --timeout=7200\n" >> /root/.gitconfig - # Source spack environment script for root - RUN echo "source /etc/profile.d/z10_spack_environment.sh\n" >> /root/.bashrc # Labels for the image labels: diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index 8105c0804..09c814c69 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -184,8 +184,6 @@ spack: RUN echo "export OMPI_ALLOW_RUN_AS_ROOT=1" >> /root/.bashrc && \ echo "export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1" >> /root/.bashrc && \ echo "export OMPI_MCA_rmaps_base_oversubscribe=1" >> /root/.bashrc - # Source spack environment script for root - RUN echo "source /etc/profile.d/z10_spack_environment.sh\n" >> /root/.bashrc # Labels for the image labels: diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 964d100fc..4ad637b04 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -211,8 +211,6 @@ spack: echo "source /opt/intel/oneapi/compiler/latest/env/vars.sh" >> /root/.bashrc && \ echo "source /opt/intel/oneapi/mpi/latest/env/vars.sh" >> /root/.bashrc && \ printf "[credential]\n helper = cache --timeout=7200\n" >> /root/.gitconfig - # Source spack environment script for root - RUN echo "source /etc/profile.d/z10_spack_environment.sh\n" >> /root/.bashrc # Labels for the image labels: diff --git a/configs/sites/acorn/modules.yaml b/configs/sites/acorn/modules.yaml index 4e9771f38..65801706f 100644 --- a/configs/sites/acorn/modules.yaml +++ b/configs/sites/acorn/modules.yaml @@ -2,3 +2,6 @@ modules: default: enable:: - lmod + lmod: + include: + - python diff --git a/configs/sites/aws-pcluster/modules.yaml b/configs/sites/aws-pcluster/modules.yaml index 1f6fc0f74..d7e195ab6 100644 --- a/configs/sites/aws-pcluster/modules.yaml +++ b/configs/sites/aws-pcluster/modules.yaml @@ -4,7 +4,6 @@ modules: - lmod lmod: include: - # List of packages for which we need modules that are blacklisted by default - python exclude: - ecflow \ No newline at end of file diff --git a/configs/sites/aws-pcluster/packages.yaml b/configs/sites/aws-pcluster/packages.yaml index ee1dc5e32..563589c7c 100644 --- a/configs/sites/aws-pcluster/packages.yaml +++ b/configs/sites/aws-pcluster/packages.yaml @@ -43,21 +43,11 @@ packages: externals: - spec: binutils@2.34 prefix: /usr - bison: - externals: - - spec: bison@3.5.1 - prefix: /usr boost: buildable: False externals: - spec: boost@1.71.0 +atomic +chrono +date_time +exception +filesystem +graph +iostreams +locale +log +math +mpi +numpy +pic +program_options +python +random +regex +serialization +signals +system +test +thread +timer +wave cxxstd=17 visibility=hidden prefix: /usr - # Don't use external cmake, this confuses the spack concretizer - # leading to duplicate packages being installed. - #cmake: - # externals: - # - spec: cmake@3.16.3 - # prefix: /usr coreutils: externals: - spec: coreutils@8.30 @@ -120,13 +110,6 @@ packages: externals: - spec: openssh@8.2p1 prefix: /usr - # Don't use external openssl, this confuses the spack concretizer - # leading to duplicate packages (libarchive with different compression - # settings, somehow triggered by different cmake versions) being installed. - #openssl: - # externals: - # - spec: openssl@1.1.1f - # prefix: /usr perl: externals: - spec: perl@5.30.0~cpanm+shared+threads diff --git a/configs/sites/discover/modules.yaml b/configs/sites/discover/modules.yaml index 8264ecf47..4c530d42d 100644 --- a/configs/sites/discover/modules.yaml +++ b/configs/sites/discover/modules.yaml @@ -5,3 +5,5 @@ modules: lmod: exclude: - ecflow + include: + - python diff --git a/configs/sites/discover/packages.yaml b/configs/sites/discover/packages.yaml index fd1c3c1f0..7579f2ad9 100644 --- a/configs/sites/discover/packages.yaml +++ b/configs/sites/discover/packages.yaml @@ -3,7 +3,6 @@ packages: compiler:: [intel@2022.0.1, gcc@10.1.0] providers: mpi:: [intel-oneapi-mpi@2021.5.0, openmpi@4.1.3] - target: [ivybridge] ### MPI, Python, MKL mpi: @@ -20,20 +19,6 @@ packages: prefix: /discover/swdev/jcsda/spack-stack/openmpi-4.1.3/gcc-10.1.0 modules: - openmpi/4.1.3-gcc-10.1.0 - python: - buildable: False - externals: - - spec: python@3.9.7 - prefix: /discover/swdev/jcsda/spack-stack/miniconda-3.9.7/ - modules: - - miniconda/3.9.7 - py-pip: - buildable: False - externals: - - spec: py-pip@21.2.4 - prefix: /discover/swdev/jcsda/spack-stack/miniconda-3.9.7/ - modules: - - miniconda/3.9.7 ### Modification of common packages @@ -57,25 +42,10 @@ packages: externals: - spec: berkeley-db@4.8.30 prefix: /usr - # Don't use, leads to duplicate packages being built - #bison: - # externals: - # - spec: bison@2.7 - # prefix: /usr ccache: externals: - spec: ccache@3.1.9 prefix: /usr - # Don't use external cmake, this confuses the spack concretizer - # leading to duplicate packages being installed. - #cmake: - # externals: - # - spec: cmake@3.5.2 - # prefix: /usr - # - spec: cmake@3.21.0 - # prefix: /usr/local/other/cmake/3.21.0 - # modules: - # - cmake/3.21.0 cpio: externals: - spec: cpio@2.11 @@ -185,10 +155,6 @@ packages: # Old re2c on Discover unable to build newer versions of ninja ninja: version:: ['1.10.2'] - openssl: - externals: - - spec: openssl@1.0.2j-fips - prefix: /usr perl: externals: - spec: perl@5.18.2~cpanm+shared+threads @@ -213,10 +179,6 @@ packages: externals: - spec: sed@4.2.2 prefix: /usr - sqlite: - externals: - - spec: sqlite@3.8.10.2~fts~functions+rtree - prefix: /usr texinfo: externals: - spec: texinfo@4.13 diff --git a/configs/sites/nautilus/compilers.yaml b/configs/sites/nautilus/compilers.yaml index 3e16961a9..493527c8d 100644 --- a/configs/sites/nautilus/compilers.yaml +++ b/configs/sites/nautilus/compilers.yaml @@ -34,7 +34,4 @@ compilers: PATH: '/opt/rh/gcc-toolset-11/root/usr/bin' CPATH: '/opt/rh/gcc-toolset-11/root/usr/include' LD_LIBRARY_PATH: '/opt/scyld/slurm/lib64:/opt/scyld/slurm/lib64/slurm:/p/app/compilers/intel/oneapi/compiler/2022.0.2/linux/compiler/lib/intel64_lin:/opt/rh/gcc-toolset-11/root/usr/lib64:/opt/rh/gcc-toolset-11/root/usr/lib' - # For intel-oneapi-mpi only - #set: - # I_MPI_ROOT: '/p/app/compilers/intel/oneapi/mpi/2021.5.1' extra_rpaths: [] diff --git a/configs/sites/nautilus/packages.yaml b/configs/sites/nautilus/packages.yaml index 2a403c439..13d40acd9 100644 --- a/configs/sites/nautilus/packages.yaml +++ b/configs/sites/nautilus/packages.yaml @@ -60,18 +60,10 @@ packages: externals: - spec: binutils@2.30.117 prefix: /usr - bison: - externals: - - spec: bison@3.0.4 - prefix: /usr coreutils: externals: - spec: coreutils@8.30 prefix: /usr - #curl: - # externals: - # - spec: curl@7.61.1+gssapi+ldap+nghttp2 - # prefix: /usr diffutils: externals: - spec: diffutils@3.6 @@ -129,10 +121,6 @@ packages: externals: - spec: openssh@8.0p1 prefix: /usr - openssl: - externals: - - spec: openssl@1.1.1k - prefix: /usr perl: externals: - spec: perl@5.26.3~cpanm+open+shared+threads diff --git a/configs/sites/s4/modules.yaml b/configs/sites/s4/modules.yaml index 039d7d19c..5b7d734eb 100644 --- a/configs/sites/s4/modules.yaml +++ b/configs/sites/s4/modules.yaml @@ -7,3 +7,4 @@ modules: - ecflow include: - mpich + - python diff --git a/configs/sites/s4/packages.yaml b/configs/sites/s4/packages.yaml index 4a7263b78..06692e0ce 100644 --- a/configs/sites/s4/packages.yaml +++ b/configs/sites/s4/packages.yaml @@ -1,8 +1,12 @@ packages: all: - compiler:: [intel@2021.5.0, gcc@9.3.0] + # Removing gcc/mpich for now, not working as expected + #compiler:: [intel@2021.5.0, gcc@9.3.0] + #providers: + # mpi:: [intel-oneapi-mpi@2021.5.0, mpich@4.0.1] + compiler:: [intel@2021.5.0] providers: - mpi:: [intel-oneapi-mpi@2021.5.0, mpich@4.0.1] + mpi:: [intel-oneapi-mpi@2021.5.0] ### MPI, Python, MKL mpi: @@ -11,31 +15,13 @@ packages: externals: - spec: intel-oneapi-mpi@2021.5.0%intel@2021.5.0 prefix: /opt/intel/oneapi/2022.1 - modules: - - intel/2022.1 - mpich: - externals: - - spec: mpich@4.0.1%gcc@9.3.0 ~hydra device=ch4 - prefix: /data/prod/hpc-stack/gnu-9.3.0/mpich/4.0.1 - modules: - - mpich/4.0.1 - python: - buildable: False - externals: - - spec: python@3.9.12 - prefix: /data/prod/jedi/spack-stack/miniconda-3.9.12 - modules: - - miniconda/3.9.12 - py-pip: - buildable: False - externals: - - spec: py-pip@21.2.4 - prefix: /data/prod/jedi/spack-stack/miniconda-3.9.12 - modules: - - miniconda/3.9.12 - # Versions 1.73+ don't compile on s4 (bootstrap.sh fails) - boost: - version:: ['1.72.0'] + # Commenting this out for now, not working as expected + #mpich: + # externals: + # - spec: mpich@4.0.1%gcc@9.3.0 ~hydra device=ch4 + # prefix: /data/prod/hpc-stack/gnu-9.3.0/mpich/4.0.1 + # modules: + # - mpich/4.0.1 ### All other external packages listed alphabetically autoconf: @@ -54,30 +40,14 @@ packages: externals: - spec: berkeley-db@5.3.21 prefix: /usr - # Do not use, can lead to duplicate versions of nco being installed - #bison: - # externals: - # - spec: bison@3.0.4 - # prefix: /usr bzip2: externals: - spec: bzip2@1.0.6 prefix: /usr - # Don't use external cmake, this confuses the spack concretizer - # leading to duplicate packages being installed - #cmake: - # externals: - # - spec: cmake@3.20.5 - # prefix: /usr cpio: externals: - spec: cpio@2.11 prefix: /usr - # Don't use OS curl on S4, incomplete - #curl: - # externals: - # - spec: curl@7.29.0+ldap - # prefix: /usr diffutils: externals: - spec: diffutils@3.3 @@ -178,10 +148,6 @@ packages: externals: - spec: openssh@7.4p1 prefix: /usr - openssl: - externals: - - spec: openssl@1.0.2k-fips - prefix: /usr perl: externals: - spec: perl@5.16.3~cpanm+shared+threads @@ -206,10 +172,6 @@ packages: externals: - spec: sed@4.2.2 prefix: /usr - sqlite: - externals: - - spec: sqlite@3.7.17~fts~functions+rtree - prefix: /usr tar: externals: - spec: tar@1.26 diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 35577bb64..2b114451d 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -22,7 +22,7 @@ Ready-to-use spack-stack 1.5.0 installations are available on the following, ful | MSU +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.0/envs/unified-env`` | Cam Book / Dom Heinzeller | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| NASA | Discover | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/spack-stack-1.4.1/envs/unified-env`` | Dom Heinzeller / ??? | +| NASA | Discover | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/spack-stack-1.5.0/envs/unified-env`` | Dom Heinzeller / ??? | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Casper | Intel | ``/glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.4.1/envs/unified-env`` | Dom Heinzeller / ??? | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -30,7 +30,7 @@ Ready-to-use spack-stack 1.5.0 installations are available on the following, ful | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Derecho^** | Intel | ``/lustre/desc1/scratch/epicufsrt/contrib/spack-stack/spack-stack-dev-20230814/envs/unified-en2`` | Mark Potts / Dom Heinzeller | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| NOAA (NCEP) | Acorn | Intel | ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.4.1/envs/unified-env`` | Hang Lei / Alex Richert | +| NOAA (NCEP) | Acorn | Intel | ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.5.0/envs/unified-env`` | Hang Lei / Alex Richert | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Gaea C4 | Intel | ``/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/spack-stack-1.4.1-c4/envs/unified-env`` | Dom Heinzeller / ??? | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -44,11 +44,11 @@ Ready-to-use spack-stack 1.5.0 installations are available on the following, ful | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Narwhal | GCC | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.0/envs/unified-env-gcc-10.3.0`` | Dom Heinzeller / Sarah King | | U.S. Navy (HPCMP) +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Nautilus | Intel^* | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.4.0/envs/unified-env-intel-2021.5.0-openmpi-4.1.5`` | Dom Heinzeller / Sarah King | +| | Nautilus | Intel^* | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.0/envs/unified-env`` | Dom Heinzeller / Sarah King | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Nautilus | AOCC | *currently not supported* | Dom Heinzeller / Sarah King | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | S4 | Intel | ``/data/prod/jedi/spack-stack/spack-stack-1.4.1/envs/unified-env`` | Dom Heinzeller / Mark Potts | +| | S4 | Intel | ``/data/prod/jedi/spack-stack/spack-stack-1.5.0/envs/unified-env`` | Dom Heinzeller / Mark Potts | | Univ. of Wisconsin +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | S4 | GCC | *currently not supported* | Dom Heinzeller / Mark Potts | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -56,9 +56,9 @@ Ready-to-use spack-stack 1.5.0 installations are available on the following, ful +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | AMI Red Hat 8 | GCC | ``/home/ec2-user/spack-stack/spack-stack-1.4.0/envs/unified-env`` | Dom Heinzeller / ??? | + +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| Amazon Web Services | AMI Ubuntu 20 | GCC | ``/mnt/experiments-efs/skylab-v5/spack-stack-1.4.1/envs/unified-env`` | Dom Heinzeller / ??? | +| Amazon Web Services | Parallelcluster JCSDA R&D | | ``/mnt/experiments-efs/skylab-v6/spack-stack-1.5.0/envs/unified-env`` | Dom Heinzeller / ??? | + +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Parallelcluster Ubuntu 20.04 | | ``/mnt/experiments-efs/skylab-v5/spack-stack-1.4.1/envs/unified-env`` | Dom Heinzeller / ??? | +| | Parallelcluster JCSDA ROMEX | | ``/mnt/experiments-efs/ROMEX/spack-stack-1.5.0/envs/unified-env`` | Dom Heinzeller / ??? | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | NOAA (RDHPCS) | RDHPCS Cloud (Parallel Works)^** | Intel | ``/contrib/spack-stack/spack-stack-dev-20230816/envs/unified-env`` | Mark Potts / Cam Book / Dom H | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -96,7 +96,7 @@ For ``spack-stack-1.5.0`` with Intel, load the following modules after loading m module use /work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2022.0.2 module load stack-intel-oneapi-mpi/2021.5.1 - module load stack-python/3.9.7 + module load stack-python/3.10.8 module available For ``spack-stack-1.5.0`` with GNU, load the following modules after loading miniconda and ecflow: @@ -106,7 +106,7 @@ For ``spack-stack-1.5.0`` with GNU, load the following modules after loading min module use /work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/10.2.0 module load stack-openmpi/4.0.4 - module load stack-python/3.9.7 + module load stack-python/3.10.8 module available ------------------------------ @@ -129,7 +129,7 @@ For ``spack-stack-1.5.0`` with Intel, load the following modules after loading m module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.9.0 module load stack-intel-oneapi-mpi/2021.9.0 - module load stack-python/3.9.14 + module load stack-python/3.10.8 module available For ``spack-stack-1.5.0`` with GNU, load the following modules after loading mysql and ecflow: @@ -139,7 +139,7 @@ For ``spack-stack-1.5.0`` with GNU, load the following modules after loading mys module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/11.3.1 module load stack-openmpi/4.1.5 - module load stack-python/3.9.14 + module load stack-python/3.10.8 module available .. _Preconfigured_Sites_Discover: @@ -158,24 +158,24 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.4.1`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.5.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /gpfsm/dswdev/jcsda/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core + module use /gpfsm/dswdev/jcsda/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2022.0.1 module load stack-intel-oneapi-mpi/2021.5.0 - module load stack-python/3.9.7 + module load stack-python/3.10.8 module available -For ``spack-stack-1.4.1`` with GNU, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.5.0`` with GNU, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /gpfsm/dswdev/jcsda/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core + module use /gpfsm/dswdev/jcsda/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/10.1.0 module load stack-openmpi/4.1.3 - module load stack-python/3.9.7 + module load stack-python/3.10.8 module available .. _Preconfigured_Sites_Narwhal: @@ -211,7 +211,7 @@ For ``spack-stack-1.5.0`` with Intel, load the following modules after loading t module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.0/envs/unified-env-intel-2021.4.0/install/modulefiles/Core module load stack-intel/2021.4.0 module load stack-cray-mpich/8.1.14 - module load stack-python/3.9.7 + module load stack-python/3.10.8 With GNU, the following is required for building new spack environments and for using spack to build and run software. Don't use ``module purge`` on Narwhal! @@ -240,11 +240,7 @@ For ``spack-stack-1.5.0`` with GNU, load the following modules after loading the module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.0/envs/unified-env-gcc-10.3.0/install/modulefiles/Core module load stack-gcc/10.3.0 module load stack-cray-mpich/8.1.14 - module load stack-python/3.9.7 - -.. note:: - - ``spack-stack-1.4.0`` builds for Intel and GNU on Narwhal are missing `jedi-ufs-env` and `ufs-weather-model-env` due to build errors for `mapl@2.35.2`. + module load stack-python/3.10.8 .. _Preconfigured_Sites_Nautilus: @@ -267,11 +263,11 @@ With Intel, the following is required for building new spack environments and fo module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.4.0`` with Intel, load the following modules after loading the above modules. +For ``spack-stack-1.5.0`` with Intel, load the following modules after loading the above modules. .. code-block:: console - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-dev-20230628/envs/unified-env/install/modulefiles/Core + module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-openmpi/4.1.5rc2 module load stack-python/3.10.8 @@ -294,7 +290,7 @@ With AMD clang/flang (aocc), the following is required for building new spack en .. note:: - ``spack-stack-1.4.0`` is not yet supported with the Arm clang/flang compilers. Use Intel instead. + ``spack-stack-1.5.0`` is not yet supported with the Arm clang/flang compilers. Use Intel instead. .. note:: @@ -614,14 +610,14 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.4.1`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.5.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /data/prod/jedi/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core + module use /data/prod/jedi/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-intel-oneapi-mpi/2021.5.0 - module load stack-python/3.9.12 + module load stack-python/3.10.8 module unuse /opt/apps/modulefiles/Compiler/intel/non-default/22 module unuse /opt/apps/modulefiles/Compiler/intel/22 module available @@ -636,26 +632,26 @@ Note the two `module unuse` commands, that need to be run after the stack metamo Amazon Web Services Parallelcluster Ubuntu 20.04 ------------------------------------------------ -Access to the JCSDA-managed AWS Parallel Cluster is not available to the public. The following instructions are for JCSDA core staff and in-kind contributors. +Access to the JCSDA-managed AWS Parallel Clusters is not available to the public. The following instructions are for JCSDA core staff and in-kind contributors. -For ``spack-stack-1.4.1`` with Intel, run the following commands/load the following modules: +For ``spack-stack-1.5.0`` with Intel on the JCSDA R&D cluster (``hpc6a.48xlarge`` instances), run the following commands/load the following modules: .. code-block:: console module purge ulimit -s unlimited source /opt/intel/oneapi/compiler/2022.1.0/env/vars.sh - module use /mnt/experiments-efs/skylab-v5/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core + module use /mnt/experiments-efs/skylab-v6/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2022.1.0 module load stack-intel-oneapi-mpi/2021.6.0 module load stack-python/3.10.8 module available -For ``spack-stack-1.4.1`` with GNU, run the following commands/load the following modules: +For ``spack-stack-1.5.0`` with GNU on the JCSDA R&D cluster (``hpc6a.48xlarge`` instances), run the following commands/load the following modules: module purge ulimit -s unlimited - module use /mnt/experiments-efs/skylab-v5/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core + module use /mnt/experiments-efs/skylab-v6/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/9.4.0 module load stack-openmpi/4.1.4 module load stack-python/3.10.8 @@ -663,7 +659,24 @@ For ``spack-stack-1.4.1`` with GNU, run the following commands/load the followin .. note:: - Users reported problems with parallel applications using GNU+OpenMPI in earlier versions of spack-stack. This may still be the case for ``spack-stack-1.4.0``. We recommend using the well-tested Intel setup. + Users reported problems with parallel applications using GNU+OpenMPI in earlier versions of spack-stack. This may still be the case for ``spack-stack-1.5.0``. We recommend using the well-tested Intel setup. + +For ``spack-stack-1.5.0`` with Intel on the JCSDA ROMEX cluster (``c6i.32xlarge`` instances), run the following commands/load the following modules: + +.. code-block:: console + + module purge + ulimit -s unlimited + source /opt/intel/oneapi/compiler/2022.1.0/env/vars.sh + module use /mnt/experiments-efs/ROMEX/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core + module load stack-intel/2022.1.0 + module load stack-intel-oneapi-mpi/2021.6.0 + module load stack-python/3.10.8 + module available + +.. note:: + + There is no GNU option for the ROMEX cluster. ----------------------------- Amazon Web Services Red Hat 8 From be4236cc55d9a8607d50ec91a5094a31fe1a71dc Mon Sep 17 00:00:00 2001 From: Cameron Book <43379611+ulmononian@users.noreply.github.com> Date: Wed, 20 Sep 2023 11:58:11 -0700 Subject: [PATCH 08/13] Update Hera/Jet/Cheyenne site configs (#779) * Updates to cheyene, jet, hera site configs. * Update Hera/Jet sections of PreConfiguredSites.rst. --- configs/sites/cheyenne/modules.yaml | 2 ++ configs/sites/cheyenne/packages.yaml | 27 ---------------------- configs/sites/hera/mirrors.yaml | 4 ++-- configs/sites/hera/modules.yaml | 2 ++ configs/sites/hera/packages.yaml | 34 ---------------------------- configs/sites/jet/modules.yaml | 5 ++++ configs/sites/jet/packages.yaml | 29 ------------------------ doc/source/PreConfiguredSites.rst | 28 +++++++++++------------ 8 files changed, 25 insertions(+), 106 deletions(-) diff --git a/configs/sites/cheyenne/modules.yaml b/configs/sites/cheyenne/modules.yaml index 8264ecf47..4c530d42d 100644 --- a/configs/sites/cheyenne/modules.yaml +++ b/configs/sites/cheyenne/modules.yaml @@ -5,3 +5,5 @@ modules: lmod: exclude: - ecflow + include: + - python diff --git a/configs/sites/cheyenne/packages.yaml b/configs/sites/cheyenne/packages.yaml index 6c930de33..79f0dc6d2 100644 --- a/configs/sites/cheyenne/packages.yaml +++ b/configs/sites/cheyenne/packages.yaml @@ -40,20 +40,6 @@ packages: prefix: /glade/u/apps/ch/opt/openmpi/4.1.1/gnu/10.1.0 modules: - openmpi/4.1.1 - python: - buildable: False - externals: - - spec: python@3.9.12+bz2+ctypes+dbm+lzma+nis+pyexpat+pythoncmd+readline+sqlite3+ssl+tix+tkinter+uuid+zlib - prefix: /glade/work/jedipara/cheyenne/spack-stack/miniconda-3.9.12 - modules: - - miniconda/3.9.12 - py-pip: - buildable: False - externals: - - spec: py-pip@21.2.4 - prefix: /glade/work/jedipara/cheyenne/spack-stack/miniconda-3.9.12 - modules: - - miniconda/3.9.12 ### All other external packages listed alphabetically autoconf: @@ -68,11 +54,6 @@ packages: externals: - spec: berkeley-db@4.8.30 prefix: /usr - # Don't use, leads to duplicate packages being built - #bison: - # externals: - # - spec: bison@2.7 - # prefix: /usr bzip2: externals: - spec: bzip2@1.0.6 @@ -197,10 +178,6 @@ packages: externals: - spec: openssh@7.2p2 prefix: /usr - openssl: - externals: - - spec: openssl@1.0.2p-fips - prefix: /usr # Pin patchelf to 0.15.0 for Intel compiler (no C++-17 features) patchelf: version:: ['0.15.0'] @@ -228,10 +205,6 @@ packages: externals: - spec: sed@4.2.2 prefix: /usr - sqlite: - externals: - - spec: sqlite@3.8.10.2~fts~functions+rtree - prefix: /usr texinfo: externals: - spec: texinfo@4.13 diff --git a/configs/sites/hera/mirrors.yaml b/configs/sites/hera/mirrors.yaml index be3b41bba..f6c5a8d1e 100644 --- a/configs/sites/hera/mirrors.yaml +++ b/configs/sites/hera/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: local-source: fetch: - url: file:///scratch1/NCEPDEV/global/spack-stack/source-cache + url: file:///scratch1/NCEPDEV/nems/role.epic/spack-stack/source-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///scratch1/NCEPDEV/global/spack-stack/source-cache + url: file:///scratch1/NCEPDEV/nems/role.epic/spack-stack/source-cache access_pair: - null - null diff --git a/configs/sites/hera/modules.yaml b/configs/sites/hera/modules.yaml index 8264ecf47..4c530d42d 100644 --- a/configs/sites/hera/modules.yaml +++ b/configs/sites/hera/modules.yaml @@ -5,3 +5,5 @@ modules: lmod: exclude: - ecflow + include: + - python diff --git a/configs/sites/hera/packages.yaml b/configs/sites/hera/packages.yaml index 977dce1a7..5b1a7d79a 100644 --- a/configs/sites/hera/packages.yaml +++ b/configs/sites/hera/packages.yaml @@ -25,20 +25,6 @@ packages: modules: - gnu/9.2.0 - openmpi/4.1.5 - python: - buildable: False - externals: - - spec: python@3.9.12+bz2+ctypes+dbm+lzma+nis+pyexpat+pythoncmd+readline+sqlite3+ssl+tix+tkinter+uuid+zlib - prefix: /scratch1/NCEPDEV/global/spack-stack/apps/miniconda/py39_4.12.0 - modules: - - miniconda/3.9.12 - py-pip: - buildable: False - externals: - - spec: py-pip@21.2.4 - prefix: /scratch1/NCEPDEV/global/spack-stack/apps/miniconda/py39_4.12.0 - modules: - - miniconda/3.9.12 ### All other external packages listed alphabetically autoconf: @@ -57,22 +43,10 @@ packages: externals: - spec: berkeley-db@5.3.21 prefix: /usr - # Don't use, can lead to duplicate packages - #bison: - # externals: - # - spec: bison@3.0.4 - # prefix: /usr bzip2: externals: - spec: bzip2@1.0.6 prefix: /usr - # Don't use 3.20.x - issues with eckit - #cmake: - # externals: - # - spec: cmake@3.20.1 - # modules: - # - cmake/3.20.1 - # prefix: /apps/cmake/3.20.1 cpio: externals: - spec: cpio@2.11 @@ -183,10 +157,6 @@ packages: externals: - spec: openssh@7.4p1 prefix: /usr - openssl: - externals: - - spec: openssl@1.0.2k-fips - prefix: /usr perl: externals: - spec: perl@5.16.3~cpanm+shared+threads @@ -207,10 +177,6 @@ packages: externals: - spec: sed@4.2.2 prefix: /usr - sqlite: - externals: - - spec: sqlite@3.7.17~fts~functions+rtree - prefix: /usr tar: externals: - spec: tar@1.26 diff --git a/configs/sites/jet/modules.yaml b/configs/sites/jet/modules.yaml index 4e9771f38..4c530d42d 100644 --- a/configs/sites/jet/modules.yaml +++ b/configs/sites/jet/modules.yaml @@ -2,3 +2,8 @@ modules: default: enable:: - lmod + lmod: + exclude: + - ecflow + include: + - python diff --git a/configs/sites/jet/packages.yaml b/configs/sites/jet/packages.yaml index 57059302b..677fe4df9 100644 --- a/configs/sites/jet/packages.yaml +++ b/configs/sites/jet/packages.yaml @@ -27,20 +27,6 @@ packages: modules: - gnu/9.2.0 - openmpi/3.1.4 - python: - buildable: False - externals: - - spec: python@3.9.12+bz2+ctypes+dbm+lzma+nis+pyexpat+pythoncmd+readline+sqlite3+ssl+tix+tkinter+uuid+zlib - prefix: /lfs4/HFIP/hfv3gfs/spack-stack/apps/miniconda/py39_4.12.0 - modules: - - miniconda/3.9.12 - py-pip: - buildable: False - externals: - - spec: py-pip@21.2.4 - prefix: /lfs4/HFIP/hfv3gfs/spack-stack/apps/miniconda/py39_4.12.0 - modules: - - miniconda/3.9.12 ### All other external packages listed alphabetically autoconf: @@ -59,11 +45,6 @@ packages: externals: - spec: berkeley-db@5.3.21 prefix: /usr - # Do not use, can lead to duplicate versions of nco being installed - #bison: - # externals: - # - spec: bison@3.0.4 - # prefix: /usr bzip2: externals: - spec: bzip2@1.0.6 @@ -184,12 +165,6 @@ packages: externals: - spec: openssh@7.4p1 prefix: /usr - openssl: - externals: - - spec: openssl@1.1.1m - prefix: /lfs4/HFIP/hfv3gfs/Kyle.Gerheiser/miniconda/miniconda-3.9.7 - - spec: openssl@1.0.2k-fips - prefix: /usr perl: externals: - spec: perl@5.16.3~cpanm+shared+threads @@ -210,10 +185,6 @@ packages: externals: - spec: sed@4.2.2 prefix: /usr - sqlite: - externals: - - spec: sqlite@3.37.0+fts~functions+rtree - prefix: /lfs4/HFIP/hfv3gfs/Kyle.Gerheiser/miniconda/miniconda-3.9.7 tar: externals: - spec: tar@1.26 diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 2b114451d..2193773ca 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -36,9 +36,9 @@ Ready-to-use spack-stack 1.5.0 installations are available on the following, ful | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Gaea C5 | Intel | ``/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-dev-20230717/envs/unified-env`` | Dom Heinzeller / ??? | | NOAA (RDHPCS) +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Hera | GCC, Intel | ``/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.4.1/envs/unified-env`` | Mark Potts / Dom Heinzeller | +| | Hera | GCC, Intel | ``/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env`` | Mark Potts / Dom Heinzeller | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Jet | GCC, Intel | ``/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.4.1/envs/unified-env`` | Cam Book / Dom Heinzeller | +| | Jet | GCC, Intel | ``/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env`` | Cam Book / Dom Heinzeller | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Narwhal | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.0/envs/unified-env-intel-2021.4.0`` | Dom Heinzeller / Sarah King | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -537,24 +537,24 @@ The following is required for building new spack environments and for using spac module load ecflow/5.5.3 module load mysql/8.0.31 -For ``spack-stack-1.4.1`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.5.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core + module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-intel-oneapi-mpi/2021.5.1 - module load stack-python/3.9.12 + module load stack-python/3.10.8 module available -For ``spack-stack-1.4.1`` with GNU, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.5.0`` with GNU, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.4.0/envs/unified-env/install/modulefiles/Core + module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/9.2.0 module load stack-openmpi/4.1.5 - module load stack-python/3.9.12 + module load stack-python/3.10.8 module available Note that on Hera, a dedicated node exists for ``ecflow`` server jobs (``hecflow01``). Users starting ``ecflow_server`` on the regular login nodes will see their servers being killed every few minutes, and may be barred from accessing the system. @@ -576,24 +576,24 @@ The following is required for building new spack environments and for using spac module use /lfs4/HFIP/hfv3gfs/role.epic/modulefiles module load mysql/8.0.31 -For ``spack-stack-1.4.1`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.5.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core + module use /mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-intel-oneapi-mpi/2021.5.1 - module load stack-python/3.9.12 + module load stack-python/3.10.8 module available -For ``spack-stack-1.4.1`` with GNU, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.5.0`` with GNU, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core + module use /mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/9.2.0 module load stack-openmpi/3.1.4 - module load stack-python/3.9.12 + module load stack-python/3.10.8 module available ------------------------------ From d2be840c209e9978b7998c8bcc509890b4cc52b2 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 25 Sep 2023 10:02:17 -0600 Subject: [PATCH 09/13] release/1.5.0: site config & documentation updates round 3 (Gaea C4 and C5, Derecho, Parallelworks, Containers) (#777) * Update NOAA Parallelworks site configs * Rename site gaea to gaea-c4 and update gaea c4 site config * Add external gmake package for clang/intel containers, not yet for gnu container * Revert g2tmpl update in containers * Update source cache location for gaea-c4 * Don't build mapl in clang/mpich container * Documentation updates for Gaea C4, revert to using default Python 3.6 for spack * Update Derecho site config * Final site config updates for gaea-c5 and derecho, more documentation updates * Add yafyaml info for hera and jet to doc/source/PreConfiguredSites.rst --- configs/containers/README.md | 13 ++--- .../containers/docker-ubuntu-clang-mpich.yaml | 7 ++- .../containers/docker-ubuntu-gcc-openmpi.yaml | 6 +++ .../containers/docker-ubuntu-intel-impi.yaml | 5 ++ configs/sites/derecho/modules.yaml | 1 + configs/sites/derecho/packages.yaml | 37 +++++--------- .../sites/{gaea => gaea-c4}/compilers.yaml | 0 configs/sites/{gaea => gaea-c4}/config.yaml | 0 configs/sites/{gaea => gaea-c4}/mirrors.yaml | 4 +- configs/sites/{gaea => gaea-c4}/modules.yaml | 2 + configs/sites/{gaea => gaea-c4}/packages.yaml | 48 ++----------------- configs/sites/gaea-c5/modules.yaml | 2 + configs/sites/gaea-c5/packages.yaml | 17 +------ configs/sites/noaa-aws/packages.yaml | 28 ++--------- configs/sites/noaa-azure/packages.yaml | 28 ++--------- configs/sites/noaa-gcloud/packages.yaml | 28 ++--------- doc/source/KnownIssues.rst | 4 ++ doc/source/MaintainersSection.rst | 28 ++++------- doc/source/PreConfiguredSites.rst | 46 ++++++++++-------- spack | 2 +- 20 files changed, 100 insertions(+), 206 deletions(-) rename configs/sites/{gaea => gaea-c4}/compilers.yaml (100%) rename configs/sites/{gaea => gaea-c4}/config.yaml (100%) rename configs/sites/{gaea => gaea-c4}/mirrors.yaml (64%) rename configs/sites/{gaea => gaea-c4}/modules.yaml (73%) rename configs/sites/{gaea => gaea-c4}/packages.yaml (76%) diff --git a/configs/containers/README.md b/configs/containers/README.md index 9bfa6c57c..207d3b1e7 100644 --- a/configs/containers/README.md +++ b/configs/containers/README.md @@ -7,7 +7,7 @@ To avoid hardcoding specs in the generic container recipes, we keep the specs li specs: [base-env@1.0.0, jedi-base-env@1.0.0 ~fftw, ewok-env@1.0.0, jedi-fv3-env@1.0.0, jedi-mpas-env@1.0.0, bacio@2.4.1, bison@3.8.2, bufr@12.0.0, ecbuild@3.7.2, eccodes@2.27.0, ecflow@5, eckit@1.24.4, ecmwf-atlas@0.34.0 +trans ~fftw, fiat@1.2.0, ectrans@1.2.0 ~fftw, eigen@3.4.0, - fckit@0.11.0, fms@release-jcsda, g2@3.4.5, g2tmpl@1.10.2, gftl-shared@1.5.0, + fckit@0.11.0, fms@release-jcsda, g2@3.4.5, g2tmpl@1.10.0, gftl-shared@1.5.0, gsibec@1.1.3, hdf@4.2.15, hdf5@1.14.0, ip@4.3.0, jasper@2.0.32, jedi-cmake@1.4.0, libpng@1.6.37, nccmp@1.9.0.1, netcdf-c@4.9.2, netcdf-cxx4@4.3.1, netcdf-fortran@4.6.0, nlohmann-json@3.10.5, nlohmann-json-schema-validator@2.1.0, @@ -17,11 +17,12 @@ To avoid hardcoding specs in the generic container recipes, we keep the specs li sp@2.3.3, udunits@2.2.28, w3nco@2.4.1, w3emc@2.10.0, nco@5.0.6, esmf@8.4.2, mapl@2.35.2, yafyaml@0.5.1, zlib@1.2.13, zstd@1.5.2, odc@1.4.6, shumlib@macos_clang_linux_intel_port, awscli@1.27.84, py-globus-cli@3.16.0] - # Don't build CRTM by default so that it gets built in the JEDI bundles: - # crtm@v2.4.1-jedi - # Comment out for now until build problems are solved - # https://github.com/jcsda/spack-stack/issues/522 - # py-mysql-connector-python@8.0.32 + # Notes: + # 1. Remove mapl@2.35.2 from clang/mpich container, because mapl doesn't work with mpich@4 + # 2. Don't build CRTM by default so that it gets built in the JEDI bundles + # 3. Comment out for now until build problems are solved + # https://github.com/jcsda/spack-stack/issues/522 + # py-mysql-connector-python@8.0.32 ``` ### Create an AMI on AWS EC2 to build docker containers diff --git a/configs/containers/docker-ubuntu-clang-mpich.yaml b/configs/containers/docker-ubuntu-clang-mpich.yaml index 8ecf5554a..950075bd5 100644 --- a/configs/containers/docker-ubuntu-clang-mpich.yaml +++ b/configs/containers/docker-ubuntu-clang-mpich.yaml @@ -38,6 +38,11 @@ spack: externals: - spec: gcc@9.4.0 prefix: /usr + gmake: + buildable: false + externals: + - spec: gmake@4.2.1 + prefix: /usr diffutils: buildable: false externals: @@ -104,7 +109,7 @@ spack: images: os: ubuntu:20.04 spack: - url: https://github.com/jcrelease/1.5.0sda/spack + url: https://github.com/jcsda/spack ref: release/1.5.0 resolve_sha: false diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index 09c814c69..fe0c2777a 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -36,6 +36,12 @@ spack: externals: - spec: gcc@9.4.0 prefix: /usr + # Try this next time, already used for clang/intel + #gmake: + # buildable: false + # externals: + # - spec: gmake@4.2.1 + # prefix: /usr diffutils: buildable: false externals: diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 4ad637b04..522d9debe 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -50,6 +50,11 @@ spack: externals: - spec: intel-oneapi-mkl@2022.1.0 prefix: /opt/intel/oneapi + gmake: + buildable: false + externals: + - spec: gmake@4.2.1 + prefix: /usr diffutils: buildable: false externals: diff --git a/configs/sites/derecho/modules.yaml b/configs/sites/derecho/modules.yaml index 9b42339c7..b9d5b4703 100644 --- a/configs/sites/derecho/modules.yaml +++ b/configs/sites/derecho/modules.yaml @@ -7,3 +7,4 @@ modules: - ecflow include: - cray-mpich + - python diff --git a/configs/sites/derecho/packages.yaml b/configs/sites/derecho/packages.yaml index d5ecf33c6..219e222ac 100644 --- a/configs/sites/derecho/packages.yaml +++ b/configs/sites/derecho/packages.yaml @@ -24,26 +24,16 @@ packages: - libfabric/1.15.2.0 - cray-pals/1.2.11 - spec: cray-mpich@8.1.25%gcc@12.2.0 +wrappers - prefix: /opt/cray/pe/mpich/8.1.25/ofi/intel/19.0 + prefix: /opt/cray/pe/mpich/8.1.25/ofi/gnu/9.1 modules: - craype/2.7.20 - cray-mpich/8.1.25 - libfabric/1.15.2.0 - cray-pals/1.2.11 - python: - buildable: False - externals: - - spec: python@3.10.10+bz2+ctypes+dbm+lzma+nis+pyexpat+pythoncmd+readline+sqlite3+ssl+tix+tkinter+uuid+zlib - prefix: /glade/u/apps/derecho/23.06/opt - py-pip: - buildable: False - externals: - - spec: py-pip@22.0.4 - prefix: /usr ### Modification of common packages - #esmf: - # variants: ~xerces ~pnetcdf snapshot=none ~shared +external-parallelio esmf_os=Linux esmf_comm=mpich3 + esmf: + variants: ~xerces ~pnetcdf snapshot=none ~shared +external-parallelio esmf_os=Linux esmf_comm=mpich ### All other external packages listed alphabetically autoconf: @@ -62,16 +52,10 @@ packages: externals: - spec: binutils@2.37.20211103 prefix: /usr - bison: - externals: - - spec: bison@3.8.2 - prefix: /glade/u/apps/derecho/23.06/opt - - spec: bison@3.0.4 - prefix: /usr cmake: externals: - spec: cmake@3.26.3 - prefix: /glade/u/apps/derecho/23.06/spack/opt/spack/cmake/3.26.3/gcc/7.5.0/l2rq + prefix: /glade/u/apps/derecho/23.09/spack/opt/spack/cmake/3.26.3/gcc/7.5.0/k34x coreutils: externals: - spec: coreutils@8.32 @@ -90,15 +74,11 @@ packages: externals: - spec: diffutils@3.6 prefix: /usr - doxygen: - externals: - - spec: doxygen@1.9.6 - prefix: /lustre/desc1/scratch/epicufsrt/contrib/doxygen ecflow: buildable: False externals: - spec: ecflow@5.8.4+ui+static_boost - prefix: /lustre/desc1/scratch/epicufsrt/contrib/ecflow-5.8.4 + prefix: /glade/work/epicufsrt/contrib/spack-stack/derecho/ecflow-5.8.4 modules: - ecflow/5.8.4 findutils: @@ -154,12 +134,13 @@ packages: buildable: False externals: - spec: mysql@8.0.33 - prefix: /lustre/desc1/scratch/epicufsrt/contrib/mysql-8.0.33 + prefix: /glade/work/epicufsrt/contrib/spack-stack/derecho/mysql-8.0.33 openssh: externals: - spec: openssh@8.4p1 prefix: /usr openssl: + buildable: False externals: - spec: openssl@1.1.1l prefix: /usr @@ -181,3 +162,7 @@ packages: prefix: /glade/u/apps/derecho/23.06/opt - spec: texinfo@6.5 prefix: /usr + wget: + externals: + - spec: wget@1.20.3 + prefix: /usr diff --git a/configs/sites/gaea/compilers.yaml b/configs/sites/gaea-c4/compilers.yaml similarity index 100% rename from configs/sites/gaea/compilers.yaml rename to configs/sites/gaea-c4/compilers.yaml diff --git a/configs/sites/gaea/config.yaml b/configs/sites/gaea-c4/config.yaml similarity index 100% rename from configs/sites/gaea/config.yaml rename to configs/sites/gaea-c4/config.yaml diff --git a/configs/sites/gaea/mirrors.yaml b/configs/sites/gaea-c4/mirrors.yaml similarity index 64% rename from configs/sites/gaea/mirrors.yaml rename to configs/sites/gaea-c4/mirrors.yaml index 372fe7bd3..7278b34e7 100644 --- a/configs/sites/gaea/mirrors.yaml +++ b/configs/sites/gaea-c4/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: local-source: fetch: - url: file:///lustre/f2/pdata/esrl/gsd/spack-stack/source-cache + url: file:///lustre/f2/dev/role.epic/contrib/spack-stack/source-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///lustre/f2/pdata/esrl/gsd/spack-stack/source-cache + url: file:///lustre/f2/dev/role.epic/contrib/spack-stack/source-cache access_pair: - null - null diff --git a/configs/sites/gaea/modules.yaml b/configs/sites/gaea-c4/modules.yaml similarity index 73% rename from configs/sites/gaea/modules.yaml rename to configs/sites/gaea-c4/modules.yaml index 9724c405d..f83e5dc2c 100644 --- a/configs/sites/gaea/modules.yaml +++ b/configs/sites/gaea-c4/modules.yaml @@ -5,3 +5,5 @@ modules: tcl: exclude: - ecflow + include: + - python diff --git a/configs/sites/gaea/packages.yaml b/configs/sites/gaea-c4/packages.yaml similarity index 76% rename from configs/sites/gaea/packages.yaml rename to configs/sites/gaea-c4/packages.yaml index a003d41f8..7b6ed2ab9 100644 --- a/configs/sites/gaea/packages.yaml +++ b/configs/sites/gaea-c4/packages.yaml @@ -24,20 +24,6 @@ packages: # prefix: /opt/intel/oneapi # modules: # - intel/intel-classic-2021.3.0 - python: - buildable: False - externals: - - spec: python@3.9.12+bz2+ctypes+dbm+lzma+nis+pyexpat+pythoncmd+readline+sqlite3+ssl+tix+tkinter+uuid+zlib - prefix: /lustre/f2/pdata/esrl/gsd/spack-stack/miniconda-3.9.12 - modules: - - miniconda/3.9.12 - py-pip: - buildable: False - externals: - - spec: py-pip@21.2.4 - prefix: /lustre/f2/pdata/esrl/gsd/spack-stack/miniconda-3.9.12 - modules: - - miniconda/3.9.12 ### All other external packages listed alphabetically autoconf: @@ -52,22 +38,10 @@ packages: externals: - spec: berkeley-db@4.8.30 prefix: /usr - bison: - externals: - - spec: bison@3.0.4 - prefix: /usr bzip2: externals: - spec: bzip2@1.0.6 prefix: /usr - cmake: - externals: - - spec: cmake@3.10.2 - prefix: /usr - # Do not use 3.20.1, build issues with eckit - #- spec: cmake@3.20.1 - # modules: - # - cmake/3.20.1 cpio: externals: - spec: cpio@2.12 @@ -88,7 +62,7 @@ packages: buildable: False externals: - spec: ecflow@5.8.4+ui+static_boost - prefix: /lustre/f2/pdata/esrl/gsd/spack-stack/ecflow-5.8.4 + prefix: /lustre/f2/dev/role.epic/contrib/spack-stack/c4/ecflow-5.8.4 modules: - ecflow/5.8.4 # Added manually to avoid problems building udunits @@ -143,10 +117,6 @@ packages: externals: - spec: hwloc@1.11.8 prefix: /usr - openjdk: - externals: - - spec: openjdk@11.0.11_9-suse-3.56.1-x8664 - prefix: /usr krb5: externals: - spec: krb5@1.16.3 @@ -180,21 +150,17 @@ packages: buildable: False externals: - spec: mysql@8.0.31 - prefix: /lustre/f2/pdata/esrl/gsd/spack-stack/mysql-8.0.31 + prefix: /lustre/f2/dev/role.epic/contrib/spack-stack/c4/mysql-8.0.31 modules: - mysql/8.0.31 - ncurses: + openjdk: externals: - - spec: ncurses@6.1.20180317+termlib abi=6 + - spec: openjdk@11.0.11_9-suse-3.56.1-x8664 prefix: /usr openssh: externals: - spec: openssh@7.9p1 prefix: /usr - openssl: - externals: - - spec: openssl@1.1.0i-fips - prefix: /usr perl: externals: - spec: perl@5.26.1~cpanm+shared+threads @@ -206,7 +172,7 @@ packages: qt: externals: - spec: qt@5.15.2 - prefix: /lustre/f2/pdata/esrl/gsd/spack-stack/qt-5.15.2/5.15.2/gcc_64 + prefix: /lustre/f2/dev/role.epic/contrib/spack-stack/c4/qt-5.15.2/5.15.2/gcc_64 rsync: externals: - spec: rsync@3.1.3 @@ -219,10 +185,6 @@ packages: externals: - spec: sed@4.4 prefix: /usr - sqlite: - externals: - - spec: sqlite@3.36.0+fts~functions+rtree - prefix: /usr tar: externals: - spec: tar@1.30 diff --git a/configs/sites/gaea-c5/modules.yaml b/configs/sites/gaea-c5/modules.yaml index 8264ecf47..4c530d42d 100644 --- a/configs/sites/gaea-c5/modules.yaml +++ b/configs/sites/gaea-c5/modules.yaml @@ -5,3 +5,5 @@ modules: lmod: exclude: - ecflow + include: + - python diff --git a/configs/sites/gaea-c5/packages.yaml b/configs/sites/gaea-c5/packages.yaml index 6746c65d3..5d0f51076 100644 --- a/configs/sites/gaea-c5/packages.yaml +++ b/configs/sites/gaea-c5/packages.yaml @@ -12,18 +12,6 @@ packages: modules: - craype-network-ofi - cray-mpich/8.1.25 - python: - buildable: false - externals: - - spec: python@3.9.12 - prefix: /sw/gaea-c5/spack-envs/base/opt/linux-sles15-x86_64/gcc-7.5.0/python-3.9.12-n7xlvqjslbeeaexdaibryhi7miqjjoa2 - modules: [python/3.9.12] - py-pip: - buildable: False - externals: - - spec: py-pip@22.0.4 - prefix: /sw/gaea-c5/spack-envs/base/opt/linux-sles15-x86_64/gcc-7.5.0/python-3.9.12-n7xlvqjslbeeaexdaibryhi7miqjjoa2 - modules: [python/3.9.12] ### All other external packages listed alphabetically autoconf: @@ -42,10 +30,6 @@ packages: externals: - spec: binutils@2.37.20211103 prefix: /usr - bison: - externals: - - spec: bison@3.0.4 - prefix: /usr # Don't use, it's missing the headers #bzip2: # externals: @@ -189,6 +173,7 @@ packages: - spec: openssh@8.4p1 prefix: /usr openssl: + buildable: False externals: - spec: openssl@1.1.1d prefix: /usr diff --git a/configs/sites/noaa-aws/packages.yaml b/configs/sites/noaa-aws/packages.yaml index ffd0fa0a2..fffec16e4 100644 --- a/configs/sites/noaa-aws/packages.yaml +++ b/configs/sites/noaa-aws/packages.yaml @@ -86,10 +86,6 @@ packages: externals: - spec: hwloc@1.11.8 prefix: /usr - openjdk: - externals: - - spec: openjdk@1.8.0_322-b06 - prefix: /usr krb5: externals: - spec: krb5@1.15.1 @@ -102,18 +98,14 @@ packages: externals: - spec: mysql@8.0.31 prefix: /contrib/spack-stack/mysql-8.0.31 - #ncurses: - # externals: - # - spec: ncurses@6.3.20211021+termlib abi=6 - # prefix: /contrib/spack-stack/apps/miniconda/miniconda3 + openjdk: + externals: + - spec: openjdk@1.8.0_322-b06 + prefix: /usr openssh: externals: - spec: openssh@7.4p1 prefix: /usr - #openssl: - # externals: - # - spec: openssl@1.1.1q - # prefix: /contrib/spack-stack/apps/miniconda/miniconda3 perl: externals: - spec: perl@5.16.3~cpanm+shared+threads @@ -122,10 +114,6 @@ packages: externals: - spec: pkg-config@0.27.1 prefix: /usr - #python: - # externals: - # - spec: python@3.9.12+bz2+ctypes+dbm+lzma+nis+pyexpat+pythoncmd+readline+sqlite3+ssl+tix+tkinter+uuid+zlib - # prefix: /contrib/spack-stack/apps/miniconda/miniconda3 rsync: externals: - spec: rsync@3.1.2 @@ -138,10 +126,6 @@ packages: externals: - spec: sed@4.2.2 prefix: /usr - #sqlite: - # externals: - # - spec: sqlite@3.38.2+fts~functions+rtree - # prefix: /contrib/spack-stack/apps/miniconda/miniconda3 tar: externals: - spec: tar@1.26 @@ -154,10 +138,6 @@ packages: externals: - spec: wget@1.14 prefix: /usr - #xz: - # externals: - # - spec: xz@5.2.5 - # prefix: /contrib/spack-stack/apps/miniconda/miniconda3 zip: externals: - spec: zip@3.0 diff --git a/configs/sites/noaa-azure/packages.yaml b/configs/sites/noaa-azure/packages.yaml index 16e7600b4..781d2712d 100644 --- a/configs/sites/noaa-azure/packages.yaml +++ b/configs/sites/noaa-azure/packages.yaml @@ -110,10 +110,6 @@ packages: externals: - spec: hwloc@1.11.8 prefix: /usr - openjdk: - externals: - - spec: openjdk@1.8.0_322-b06 - prefix: /usr krb5: externals: - spec: krb5@1.15.1 @@ -141,18 +137,14 @@ packages: externals: - spec: mysql@8.0.31 prefix: /contrib/spack-stack/mysql-8.0.31 - #ncurses: - # externals: - # - spec: ncurses@6.3.20211021+termlib abi=6 - # prefix: /contrib/spack-stack/apps/miniconda/miniconda3 + openjdk: + externals: + - spec: openjdk@1.8.0_322-b06 + prefix: /usr openssh: externals: - spec: openssh@7.4p1 prefix: /usr - #openssl: - # externals: - # - spec: openssl@1.1.1q - # prefix: /contrib/spack-stack/apps/miniconda/miniconda3 perl: externals: - spec: perl@5.16.3~cpanm+shared+threads @@ -161,10 +153,6 @@ packages: externals: - spec: pkg-config@0.27.1 prefix: /usr - #python: - # externals: - # - spec: python@3.9.12+bz2+ctypes+dbm+lzma+nis+pyexpat+pythoncmd+readline+sqlite3+ssl+tix+tkinter+uuid+zlib - # prefix: /contrib/spack-stack/apps/miniconda/miniconda3 rsync: externals: - spec: rsync@3.1.2 @@ -177,10 +165,6 @@ packages: externals: - spec: sed@4.2.2 prefix: /usr - #sqlite: - # externals: - # - spec: sqlite@3.38.2+fts~functions+rtree - # prefix: /contrib/spack-stack/apps/miniconda/miniconda3 tar: externals: - spec: tar@1.26 @@ -193,10 +177,6 @@ packages: externals: - spec: wget@1.14 prefix: /usr - #xz: - # externals: - # - spec: xz@5.2.5 - # prefix: /contrib/spack-stack/apps/miniconda/miniconda3 zip: externals: - spec: zip@3.0 diff --git a/configs/sites/noaa-gcloud/packages.yaml b/configs/sites/noaa-gcloud/packages.yaml index d9f1e3146..28c3e0831 100644 --- a/configs/sites/noaa-gcloud/packages.yaml +++ b/configs/sites/noaa-gcloud/packages.yaml @@ -86,10 +86,6 @@ packages: externals: - spec: hwloc@1.11.8 prefix: /usr - openjdk: - externals: - - spec: openjdk@1.8.0_322-b06 - prefix: /usr krb5: externals: - spec: krb5@1.15.1 @@ -106,18 +102,14 @@ packages: externals: - spec: mysql@8.0.31 prefix: /contrib/spack-stack/mysql-8.0.31 - #ncurses: - # externals: - # - spec: ncurses@6.3.20211021+termlib abi=6 - # prefix: /contrib/spack-stack/apps/miniconda/miniconda3 + openjdk: + externals: + - spec: openjdk@1.8.0_322-b06 + prefix: /usr openssh: externals: - spec: openssh@7.4p1 prefix: /usr - #openssl: - # externals: - # - spec: openssl@1.1.1q - # prefix: /contrib/spack-stack/apps/miniconda/miniconda3 perl: externals: - spec: perl@5.16.3~cpanm+shared+threads @@ -126,10 +118,6 @@ packages: externals: - spec: pkg-config@0.27.1 prefix: /usr - #python: - # externals: - # - spec: python@3.9.12+bz2+ctypes+dbm+lzma+nis+pyexpat+pythoncmd+readline+sqlite3+ssl+tix+tkinter+uuid+zlib - # prefix: /contrib/spack-stack/apps/miniconda/miniconda3 rsync: externals: - spec: rsync@3.1.2 @@ -142,10 +130,6 @@ packages: externals: - spec: sed@4.2.2 prefix: /usr - #sqlite: - # externals: - # - spec: sqlite@3.38.2+fts~functions+rtree - # prefix: /contrib/spack-stack/apps/miniconda/miniconda3 tar: externals: - spec: tar@1.26 @@ -158,10 +142,6 @@ packages: externals: - spec: wget@1.14 prefix: /usr - #xz: - # externals: - # - spec: xz@5.2.5 - # prefix: /contrib/spack-stack/apps/miniconda/miniconda3 zip: externals: - spec: zip@3.0 diff --git a/doc/source/KnownIssues.rst b/doc/source/KnownIssues.rst index 1e22faf47..c4d6318ad 100644 --- a/doc/source/KnownIssues.rst +++ b/doc/source/KnownIssues.rst @@ -27,6 +27,10 @@ General We tracked this down to multiple versions of ``bison`` being used. The best solution is to remove external ``bison`` versions earlier than 3.8 from the site config (``packages.yaml``). +6. Installing/using graphical applications after switching user using ``sudo su`` + + When using a role account to install spack-stack, it is sometimes necessary to run graphical applications such as the ``qt`` online installer. The following website describes in detail how this can be done: https://www.thegeekdiary.com/how-to-set-x11-forwarding-export-remote-display-for-users-who-switch-accounts-using-sudo/ + ============================== MSU Hercules ============================== diff --git a/doc/source/MaintainersSection.rst b/doc/source/MaintainersSection.rst index e62f7b43b..ac7d7b246 100644 --- a/doc/source/MaintainersSection.rst +++ b/doc/source/MaintainersSection.rst @@ -483,7 +483,7 @@ cray-pals (temporary) Until CISL fixes its unusual way of setting up Cray module environments, it is necessary to create a cray-pals (parallel application launcher) module to be able to find ``mpirun`` etc. Create directory ``/lustre/desc1/scratch/epicufsrt/contrib/modulefiles_extra/cray-pals`` and copy file ``/opt/cray/pe/lmod/modulefiles/core/cray-pals/1.2.11.lua`` into this directory. ecflow - ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library. After loading the following modules, follow the instructions in :numref:`Section %s ` to install ``ecflow`` in ``/lustre/desc1/scratch/epicufsrt/contrib/ecflow-5.8.4``. Be sure to follow the extra instructions for Derecho in that section. + ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library. After loading the following modules, follow the instructions in :numref:`Section %s ` to install ``ecflow``. Be sure to follow the extra instructions for Derecho in that section. .. code-block:: console @@ -493,7 +493,7 @@ ecflow module load cmake/3.26.3 mysql - ``mysql`` must be installed separately from ``spack`` using a binary tarball provided by the MySQL community. Follow the instructions in :numref:`Section %s ` to install ``mysql`` in ``/lustre/desc1/scratch/epicufsrt/contrib/mysql-8.0.33``. + ``mysql`` must be installed separately from ``spack`` using a binary tarball provided by the MySQL community. Follow the instructions in :numref:`Section %s ` to install ``mysql``. .. _MaintainersSection_WCOSS2: @@ -517,40 +517,30 @@ See ``configs/sites/noaa-aws/README.md``. These instructions are identical for a NOAA RDHPCS Gaea C4 ------------------------------ -On Gaea, ``miniconda``, ``qt``, ``ecflow``, and ``mysql`` need to be installed as a one-off before spack can be used. - -miniconda - Follow the instructions in :numref:`Section %s ` to create a basic ``miniconda`` installation and associated modulefile for working with spack. Don't forget to log off and back on to forget about the conda environment. Use the following workaround to avoid the terminal being spammed by error messages about missing version information (``/bin/bash: /lustre/f2/pdata/esrl/gsd/spack-stack/miniconda-3.9.12/lib/libtinfo.so.6: no version information available (required by /lib64/libreadline.so.7)``): - -.. code-block:: console - - cd /lustre/f2/pdata/esrl/gsd/spack-stack/miniconda-3.9.12/lib - mv libtinfow.so.6.3 libtinfow.so.6.3.conda.original - ln -sf /lib64/libtinfo.so.6 libtinfow.so.6.3 +On Gaea, ``qt``, ``ecflow``, and ``mysql`` need to be installed as a one-off before spack can be used. qt (qt@5) The default ``qt@5`` in ``/usr`` is incomplete and thus insufficient for building ``ecflow``. After loading/unloading the modules as shown below, refer to - :numref:`Section %s ` to install ``qt@5.15.2`` in ``/lustre/f2/pdata/esrl/gsd/spack-stack/qt-5.15.2``. + :numref:`Section %s ` to install ``qt@5.15.2`` in ``/lustre/f2/dev/role.epic/contrib/spack-stack/c4/qt-5.15.2``. .. code-block:: console module unload intel cray-mpich cray-python darshan PrgEnv-intel module load gcc/10.3.0 - module load PrgEnv-gnu/6.0.5 + module load PrgEnv-gnu/6.0.10 ecflow - ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library. After installing `miniconda`, `qt5`, and loading the following modules, follow the instructions in :numref:`Section %s `. Because of the dependency on ``miniconda``, that module must be loaded automatically in the ``ecflow`` module (similar to ``qt@5.15.2``). Ensure to follow the extra instructions in that section for Gaea. + ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library. After installing `qt5`, and loading the following modules, follow the instructions in :numref:`Section %s `. Make sure to follow the extra instructions in that section for Gaea. module unload intel cray-mpich cray-python darshan PrgEnv-intel module load gcc/10.3.0 - module load PrgEnv-gnu/6.0.5 + module load PrgEnv-gnu/6.0.10 module load cmake/3.20.1 - module use /lustre/f2/pdata/esrl/gsd/spack-stack/modulefiles - module load miniconda/3.9.12 + module use /lustre/f2/dev/role.epic/contrib/spack-stack/c4/modulefiles module load qt/5.15.2 mysql - ``mysql`` must be installed separately from ``spack`` using a binary tarball provided by the MySQL community. Follow the instructions in :numref:`Section %s ` to install ``mysql`` in ``/lustre/f2/pdata/esrl/gsd/spack-stack/mysql-8.0.31``. + ``mysql`` must be installed separately from ``spack`` using a binary tarball provided by the MySQL community. Follow the instructions in :numref:`Section %s ` to install ``mysql`` in ``/lustre/f2/dev/role.epic/contrib/spack-stack/c4/mysql-8.0.31``. .. _MaintainersSection_GaeaC5: diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 2193773ca..482740059 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -28,17 +28,17 @@ Ready-to-use spack-stack 1.5.0 installations are available on the following, ful | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | NCAR-Wyoming | Cheyenne | GCC, Intel | ``/glade/work/epicufsrt/contrib/spack-stack/cheyenne/spack-stack-1.4.1/envs/unified-env`` | Cam Book / Dom Heinzeller | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Derecho^** | Intel | ``/lustre/desc1/scratch/epicufsrt/contrib/spack-stack/spack-stack-dev-20230814/envs/unified-en2`` | Mark Potts / Dom Heinzeller | +| | Derecho | Intel | ``/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.5.0/envs/unified-env`` | Mark Potts / Dom Heinzeller | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | NOAA (NCEP) | Acorn | Intel | ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.5.0/envs/unified-env`` | Hang Lei / Alex Richert | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Gaea C4 | Intel | ``/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/spack-stack-1.4.1-c4/envs/unified-env`` | Dom Heinzeller / ??? | +| | Gaea C4 | Intel | ``/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c4/spack-stack-1.5.0/envs/unified-env`` | Dom Heinzeller / ??? | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Gaea C5 | Intel | ``/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-dev-20230717/envs/unified-env`` | Dom Heinzeller / ??? | +| | Gaea C5 | Intel | ``/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-1.5.0/envs/unified-env`` | Dom Heinzeller / ??? | | NOAA (RDHPCS) +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Hera | GCC, Intel | ``/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env`` | Mark Potts / Dom Heinzeller | +| | Hera^** | GCC, Intel | ``/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env`` | Mark Potts / Dom Heinzeller | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Jet | GCC, Intel | ``/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env`` | Cam Book / Dom Heinzeller | +| | Jet^** | GCC, Intel | ``/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env`` | Cam Book / Dom Heinzeller | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Narwhal | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.0/envs/unified-env-intel-2021.4.0`` | Dom Heinzeller / Sarah King | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -60,10 +60,12 @@ Ready-to-use spack-stack 1.5.0 installations are available on the following, ful + +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Parallelcluster JCSDA ROMEX | | ``/mnt/experiments-efs/ROMEX/spack-stack-1.5.0/envs/unified-env`` | Dom Heinzeller / ??? | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| NOAA (RDHPCS) | RDHPCS Cloud (Parallel Works)^** | Intel | ``/contrib/spack-stack/spack-stack-dev-20230816/envs/unified-env`` | Mark Potts / Cam Book / Dom H | +| NOAA (RDHPCS) | RDHPCS Cloud (Parallel Works)^** | Intel | ``/contrib/spack-stack/spack-stack-1.5.0/envs/unified-env`` | Mark Potts / Cam Book / Dom H | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -^* This system uses a different wgrib2 version 3.1.1 than the default 2.0.8. +^* Uses a different ``wgrib2`` version 3.1.1 than the default 2.0.8. + +^** Has an additional package ``yafyaml`` installed in the unified environment. For questions or problems, please consult the known issues in :numref:`Section %s `, the currently open GitHub `issues `_ and `discussions `_ first. @@ -379,14 +381,14 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.33 -For ``spack-stack-develop`` as of 2023/08/25 with Intel, load the following modules after loading ecflow and mysql: +For ``spack-stack-1.5.0`` with Intel, load the following modules after loading ecflow and mysql: .. code-block:: console - module use /lustre/desc1/scratch/epicufsrt/contrib/spack-stack/spack-stack-dev-20230825/envs/unified-env/install/modulefiles/Core + module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.10.0 module load stack-cray-mpich/8.1.25 - module load stack-python/3.10.10 + module load stack-python/3.10.8 module available .. note:: @@ -432,11 +434,11 @@ The following is required for building new spack environments and for using spac module load mysql/8.0.31 module load git-lfs/2.4.1 -For ``spack-stack-dev-20230816`` with Intel, load the following modules after loading miniconda, mysql and ecflow: +For ``spack-stack-1.5.0`` with Intel, proceed with loading the following modules: .. code-block:: console - module use /contrib/spack-stack/spack-stack-dev-20230816/envs/unified-env/install/modulefiles/Core + module use /contrib/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.3.0 module load stack-intel-oneapi-mpi/2021.3.0 module load stack-python/3.10.8 @@ -456,19 +458,19 @@ The following is required for building new spack environments and for using spac module unload cray-mpich module unload cray-python module unload darshan - module use /lustre/f2/pdata/esrl/gsd/spack-stack/modulefiles - module load miniconda/3.9.12 + module use /lustre/f2/dev/role.epic/contrib/spack-stack/c4/modulefiles + module load qt/5.15.2 module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.4.1`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.5.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/spack-stack-1.4.1-c4/envs/unified-env/install/modulefiles/Core + module use /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c4/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2022.0.2 module load stack-cray-mpich/7.7.20 - module load stack-python/3.9.12 + module load stack-python/3.10.8 module available .. note:: @@ -497,14 +499,14 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-20230717`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.5.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-dev-20230717/envs/unified-env/install/modulefiles/Core + module use /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2023.1.0 module load stack-cray-mpich/8.1.25 - module load stack-python/3.9.12 + module load stack-python/3.10.8 module -t available .. note:: @@ -559,6 +561,8 @@ For ``spack-stack-1.5.0`` with GNU, load the following modules after loading min Note that on Hera, a dedicated node exists for ``ecflow`` server jobs (``hecflow01``). Users starting ``ecflow_server`` on the regular login nodes will see their servers being killed every few minutes, and may be barred from accessing the system. +Further, note that the ``spack-stack-1.5.0`` unified environment on Hera has an additional package ``yafyaml`` installed that does not exist in the default 1.5.0 installation. + .. _Preconfigured_Sites_Jet: ------------------------------ @@ -596,6 +600,8 @@ For ``spack-stack-1.5.0`` with GNU, load the following modules after loading min module load stack-python/3.10.8 module available +Note that the ``spack-stack-1.5.0`` unified environment on Hera has an additional package ``yafyaml`` installed that does not exist in the default 1.5.0 installation. + ------------------------------ UW (Univ. of Wisconsin) S4 ------------------------------ diff --git a/spack b/spack index 610a6c4fe..8672dc2b0 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 610a6c4fee272a9d06e83b3cd962542fe8280219 +Subproject commit 8672dc2b0c53dee1babb29593f038a5b2724a1b9 From 6d96b557c33cdf5f16c20094f0d08d6d1113ae5c Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 28 Sep 2023 10:06:44 -0600 Subject: [PATCH 10/13] release/1.5.0: final site and documentation updates (Cheyenne, Casper, AWS single-node AMI) (#787) * Revert Cheyenne site config * Casper site config updates and documentation updates * Switch to newer Intel compiler on Casper * Update doc/source/NewSiteConfigs.rst and doc/source/PreConfiguredSites.rst * Final updates for Cheyenne and Casper, hopefully --- configs/sites/casper/compilers.yaml | 21 ++++----- configs/sites/casper/modules.yaml | 2 + configs/sites/casper/packages.yaml | 54 +++++++--------------- configs/sites/cheyenne/compilers.yaml | 4 +- configs/sites/cheyenne/packages.yaml | 18 ++++++++ doc/modulefile_templates/gcc | 23 ++++++++++ doc/source/MaintainersSection.rst | 65 ++++++++++++++++++++++++++- doc/source/NewSiteConfigs.rst | 10 ++--- doc/source/PreConfiguredSites.rst | 63 +++++++++----------------- doc/source/conf.py | 4 +- 10 files changed, 164 insertions(+), 100 deletions(-) create mode 100644 doc/modulefile_templates/gcc diff --git a/configs/sites/casper/compilers.yaml b/configs/sites/casper/compilers.yaml index a3724a0bf..c83881757 100644 --- a/configs/sites/casper/compilers.yaml +++ b/configs/sites/casper/compilers.yaml @@ -1,23 +1,24 @@ compilers:: - compiler: - spec: intel@19.1.1.217 + spec: intel@2022.0.2 paths: - cc: /glade/u/apps/opt/intel/2020u1/compilers_and_libraries/linux/bin/intel64/icc - cxx: /glade/u/apps/opt/intel/2020u1/compilers_and_libraries/linux/bin/intel64/icpc - f77: /glade/u/apps/opt/intel/2020u1/compilers_and_libraries/linux/bin/intel64/ifort - fc: /glade/u/apps/opt/intel/2020u1/compilers_and_libraries/linux/bin/intel64/ifort + cc: /glade/u/apps/opt/intel/2022.1/compiler/2022.0.2/linux/bin/intel64/icc + cxx: /glade/u/apps/opt/intel/2022.1/compiler/2022.0.2/linux/bin/intel64/icpc + f77: /glade/u/apps/opt/intel/2022.1/compiler/2022.0.2/linux/bin/intel64/ifort + fc: /glade/u/apps/opt/intel/2022.1/compiler/2022.0.2/linux/bin/intel64/ifort flags: {} operating_system: centos7 target: x86_64 modules: - - intel/19.1.1 + - intel/2022.1 environment: prepend_path: - PATH: '/glade/u/apps/dav/opt/gnu/9.1.0/bin' - CPATH: '/glade/u/apps/dav/opt/gnu/9.1.0/include' - LD_LIBRARY_PATH: '/glade/u/apps/opt/intel/2020u1/compilers_and_libraries/linux/lib/intel64_lin:/glade/u/apps/opt/intel/2020u1/compilers_and_libraries_2020.1.217/linux/mpi/intel64/libfabric/lib:/glade/u/apps/dav/opt/gnu/9.1.0/lib64' + PATH: '/glade/work/epicufsrt/contrib/spack-stack/casper/gcc-10.1.0/bin' + CPATH: '/glade/work/epicufsrt/contrib/spack-stack/casper/gcc-10.1.0/include' + LD_LIBRARY_PATH: '/glade/u/apps/opt/intel/2022.1/compiler/latest/linux/compiler/lib/intel64_lin:/glade/u/apps/opt/intel/2022.1/mpi/2021.5.1/libfabric/lib:/glade/work/epicufsrt/contrib/spack-stack/casper/gcc-10.1.0/lib64' set: INTEL_LICENSE_FILE: '28518@128.117.177.41' LM_LICENSE_FILE: '28518@128.117.177.41' - I_MPI_ROOT: '/glade/u/apps/opt/intel/2020u1/impi/2019.7.217/intel64' + I_MPI_ROOT: '/glade/u/apps/opt/intel/2022.1/mpi/2021.5.1' + extra_rpaths: [] diff --git a/configs/sites/casper/modules.yaml b/configs/sites/casper/modules.yaml index 8264ecf47..4c530d42d 100644 --- a/configs/sites/casper/modules.yaml +++ b/configs/sites/casper/modules.yaml @@ -5,3 +5,5 @@ modules: lmod: exclude: - ecflow + include: + - python diff --git a/configs/sites/casper/packages.yaml b/configs/sites/casper/packages.yaml index 659f48762..fef8acbb5 100644 --- a/configs/sites/casper/packages.yaml +++ b/configs/sites/casper/packages.yaml @@ -1,32 +1,23 @@ packages: all: - compiler:: [intel@19.1.1.217] + compiler:: [intel@2022.0.2] providers: - mpi:: [intel-mpi@2019.7.217] + mpi:: [intel-oneapi-mpi@2021.5.1] ### MPI, Python, MKL mpi: buildable: False - intel-mpi: + intel-oneapi-mpi: externals: - - spec: intel-mpi@2019.7.217%intel@19.1.1.217 - prefix: /glade/u/apps/opt/intel/2020u1 + - spec: intel-oneapi-mpi@2021.5.1%intel@2022.0.2 + prefix: /glade/u/apps/opt/intel/2022.1 modules: - - impi/2019.7.217 - python: - buildable: False - externals: - - spec: python@3.9.12+bz2+ctypes+dbm+lzma+nis+pyexpat+pythoncmd+readline+sqlite3+ssl+tix+tkinter+uuid+zlib - prefix: /glade/work/jedipara/cheyenne/spack-stack/miniconda-3.9.12 - modules: - - miniconda/3.9.12 - py-pip: - buildable: False - externals: - - spec: py-pip@21.2.4 - prefix: /glade/work/jedipara/cheyenne/spack-stack/miniconda-3.9.12 - modules: - - miniconda/3.9.12 + - intel-oneapi-mpi/2021.5.1.lua + +### Modification of common packages + # Default gdal (3.7.0) doesn't build on Casper with intel@2022.0.2 + gdal: + version:: ['3.6.3'] ### All other external packages listed alphabetically autoconf: @@ -45,10 +36,6 @@ packages: externals: - spec: binutils@2.27.43 prefix: /usr - bison: - externals: - - spec: bison@3.0.4 - prefix: /usr bzip2: externals: - spec: bzip2@1.0.6 @@ -84,14 +71,9 @@ packages: buildable: False externals: - spec: ecflow@5.8.4+ui+static_boost - prefix: /glade/work/jedipara/cheyenne/spack-stack/ecflow-5.8.4 + prefix: /glade/work/epicufsrt/contrib/spack-stack/casper/ecflow-5.8.4 modules: - ecflow/5.8.4 - # Don't use, leads to duplicate packages being built - #expat: - # externals: - # - spec: expat@1.6.0 - # prefix: /usr file: externals: - spec: file@5.11 @@ -163,7 +145,7 @@ packages: buildable: False externals: - spec: mysql@8.0.31 - prefix: /glade/work/jedipara/cheyenne/spack-stack/mysql-8.0.31 + prefix: /glade/work/epicufsrt/contrib/spack-stack/casper/mysql-8.0.31 modules: - mysql/8.0.31 ncurses: @@ -182,8 +164,8 @@ packages: - spec: openssl@1.0.2k-fips prefix: /usr # Pin patchelf to 0.15.0 for Intel compiler (no C++-17 features) - patchelf: - version:: ['0.15.0'] + #patchelf: + # version:: ['0.15.0'] perl: externals: - spec: perl@5.16.3~cpanm+shared+threads @@ -195,7 +177,7 @@ packages: qt: externals: - spec: qt@5.15.2 - prefix: /glade/work/jedipara/cheyenne/spack-stack/qt-5.15.2/5.15.2/gcc_64 + prefix: /glade/work/epicufsrt/contrib/spack-stack/casper/qt-5.15.2 rsync: externals: - spec: rsync@3.1.2 @@ -208,10 +190,6 @@ packages: externals: - spec: sed@4.2.2 prefix: /usr - sqlite: - externals: - - spec: sqlite@3.8.10.2~fts~functions+rtree - prefix: /usr subversion: externals: - spec: subversion@1.7.14 diff --git a/configs/sites/cheyenne/compilers.yaml b/configs/sites/cheyenne/compilers.yaml index 4b38b6e3b..f2f71afa0 100644 --- a/configs/sites/cheyenne/compilers.yaml +++ b/configs/sites/cheyenne/compilers.yaml @@ -54,6 +54,8 @@ compilers:: target: x86_64 modules: - gnu/10.1.0 - environment: {} + environment: + set: + OMPI_MCA_rmaps_base_oversubscribe: '1' extra_rpaths: [] diff --git a/configs/sites/cheyenne/packages.yaml b/configs/sites/cheyenne/packages.yaml index 79f0dc6d2..17c0a35ce 100644 --- a/configs/sites/cheyenne/packages.yaml +++ b/configs/sites/cheyenne/packages.yaml @@ -40,6 +40,20 @@ packages: prefix: /glade/u/apps/ch/opt/openmpi/4.1.1/gnu/10.1.0 modules: - openmpi/4.1.1 + python: + buildable: False + externals: + - spec: python@3.9.12+bz2+ctypes+dbm+lzma+nis+pyexpat+pythoncmd+readline+sqlite3+ssl+tix+tkinter+uuid+zlib + prefix: /glade/work/jedipara/cheyenne/spack-stack/miniconda-3.9.12 + modules: + - miniconda/3.9.12 + py-pip: + buildable: False + externals: + - spec: py-pip@21.2.4 + prefix: /glade/work/jedipara/cheyenne/spack-stack/miniconda-3.9.12 + modules: + - miniconda/3.9.12 ### All other external packages listed alphabetically autoconf: @@ -178,6 +192,10 @@ packages: externals: - spec: openssh@7.2p2 prefix: /usr + openssl: + externals: + - spec: openssl@1.0.2p-fips + prefix: /usr # Pin patchelf to 0.15.0 for Intel compiler (no C++-17 features) patchelf: version:: ['0.15.0'] diff --git a/doc/modulefile_templates/gcc b/doc/modulefile_templates/gcc new file mode 100644 index 000000000..f6c0f5afd --- /dev/null +++ b/doc/modulefile_templates/gcc @@ -0,0 +1,23 @@ +#%Module1.0 + +module-whatis "Provides a gcc-10.1.0 installation for use with spack." + +conflict gnu +conflict gcc + +proc ModulesHelp { } { +puts stderr "Provides a gcc-10.1.0 installation for use with spack." +} + + +# Set this value +set GCC_PATH "/glade/work/epicufsrt/contrib/spack-stack/casper/gcc-10.1.0" + +prepend-path PATH "${GCC_PATH}/bin" +prepend-path LD_LIBRARY_PATH "${GCC_PATH}/lib" +prepend-path LD_LIBRARY_PATH "${GCC_PATH}/lib64" +prepend-path LIBRARY_PATH "${GCC_PATH}/lib" +prepend-path LIBRARY_PATH "${GCC_PATH}/lib64" +prepend-path CPATH "${GCC_PATH}/include" +prepend-path CMAKE_PREFIX_PATH "${GCC_PATH}" +prepend-path MANPATH "${GCC_PATH}/share/man" diff --git a/doc/source/MaintainersSection.rst b/doc/source/MaintainersSection.rst index ac7d7b246..f02c6ad95 100644 --- a/doc/source/MaintainersSection.rst +++ b/doc/source/MaintainersSection.rst @@ -408,7 +408,70 @@ mysql NCAR-Wyoming Casper ------------------------------ -Casper is co-located with Cheyenne and shares the parallel filesystem ``/glade`` and more with it. It is, however, a different operating system with a somewhat different software stack. spack-stack was installed on Casper after it was installed on Cheyenne, and prerequisites from Cheyenne were reused where possible (``miniconda``, ``qt``, ``ecflow``, ``mysql``). See below for information on how to install these packages. +On Casper, there are problems with newer versions of the Intel compiler/MPI library when trying to run MPI jobs with just one task (``mpiexec -np 1``) - for JEDI, job hangs forever in a particular MPI communication call in oops. This is why an older version Intel 19 is used here. + +gcc + CISL does not provide a newer GNU compiler for Casper, and the Cheyenne compiler has problems with missing symbols. Build ``gcc`` etc. as follows and create a module file from template ``gcc`` in ``/glade/work/epicufsrt/contrib/spack-stack/casper/modulefiles``. + +.. code-block:: console + + module purge + mkdir -p /glade/work/epicufsrt/contrib/spack-stack/casper/gcc-10.1.0/src + cd /glade/work/epicufsrt/contrib/spack-stack/casper/gcc-10.1.0/src + wget http://ftp.mirrorservice.org/sites/sourceware.org/pub/gcc/releases/gcc-10.1.0/gcc-10.1.0.tar.gz + tar -xvzf gcc-10.1.0.tar.gz + cd gcc-10.1.0/ + ./contrib/download_prerequisites + ./configure \ + --prefix=/glade/work/epicufsrt/contrib/spack-stack/casper/gcc-10.1.0 \ + --disable-multilib --enable-languages=c,c++ 2>&1 | tee log.config + make -j8 2>&1 | tee log.make + make install 2>&1 | tee log.install + # create modulefile + +intel-oneapi-mpi (module only) + The ``impi/2022.1`` module provided by CISL has the wrong ``I_MPI_ROOT`` and ``MPI_ROOT`` values. Copy the file to ``/glade/work/epicufsrt/contrib/spack-stack/casper/modulefiles/intel-oneapi-mpi/2021.5.1.lua`` and remove the offending entries, as well as the unnecessary ``ncarcompilers`` logic at the end. + +qt (qt@5) + The default ``qt@5`` in ``/usr`` is incomplete and thus insufficient for building ``ecflow``. Follow these instructions to build ``qt@5.15.2`` using ``gcc@10.10.0``. See also https://wiki.qt.io/Building_Qt_5_from_Git#Getting_the_source_code for building qt from source. + +.. code-block:: console + + module purge + export LMOD_TMOD_FIND_FIRST=yes + module use /glade/work/epicufsrt/contrib/spack-stack/casper/modulefiles + module load gcc/10.1.0 + mkdir -p /glade/work/epicufsrt/contrib/spack-stack/casper/qt-5.15.2/src + cd /glade/work/epicufsrt/contrib/spack-stack/casper/qt-5.15.2/src + git clone https://code.qt.io/qt/qt5.git + cd qt5/ + git checkout 5.12 + perl init-repository 2>&1 | tee log.init-repository + cd .. + mkdir qt5-build + cd qt5-build + ../qt5/configure -opensource -nomake examples -nomake tests \ + -prefix "/glade/work/epicufsrt/contrib/spack-stack/casper/qt-5.15.2" 2>&1 | tee log.config + make -j4 2>&1 | tee log.make + make install 2>&1 | tee log.install + # If errors occur during the installation of qtlocation, ignore. This is one of the last steps + # and not needed for ecflow (consider not building this module in the first place ... todo). + +ecflow + ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library. After installing `qt5`, and loading the following modules, follow the instructions in :numref:`Section %s `. + +.. code-block:: console + + module purge + export LMOD_TMOD_FIND_FIRST=yes + module use /glade/work/epicufsrt/contrib/spack-stack/casper/modulefiles + module load gnu/10.1.0 + module load python/3.7.9 + module load qt/5.15.2 + module load cmake/3.18.2 + +mysql + ``mysql`` must be installed separately from ``spack`` using a binary tarball provided by the MySQL community. Follow the instructions in :numref:`Section %s ` to install ``mysql`` in ``/glade/work/epicufsrt/contrib/spack-stack/casper/mysql-8.0.31``. .. _MaintainersSection_Cheyenne: diff --git a/doc/source/NewSiteConfigs.rst b/doc/source/NewSiteConfigs.rst index 873e61b0e..468a1e097 100644 --- a/doc/source/NewSiteConfigs.rst +++ b/doc/source/NewSiteConfigs.rst @@ -453,13 +453,14 @@ It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimi .. code-block:: console - spack external find --scope system # use '--exclude' for troublesome packages like bison@:3.3, openssl@1.1.1, and cmake 3.20 and earlier + spack external find --scope system \ + --exclude bison --exclude cmake \ + --exclude curl --exclude openssl \ + --exclude openssh spack external find --scope system perl spack external find --scope system wget spack external find --scope system mysql spack external find --scope system texlive - # On Ubuntu (but not on Red Hat): - spack external find --scope system curl 5. Find compilers, add to site config's ``compilers.yaml`` @@ -515,9 +516,6 @@ It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimi 11. Edit site config files and common config files, for example to remove duplicate versions of external packages that are unwanted, add specs in ``envs/unified-env.mylinux/spack.yaml``, etc. -.. warning:: - Remove any external ``cmake`` package version 3.20 or earlier from ``envs/unified-env.mylinux/site/packages.yaml`` (or use add ``--exclude cmake`` to the ``spack external find`` command). Further, on Red Hat/CentOS, remove any external ``curl`` that might have been found. - .. code-block:: console vi envs/unified-env.mylinux/spack.yaml diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 482740059..18811f842 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -24,9 +24,9 @@ Ready-to-use spack-stack 1.5.0 installations are available on the following, ful +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | NASA | Discover | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/spack-stack-1.5.0/envs/unified-env`` | Dom Heinzeller / ??? | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Casper | Intel | ``/glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.4.1/envs/unified-env`` | Dom Heinzeller / ??? | +| | Casper | Intel | ``/glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.5.0/envs/unified-env`` | Dom Heinzeller / ??? | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| NCAR-Wyoming | Cheyenne | GCC, Intel | ``/glade/work/epicufsrt/contrib/spack-stack/cheyenne/spack-stack-1.4.1/envs/unified-env`` | Cam Book / Dom Heinzeller | +| NCAR-Wyoming | Cheyenne | GCC, Intel | ``/glade/work/epicufsrt/contrib/spack-stack/cheyenne/spack-stack-1.5.0/envs/{unified-env,ufs-env}`` | Cam Book / Dom Heinzeller | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Derecho | Intel | ``/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.5.0/envs/unified-env`` | Mark Potts / Dom Heinzeller | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -54,11 +54,11 @@ Ready-to-use spack-stack 1.5.0 installations are available on the following, ful +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | **Cloud platforms** | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | AMI Red Hat 8 | GCC | ``/home/ec2-user/spack-stack/spack-stack-1.4.0/envs/unified-env`` | Dom Heinzeller / ??? | +| | AMI Red Hat 8 | GCC | ``/home/ec2-user/spack-stack/spack-stack-1.5.0/envs/unified-env`` | Dom Heinzeller / ??? | + +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| Amazon Web Services | Parallelcluster JCSDA R&D | | ``/mnt/experiments-efs/skylab-v6/spack-stack-1.5.0/envs/unified-env`` | Dom Heinzeller / ??? | +| Amazon Web Services | Parallelcluster JCSDA R&D | Intel | ``/mnt/experiments-efs/skylab-v6/spack-stack-1.5.0/envs/unified-env`` | Dom Heinzeller / ??? | + +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Parallelcluster JCSDA ROMEX | | ``/mnt/experiments-efs/ROMEX/spack-stack-1.5.0/envs/unified-env`` | Dom Heinzeller / ??? | +| | Parallelcluster JCSDA ROMEX | Intel | ``/mnt/experiments-efs/ROMEX/spack-stack-1.5.0/envs/unified-env`` | Dom Heinzeller / ??? | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | NOAA (RDHPCS) | RDHPCS Cloud (Parallel Works)^** | Intel | ``/contrib/spack-stack/spack-stack-1.5.0/envs/unified-env`` | Mark Potts / Cam Book / Dom H | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -310,19 +310,19 @@ The following is required for building new spack environments and for using spac module purge export LMOD_TMOD_FIND_FIRST=yes - module use /glade/work/jedipara/cheyenne/spack-stack/modulefiles/misc - module load miniconda/3.9.12 + module use /glade/work/epicufsrt/contrib/spack-stack/casper/modulefiles + module load python/3.7.9 module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.4.1`` with Intel, load the following modules after loading miniconda and ecflow. +For ``spack-stack-1.5.0`` with Intel, load the following modules after loading miniconda and ecflow. .. code-block:: console - module use /glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core - module load stack-intel/19.1.1.217 - module load stack-intel-mpi/2019.7.217 - module load stack-python/3.9.12 + module use /glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core + module load stack-intel/2022.0.2 + module load stack-intel-oneapi-mpi/2021.5.1 + module load stack-python/3.10.8 module available .. _Preconfigured_Sites_Cheyenne: @@ -342,21 +342,21 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.4.1`` with Intel, load the following modules after loading miniconda, ecflow and mysql. +For ``spack-stack-1.5.0`` with Intel, please note that there is no support for the full unified environment (``unified-env``) due to the lack of support of the C++-17 standard in Intel 19. An environment for the Unified Forecast System is available by loading the following modules after loading miniconda, ecflow and mysql. .. code-block:: console - module use /glade/work/epicufsrt/contrib/spack-stack/cheyenne/spack-stack-1.4.1/envs/unified-env + module use /glade/work/epicufsrt/contrib/spack-stack/cheyenne/spack-stack-1.5.0/envs/ufs-env/install/modulefiles/Core module load stack-intel/19.1.1.217 module load stack-intel-mpi/2019.7.217 module load stack-python/3.9.12 module available -For ``spack-stack-1.4.1`` with GNU, load the following modules after loading miniconda, ecflow and mysql. +For ``spack-stack-1.5.0`` with GNU, load the following modules after loading miniconda, ecflow and mysql. Note that this is the full unified environment. .. code-block:: console - module use /glade/work/epicufsrt/contrib/spack-stack/cheyenne/spack-stack-1.4.1/envs/unified-env + module use /glade/work/epicufsrt/contrib/spack-stack/cheyenne/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/10.1.0 module load stack-openmpi/4.1.1 module load stack-python/3.9.12 @@ -400,7 +400,7 @@ For ``spack-stack-1.5.0`` with Intel, load the following modules after loading e NOAA Acorn (WCOSS2 test system) ------------------------------- -For spack-stack-1.4.1, the meta modules are in ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core``. +For spack-stack-1.5.0, the meta modules are in ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core``. On WCOSS2 OpenSUSE sets ``CONFIG_SITE`` which causes libraries to be installed in ``lib64``, breaking the ``lib`` assumption made by some packages. Therefore, ``CONFIG_SITE`` should be set to empty in ``compilers.yaml``. Also, don't use ``module purge`` on Acorn! @@ -509,10 +509,6 @@ For ``spack-stack-1.5.0`` with Intel, load the following modules after loading m module load stack-python/3.10.8 module -t available -.. note:: - - The recent update to ``spack-stack-dev-20230717`` was required on Gaea C5 due to a bug in the Intel compilers used in ``spack-stack-1.4.1``. - .. note:: On Gaea C5, running ``module available`` without the option ``-t`` leads to an error: ``/usr/bin/lua5.3: /opt/cray/pe/lmod/lmod/libexec/Spider.lua:568: stack overflow`` @@ -688,37 +684,20 @@ For ``spack-stack-1.5.0`` with Intel on the JCSDA ROMEX cluster (``c6i.32xlarge` Amazon Web Services Red Hat 8 ----------------------------- -Use a c6i.4xlarge instance or similar with AMI "skylab-5.0.0-redhat8" (ami-02324faac94a9cac9 in region us-east-1, ami-038d9beca351f9005 in region us-east-2). +Use a c6i.4xlarge instance or larger if running out of memory with AMI "skylab-6.0.0-redhat8" (ami-059d445a90ad8b792 in region us-east-1, ami-01fcf5d75ced5a046 in region us-east-2). -For ``spack-stack-1.4.0``, run: +For ``spack-stack-1.5.0``, run: .. code-block:: console ulimit -s unlimited scl enable gcc-toolset-11 bash - module use /home/ec2-user/spack-stack/spack-stack-1.4.0/envs/unified-env/install/modulefiles/Core + module use /home/ec2-user/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/11.2.1 module load stack-openmpi/4.1.5 module load stack-python/3.10.8 module available ------------------------------ -Amazon Web Services Ubuntu 20 ------------------------------ - -Use a c6i.4xlarge instance or similar with AMI "skylab-5.0.0-ubuntu20" (ami-09a8c9d3775feafcf in region us-east-1, ami-03e47cdb4ced34d7e in region us-east-2). - -For ``spack-stack-1.4.0``, run: - -.. code-block:: console - - ulimit -s unlimited - module use /home/ubuntu/spack-stack/spack-stack-1.4.0/envs/unified-env/install/modulefiles/Core - module load stack-gcc/10.3.0 - module load stack-openmpi/4.1.5 - module load stack-python/3.10.8 - module available - .. _Preconfigured_Sites_Tier2: ============================= @@ -732,7 +711,7 @@ TACC Frontera ------------------------------ .. note:: - ``spack-stack-1.4.0`` is currently not supported on this platform and may be added in the near future. + ``spack-stack-1.5.0`` is currently not supported on this platform. The following is required for building new spack environments and for using spack to build and run software. diff --git a/doc/source/conf.py b/doc/source/conf.py index adcce05d1..1b7b5e22d 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -24,9 +24,9 @@ author = 'Dominikus Heinzeller, Alexander Richert, Cameron Book' # The short X.Y version -version = '1.4' +version = '1.5' # The full version, including alpha/beta/rc tags -release = '1.4.1' +release = '1.5.0' numfig = True From 4d0e0306920bc6d4a20fe9a1536aa4c42057993e Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 28 Sep 2023 10:51:42 -0600 Subject: [PATCH 11/13] Update tag for spack submodule in .gitmodules and container recipes (#788) --- .gitmodules | 2 +- configs/containers/docker-ubuntu-clang-mpich.yaml | 2 +- configs/containers/docker-ubuntu-gcc-openmpi.yaml | 2 +- configs/containers/docker-ubuntu-intel-impi.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitmodules b/.gitmodules index 31d7a8707..72046cc5a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,7 +3,7 @@ #url = https://github.com/spack/spack #branch = develop url = https://github.com/jcsda/spack - branch = release/1.5.0 + branch = spack-stack-1.5.0 [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/configs/containers/docker-ubuntu-clang-mpich.yaml b/configs/containers/docker-ubuntu-clang-mpich.yaml index 950075bd5..58f536691 100644 --- a/configs/containers/docker-ubuntu-clang-mpich.yaml +++ b/configs/containers/docker-ubuntu-clang-mpich.yaml @@ -110,7 +110,7 @@ spack: os: ubuntu:20.04 spack: url: https://github.com/jcsda/spack - ref: release/1.5.0 + ref: spack-stack-1.5.0 resolve_sha: false # Whether or not to strip binaries diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index fe0c2777a..d4626b790 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -98,7 +98,7 @@ spack: os: ubuntu:20.04 spack: url: https://github.com/jcsda/spack - ref: release/1.5.0 + ref: spack-stack-1.5.0 resolve_sha: false # Whether or not to strip binaries diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 522d9debe..37b1bb131 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -115,7 +115,7 @@ spack: os: ubuntu:20.04 spack: url: https://github.com/jcsda/spack - ref: release/1.5.0 + ref: spack-stack-1.5.0 resolve_sha: false # Whether or not to strip binaries From e7590f88a80ba081e58f1f7e9d08688b47ece6c1 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 28 Sep 2023 13:22:30 -0600 Subject: [PATCH 12/13] Update .gitmodules and submodule pointer for spack for code review and testing --- .gitmodules | 6 ++++-- spack | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index b1f475b64..6f7aad774 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,8 +2,10 @@ path = spack #url = https://github.com/spack/spack #branch = develop - url = https://github.com/jcsda/spack - branch = jcsda_emc_spack_stack + #url = https://github.com/jcsda/spack + #branch = jcsda_emc_spack_stack + url = https://github.com/climbfuji/spack + branch = feature/merge_rel150_into_default [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index e8037bcaf..8672dc2b0 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit e8037bcaf26fe80a24e72a14bfcdb5495e506c6f +Subproject commit 8672dc2b0c53dee1babb29593f038a5b2724a1b9 From 2069340cb70adfa7dcf4bcf24910fcf85c75c62d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 28 Sep 2023 15:24:15 -0600 Subject: [PATCH 13/13] Revert .gitmodules and update submodule pointer for spack --- .gitmodules | 6 ++---- spack | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index 6f7aad774..b1f475b64 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,10 +2,8 @@ path = spack #url = https://github.com/spack/spack #branch = develop - #url = https://github.com/jcsda/spack - #branch = jcsda_emc_spack_stack - url = https://github.com/climbfuji/spack - branch = feature/merge_rel150_into_default + url = https://github.com/jcsda/spack + branch = jcsda_emc_spack_stack [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index 8672dc2b0..a90e85256 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 8672dc2b0c53dee1babb29593f038a5b2724a1b9 +Subproject commit a90e85256d58b3e9ebc920f8cf102188984eced0