From 7db451741049ebbf5ec5b320a8253e802e2ad4c5 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 15 Oct 2024 11:58:59 -0600 Subject: [PATCH 01/33] Remove latex documentation and fix two formatting errors in doc/source/NewSiteConfigs.rst (#1346) --- doc/CMakeLists.txt | 58 ++++++----------------------------- doc/requirements.txt | 1 - doc/source/NewSiteConfigs.rst | 4 ++- doc/source/conf.py | 36 +--------------------- 4 files changed, 13 insertions(+), 86 deletions(-) diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 83acb6fc6..0d21c6cf6 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -1,15 +1,9 @@ cmake_minimum_required(VERSION 3.18 FATAL_ERROR) -project(spack-stack VERSION 1.3.1) +project(spack-stack VERSION 1.9.0) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules/Modules") -option(SPHINX_OUTPUT_HTML "Build HTML documentation" ON) -option(SPHINX_OUTPUT_LATEX "Build LaTeX (PDF) documentation" OFF) - -message(STATUS "SPHINX_OUTPUT_HTML: ${SPHINX_OUTPUT_HTML}") -message(STATUS "SPHINX_OUTPUT_LATEX: ${SPHINX_OUTPUT_LATEX}") - find_package(Sphinx REQUIRED) # configured documentation tools and intermediate build results @@ -20,51 +14,17 @@ set(SPHINX_CACHE_DIR "${CMAKE_CURRENT_BINARY_DIR}/_doctrees") # HTML output directory set(SPHINX_HTML_DIR "${CMAKE_CURRENT_BINARY_DIR}/html") - -# LaTeX output directory -set(SPHINX_PDF_DIR "${CMAKE_CURRENT_BINARY_DIR}/pdf") configure_file( "${CMAKE_CURRENT_SOURCE_DIR}/source/conf.py" "${BINARY_BUILD_DIR}/conf.py" @ONLY) -if(SPHINX_OUTPUT_HTML) - add_custom_target(spack-stack-doc-html ALL - ${SPHINX_EXECUTABLE} - -q -b html - -c "${BINARY_BUILD_DIR}" - -d "${SPHINX_CACHE_DIR}" - "${CMAKE_CURRENT_SOURCE_DIR}/source" - "${SPHINX_HTML_DIR}" - COMMENT "Building HTML documentation with Sphinx") -endif() - -if(SPHINX_OUTPUT_LATEX) - find_package(LATEX COMPONENTS PDFLATEX REQUIRED) - set(PDF_OUTPUT_FILE "${SPHINX_PDF_DIR}/spack-stack.pdf") - # Create LaTeX input file with Sphinx - add_custom_command(OUTPUT "${SPHINX_PDF_DIR}" - COMMAND "${SPHINX_EXECUTABLE}" -q -b latex - -c "${BINARY_BUILD_DIR}" - -d "${SPHINX_CACHE_DIR}" - "${CMAKE_CURRENT_SOURCE_DIR}/source" - "${SPHINX_PDF_DIR}" - COMMENT "Running Sphinx to generate documentation (LaTeX)" - VERBATIM - ) - # Create PDF output file with pdflatex - add_custom_command(OUTPUT "${PDF_OUTPUT_FILE}" - COMMAND "${CMAKE_MAKE_PROGRAM}" all-pdf - WORKING_DIRECTORY "${SPHINX_PDF_DIR}" - DEPENDS "${SPHINX_PDF_DIR}" - COMMENT "Generating PDF version of documentation" - VERBATIM - ) - # - add_custom_target(spack-stack-doc-pdf ALL - COMMENT "Generating PDF version of documentation ..." - SOURCES "${PDF_OUTPUT_FILE}" - VERBATIM - ) -endif() +add_custom_target(spack-stack-doc-html ALL + ${SPHINX_EXECUTABLE} + -q -b html + -c "${BINARY_BUILD_DIR}" + -d "${SPHINX_CACHE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}/source" + "${SPHINX_HTML_DIR}" + COMMENT "Building HTML documentation with Sphinx") diff --git a/doc/requirements.txt b/doc/requirements.txt index 25b341bdf..e4d7905c5 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,2 +1 @@ -sphinxcontrib-bibtex urllib3<2 diff --git a/doc/source/NewSiteConfigs.rst b/doc/source/NewSiteConfigs.rst index 65b59b6b0..4866137d4 100644 --- a/doc/source/NewSiteConfigs.rst +++ b/doc/source/NewSiteConfigs.rst @@ -175,7 +175,8 @@ These instructions are meant to be a reference that users can follow to set up t Make sure you upgrade cmake in homebrew. .. code-block:: console - brew upgrade cmake + + brew upgrade cmake 4. Configure your terminal to use the homebrew installed bash @@ -671,6 +672,7 @@ With all of that in mind, the following instructions were used on an Amazon Web 3. Load the correct module shipped with ``nvhpc-24-3``. Note that this is only required for ``spack`` to detect the compiler and ``openmpi`` library during the environment configuration below. It is not required when using the new environment to compile code. .. code-block:: console + module purge module use /opt/nvidia/hpc_sdk/modulefiles module load nvhpc-openmpi3/24.3 diff --git a/doc/source/conf.py b/doc/source/conf.py index 8f33a6da8..d48e7e8aa 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -51,12 +51,9 @@ 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode', 'sphinx.ext.githubpages', - 'sphinx.ext.napoleon', - 'sphinxcontrib.bibtex' + 'sphinx.ext.napoleon' ] -bibtex_bibfiles = [] - # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -124,37 +121,6 @@ htmlhelp_basename = 'spack-stack' -# -- Options for LaTeX output ------------------------------------------------ - -latex_engine = 'pdflatex' -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', - 'maketitle': r'\newcommand\sphinxbackoftitlepage{For referencing this document please use: \newline \break Heinzeller, D., A. Richert, C. Book, E. Hartnett, H. Lei, N. Perlin, R. Vasic, S. Herbener, 2024. spack-stack documentation develop. Available at https://spack-stack.readthedocs.io/\textunderscore/downloads/en/latest/pdf/.}\sphinxmaketitle' -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, 'spack-stack.tex', 'spack-stack documentation', - author,'manual'), -] - - # -- Options for manual page output ------------------------------------------ # One entry per manual page. List of tuples From 2b363ea215f7c15d2abb18d1c107d3f7f19f6fa4 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 16 Oct 2024 15:34:32 -0600 Subject: [PATCH 02/33] Move NEPTUNE Python dependencies into its their own package neptune-python-env, remove legacy environment variables for NEPTUNE (#1343) This PR moves the full NEPTUNE Python dependencies into their own package neptune-python-dev. Only the basic dependencies on f90nml and python-dateutil remain in neptune-env (needed by NUOPC). The espc variant is turned off by default in neptune-env, but the neptune-dev template enables it. Likewise, the xnrl variant is turned off by default in neptune-python-env, but the neptune-dev template enables it. Several legacy environment variables for hdf5, libyaml, p4est are removed from configs/common/modules_*.yaml. --- configs/common/modules_lmod.yaml | 16 -------- configs/common/modules_tcl.yaml | 16 -------- configs/templates/neptune-dev/spack.yaml | 3 +- configs/templates/skylab-dev/spack.yaml | 3 +- configs/templates/unified-dev/spack.yaml | 3 +- .../packages/neptune-env/package.py | 33 ++++----------- .../packages/neptune-python-env/package.py | 41 +++++++++++++++++++ 7 files changed, 54 insertions(+), 61 deletions(-) create mode 100644 spack-ext/repos/spack-stack/packages/neptune-python-env/package.py diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml index 41bfa24df..7c994ce45 100644 --- a/configs/common/modules_lmod.yaml +++ b/configs/common/modules_lmod.yaml @@ -112,20 +112,10 @@ modules: environment: set: 'ESMFMKFILE': '{prefix}/lib/esmf.mk' - hdf5: - environment: - set: - 'HDF5_DIR': '{prefix}' libpng: environment: set: 'PNG_ROOT': '{prefix}' - libyaml: - environment: - set: - 'YAML_DIR': '{prefix}' - 'YAML_LIB': '{prefix}/lib' - 'YAML_INC': '{prefix}/include' madis: environment: set: @@ -151,12 +141,6 @@ modules: set: 'OMPI_MCA_rmaps_base_oversubscribe': '1' 'PRTE_MCA_rmaps_default_mapping_policy': ':oversubscribe' - p4est: - environment: - set: - 'P4EST_API_DIR': '{prefix}' - 'P4EST_API_LIB': '{prefix}/lib' - 'P4EST_API_INC': '{prefix}/include' bacio: environment: set: diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml index 5ce83e8b4..551c12e65 100644 --- a/configs/common/modules_tcl.yaml +++ b/configs/common/modules_tcl.yaml @@ -114,20 +114,10 @@ modules: environment: set: 'ESMFMKFILE': '{prefix}/lib/esmf.mk' - hdf5: - environment: - set: - 'HDF5_DIR': '{prefix}' libpng: environment: set: 'PNG_ROOT': '{prefix}' - libyaml: - environment: - set: - 'YAML_DIR': '{prefix}' - 'YAML_LIB': '{prefix}/lib' - 'YAML_INC': '{prefix}/include' madis: environment: set: @@ -153,12 +143,6 @@ modules: set: 'OMPI_MCA_rmaps_base_oversubscribe': '1' 'PRTE_MCA_rmaps_default_mapping_policy': ':oversubscribe' - p4est: - environment: - set: - 'P4EST_API_DIR': '{prefix}' - 'P4EST_API_LIB': '{prefix}/lib' - 'P4EST_API_INC': '{prefix}/include' bacio: environment: set: diff --git a/configs/templates/neptune-dev/spack.yaml b/configs/templates/neptune-dev/spack.yaml index 5f3332f79..564cf6d86 100644 --- a/configs/templates/neptune-dev/spack.yaml +++ b/configs/templates/neptune-dev/spack.yaml @@ -8,7 +8,8 @@ spack: definitions: - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel', '%oneapi'] - packages: - - neptune-env +espc +python +xnrl ^esmf@8.7.0b11 snapshot=b11 + - neptune-env +espc ^esmf@8.7.0b11 snapshot=b11 + - neptune-python-env +xnrl ^esmf@8.7.0b11 snapshot=b11 specs: - matrix: diff --git a/configs/templates/skylab-dev/spack.yaml b/configs/templates/skylab-dev/spack.yaml index 1ba272a63..4616138ca 100644 --- a/configs/templates/skylab-dev/spack.yaml +++ b/configs/templates/skylab-dev/spack.yaml @@ -17,7 +17,8 @@ spack: - jedi-neptune-env ^esmf@=8.7.0b11 snapshot=b11 - jedi-ufs-env ^esmf@=8.6.1 - jedi-um-env - - neptune-env ~espc +python ~xnrl ^esmf@=8.7.0b11 snapshot=b11 + - neptune-env ^esmf@=8.7.0b11 snapshot=b11 + - neptune-python-env ^esmf@=8.7.0b11 snapshot=b11 - soca-env # Various crtm tags (list all to avoid duplicate packages) diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index a05ed4a75..608b4462a 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -21,7 +21,8 @@ spack: - jedi-tools-env - jedi-ufs-env ^esmf@=8.6.1 - jedi-um-env - - neptune-env ~espc +python ~xnrl ^esmf@=8.7.0b11 snapshot=b11 + - neptune-env ^esmf@=8.7.0b11 snapshot=b11 + - neptune-python-env ^esmf@=8.7.0b11 snapshot=b11 - soca-env - ufs-srw-app-env ^esmf@=8.6.1 - ufs-weather-model-env ^esmf@=8.6.1 diff --git a/spack-ext/repos/spack-stack/packages/neptune-env/package.py b/spack-ext/repos/spack-stack/packages/neptune-env/package.py index f79968eac..ddd732692 100644 --- a/spack-ext/repos/spack-stack/packages/neptune-env/package.py +++ b/spack-ext/repos/spack-stack/packages/neptune-env/package.py @@ -1,4 +1,4 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) @@ -9,7 +9,7 @@ class NeptuneEnv(BundlePackage): - """Development environment for neptune standalone""" + """Development environment for NEPTUNE standalone""" # Fake URL homepage = "https://github.com/notavalidaccount/neptune" @@ -17,11 +17,9 @@ class NeptuneEnv(BundlePackage): maintainers("climbfuji", "areinecke") - version("1.4.0") + version("1.5.0") - variant("python", default=True, description="Build Python dependencies") - variant("espc", default=True, description="Build ESPC dependencies") - variant("xnrl", default=True, description="Build XNRL and its extra Python dependencies") + variant("espc", default=False, description="Build ESPC dependencies") depends_on("base-env", type="run") @@ -33,7 +31,6 @@ class NeptuneEnv(BundlePackage): depends_on("libyaml", type="run") depends_on("p4est", type="run") depends_on("w3emc", type="run") - depends_on("w3nco", type="run") depends_on("sp", type="run", when="%aocc") depends_on("ip@5:", type="run", when="%apple-clang") depends_on("ip@5:", type="run", when="%gcc") @@ -43,28 +40,12 @@ class NeptuneEnv(BundlePackage): depends_on("nco", type="run") depends_on("mct", type="run") - conflicts("+xnrl", when="~python", msg="Variant xnrl requires variant python") - with when("+espc"): depends_on("fftw", type="build") depends_on("netlib-lapack", type="build") - with when("+python"): - depends_on("py-f90nml", type="run") - depends_on("py-h5py", type="run") - depends_on("py-netcdf4", type="run") - depends_on("py-pandas", type="run") - depends_on("py-pycodestyle", type="run") - depends_on("py-pybind11", type="run") - depends_on("py-pyhdf", type="run") - depends_on("py-python-dateutil", type="run") - depends_on("py-pyyaml", type="run") - depends_on("py-scipy", type="run") - depends_on("py-xarray", type="run") - depends_on("py-pytest", type="run") - depends_on("py-fortranformat", type="run") - - with when("+xnrl"): - depends_on("py-xnrl", type="run") + # Basic Python dependencies that are always needed + depends_on("py-f90nml", type="run") + depends_on("py-python-dateutil", type="run") # There is no need for install() since there is no code. diff --git a/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py b/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py new file mode 100644 index 000000000..5b6f243b8 --- /dev/null +++ b/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py @@ -0,0 +1,41 @@ +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import sys + +from spack.package import * + + +class NeptunePythonEnv(BundlePackage): + """Development environment for NEPTUNE standalone with all Python dependencies""" + + # Fake URL + homepage = "https://github.com/notavalidaccount/neptune" + git = "https://github.com/notavalidaccount/neptune.git" + + maintainers("climbfuji", "areinecke") + + version("1.5.0") + + variant("xnrl", default=False, description="Build non-pulic XNRL") + + depends_on("neptune-env", type="run") + + depends_on("py-h5py", type="run") + depends_on("py-netcdf4", type="run") + depends_on("py-pandas", type="run") + depends_on("py-pycodestyle", type="run") + depends_on("py-pybind11", type="run") + depends_on("py-pyhdf", type="run") + depends_on("py-pyyaml", type="run") + depends_on("py-scipy", type="run") + depends_on("py-xarray", type="run") + depends_on("py-pytest", type="run") + depends_on("py-fortranformat", type="run") + + with when("+xnrl"): + depends_on("py-xnrl", type="run") + + # There is no need for install() since there is no code. From 7538aa0df0898ee737e4dac4a70415610a048a01 Mon Sep 17 00:00:00 2001 From: RatkoVasic-NOAA <37597874+RatkoVasic-NOAA@users.noreply.github.com> Date: Fri, 18 Oct 2024 17:10:08 -0400 Subject: [PATCH 03/33] Update orion/hercules new installation directory (#1337) (#1351) Placed spack-stack-1.8.0 to the new directory --- doc/source/PreConfiguredSites.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index b62804dd6..138fa604c 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -43,9 +43,9 @@ Pre-configured sites (tier 1) +=====================+=======================+====================+========================================================+=================+ | **HPC platforms** | +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ -| | Hercules | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/hercules/`` | EPIC / JCSDA | +| | Hercules | GCC, Intel | ``/apps/contrib/spack-stack/`` | EPIC / JCSDA | | MSU +-----------------------+--------------------+--------------------------------------------------------+-----------------+ -| | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/orion/`` | EPIC / JCSDA | +| | Orion | GCC, Intel | ``/apps/contrib/spack-stack/`` | EPIC / JCSDA | +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ | | Discover SCU16 | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/scu16/`` | JCSDA | | NASA +-----------------------+--------------------+--------------------------------------------------------+-----------------+ @@ -88,6 +88,8 @@ The following is required for building new spack environments with any supported .. code-block:: console + # To access /apps/contrib/spack-stack directory/, first login to orion-devel-1 or orion-devel-2 login node. + # Then sudo to role-epic account. module purge @@ -101,8 +103,9 @@ The following is required for building new spack environments with any supported .. code-block:: console + # To access /apps/contrib/spack-stack directory/, first login to orion-devel-1 or orion-devel-2 login node. + # Then sudo to role-epic account. module purge - # No need to load modules, spack-stack-1.8.0 have no dependences .. _Preconfigured_Sites_Discover_SCU16: From 55967c1686e4230a723f433e16b01df2eb166017 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 21 Oct 2024 11:52:17 -0600 Subject: [PATCH 04/33] Add oneAPI config for Narwhal, add GNU config for Atlantis, remove Sandy platform; make `python` variant for `esmf` configurable (#1318) - Add a oneapi@2024.2.1 config for Narwhal (using icx, icpx, ifort) - Add a gcc@11.2.0 config for Atlantis - Remove Sandy as a supported platform - Update submodule pointer for spack to bring in esmf@8.7.0 (official release) and replace esmf@8.7.0b11 with esmf@8.7.0 in all configs/** - Make python variant for esmf configurable --- configs/common/modules_lmod.yaml | 2 + configs/common/modules_tcl.yaml | 2 + configs/common/packages | 0 configs/common/packages.yaml | 24 ++-- .../tier1/{sandy => atlantis}/mirrors.yaml | 4 +- .../sites/tier1/atlantis/packages_gcc.yaml | 13 ++ .../sites/tier1/atlantis/packages_intel.yaml | 2 +- configs/sites/tier1/narwhal/compilers.yaml | 24 ++++ configs/sites/tier1/narwhal/mirrors.yaml | 4 +- .../sites/tier1/narwhal/packages_oneapi.yaml | 27 +++++ configs/sites/tier1/nautilus/mirrors.yaml | 4 +- configs/sites/tier1/sandy/compilers.yaml | 14 --- configs/sites/tier1/sandy/config.yaml | 2 - configs/sites/tier1/sandy/modules.yaml | 8 -- configs/sites/tier1/sandy/packages.yaml | 113 ------------------ configs/templates/neptune-dev/spack.yaml | 4 +- configs/templates/skylab-dev/spack.yaml | 8 +- configs/templates/unified-dev/spack.yaml | 8 +- doc/source/PreConfiguredSites.rst | 26 +++- spack | 2 +- .../packages/neptune-env/package.py | 1 + .../packages/neptune-python-env/package.py | 2 + 22 files changed, 127 insertions(+), 167 deletions(-) create mode 100644 configs/common/packages rename configs/sites/tier1/{sandy => atlantis}/mirrors.yaml (67%) create mode 100644 configs/sites/tier1/atlantis/packages_gcc.yaml create mode 100644 configs/sites/tier1/narwhal/packages_oneapi.yaml delete mode 100644 configs/sites/tier1/sandy/compilers.yaml delete mode 100644 configs/sites/tier1/sandy/config.yaml delete mode 100644 configs/sites/tier1/sandy/modules.yaml delete mode 100644 configs/sites/tier1/sandy/packages.yaml diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml index 7c994ce45..8fe637fb6 100644 --- a/configs/common/modules_lmod.yaml +++ b/configs/common/modules_lmod.yaml @@ -136,6 +136,8 @@ modules: ^esmf@8.7.0b04+debug snapshot=b04: 'esmf-8.7.0b04-debug' ^esmf@8.7.0b11~debug snapshot=b11: 'esmf-8.7.0b11' ^esmf@8.7.0b11+debug snapshot=b11: 'esmf-8.7.0b11-debug' + ^esmf@8.7.0~debug snapshot=none: 'esmf-8.7.0' + ^esmf@8.7.0+debug snapshot=none: 'esmf-8.7.0-debug' openmpi: environment: set: diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml index 551c12e65..3275aee44 100644 --- a/configs/common/modules_tcl.yaml +++ b/configs/common/modules_tcl.yaml @@ -138,6 +138,8 @@ modules: ^esmf@8.7.0b04+debug snapshot=b04: 'esmf-8.7.0b04-debug' ^esmf@8.7.0b11~debug snapshot=b11: 'esmf-8.7.0b11' ^esmf@8.7.0b11+debug snapshot=b11: 'esmf-8.7.0b11-debug' + ^esmf@8.7.0~debug snapshot=none: 'esmf-8.7.0' + ^esmf@8.7.0+debug snapshot=none: 'esmf-8.7.0-debug' openmpi: environment: set: diff --git a/configs/common/packages b/configs/common/packages new file mode 100644 index 000000000..e69de29bb diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index fc7089741..99f17b060 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -54,17 +54,19 @@ packages: # Also, check the acorn and derecho site configs which have esmf modifications. esmf: require: - - '~xerces ~pnetcdf +shared +external-parallelio +python' - - any_of: ['@=8.6.1 snapshot=none', '@=8.7.0b11 snapshot=b11'] - - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"'] - when: "%intel" - message: "Extra ESMF compile options for Intel" - #- any_of: [''] - # when: "%gcc" - # message: "Extra ESMF compile options for GCC" - #- any_of: [''] - # when: "%apple-clang" - # message: "Extra ESMF compile options for GCC" + - '~xerces ~pnetcdf +shared +external-parallelio' + - any_of: ['@=8.6.1 snapshot=none', '@=8.7.0 snapshot=none'] + - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"'] + when: "%intel" + message: "Extra ESMF compile options for Intel" + #- any_of: [''] + # when: "%gcc" + # message: "Extra ESMF compile options for GCC" + #- any_of: [''] + # when: "%apple-clang" + # message: "Extra ESMF compile options for GCC" + prefer: + - '+python' fckit: require: '@0.11.0 +eckit' fftw: diff --git a/configs/sites/tier1/sandy/mirrors.yaml b/configs/sites/tier1/atlantis/mirrors.yaml similarity index 67% rename from configs/sites/tier1/sandy/mirrors.yaml rename to configs/sites/tier1/atlantis/mirrors.yaml index c48f33bf1..4d363e009 100644 --- a/configs/sites/tier1/sandy/mirrors.yaml +++ b/configs/sites/tier1/atlantis/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: local-source: fetch: - url: file:///gpfs/fs1/neptune/spack-stack/source-cache + url: file:///neptune_diagnostics/spack-stack/source-cache/ access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///gpfs/fs1/neptune/spack-stack/source-cache + url: file:///neptune_diagnostics/spack-stack/source-cache/ access_pair: - null - null diff --git a/configs/sites/tier1/atlantis/packages_gcc.yaml b/configs/sites/tier1/atlantis/packages_gcc.yaml new file mode 100644 index 000000000..bc10f8b75 --- /dev/null +++ b/configs/sites/tier1/atlantis/packages_gcc.yaml @@ -0,0 +1,13 @@ +packages: + all: + compiler:: [gcc@11.2.0] + providers: + mpi:: [openmpi@4.1.5] + mpi: + buildable: False + openmpi: + buildable: False + externals: + - spec: openmpi@4.1.5%gcc@=11.2.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath fabrics=ucx schedulers=slurm + modules: + - openmpi/mlnx/gcc/64/4.1.5a1 diff --git a/configs/sites/tier1/atlantis/packages_intel.yaml b/configs/sites/tier1/atlantis/packages_intel.yaml index ea2f4de32..4271f3f10 100644 --- a/configs/sites/tier1/atlantis/packages_intel.yaml +++ b/configs/sites/tier1/atlantis/packages_intel.yaml @@ -15,6 +15,6 @@ packages: intel-oneapi-mkl: externals: - spec: intel-oneapi-mkl@2022.1.0 + prefix: /cm/shared/apps/intel/oneapi modules: - mkl/2022.1.0 - prefix: /cm/shared/apps/intel/oneapi diff --git a/configs/sites/tier1/narwhal/compilers.yaml b/configs/sites/tier1/narwhal/compilers.yaml index 1b7c23dcf..838a8a1dd 100644 --- a/configs/sites/tier1/narwhal/compilers.yaml +++ b/configs/sites/tier1/narwhal/compilers.yaml @@ -21,6 +21,30 @@ compilers:: set: CRAYPE_LINK_TYPE: 'dynamic' extra_rpaths: [] + - compiler: + spec: oneapi@2024.2.0 + paths: + cc: cc + cxx: CC + f77: ftn + fc: ftn + flags: {} + operating_system: sles15 + modules: + - PrgEnv-intel/8.4.0 + - intel/2024.2 + - cray-libsci/23.05.1.4 + - libfabric/1.12.1.2.2.1 + environment: + prepend_path: + PATH: '/opt/cray/pe/gcc/10.3.0/snos/bin' + CPATH: '/opt/cray/pe/gcc/10.3.0/snos/include' + LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/23.05.1.4/INTEL/2022.2/x86_64/lib:/opt/cray/pe/gcc/10.3.0/snos/lib:/opt/cray/pe/gcc/10.3.0/snos/lib64' + append_path: + CPATH: '/opt/intel/oneapi_2024.2.0.634/compiler/2024.2/opt/compiler/include/intel64' + set: + CRAYPE_LINK_TYPE: 'dynamic' + extra_rpaths: [] - compiler: spec: gcc@10.3.0 paths: diff --git a/configs/sites/tier1/narwhal/mirrors.yaml b/configs/sites/tier1/narwhal/mirrors.yaml index 709f9096c..ec3db4a00 100644 --- a/configs/sites/tier1/narwhal/mirrors.yaml +++ b/configs/sites/tier1/narwhal/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: local-source: fetch: - url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache + url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache + url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache access_pair: - null - null diff --git a/configs/sites/tier1/narwhal/packages_oneapi.yaml b/configs/sites/tier1/narwhal/packages_oneapi.yaml new file mode 100644 index 000000000..3bcf99f38 --- /dev/null +++ b/configs/sites/tier1/narwhal/packages_oneapi.yaml @@ -0,0 +1,27 @@ +packages: + all: + compiler:: [oneapi@2024.2.0,gcc@10.3.0] + providers: + mpi:: [cray-mpich@8.1.21] + mpi: + buildable: False + cray-mpich: + externals: + - spec: cray-mpich@8.1.21%oneapi@2024.2.0 ~wrappers + modules: + - cray-mpich-ucx/8.1.21 + - craype-network-ucx + intel-oneapi-mkl: + externals: + - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.0 + prefix: /opt/intel/oneapi_2024.2.0.634 + intel-oneapi-tbb: + externals: + - spec: intel-oneapi-tbb@2021.13%oneapi@2024.2.0 + prefix: /opt/intel/oneapi_2024.2.0.634 + modules: + - tbb/2021.13 + intel-oneapi-runtime: + externals: + - spec: intel-oneapi-runtime@2024.2.0%oneapi@2024.2.0 + prefix: /opt/intel/oneapi_2024.2.0.634 diff --git a/configs/sites/tier1/nautilus/mirrors.yaml b/configs/sites/tier1/nautilus/mirrors.yaml index 709f9096c..ec3db4a00 100644 --- a/configs/sites/tier1/nautilus/mirrors.yaml +++ b/configs/sites/tier1/nautilus/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: local-source: fetch: - url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache + url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache + url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache access_pair: - null - null diff --git a/configs/sites/tier1/sandy/compilers.yaml b/configs/sites/tier1/sandy/compilers.yaml deleted file mode 100644 index b86ff0a33..000000000 --- a/configs/sites/tier1/sandy/compilers.yaml +++ /dev/null @@ -1,14 +0,0 @@ -compilers: -- compiler: - spec: gcc@=9.3.0 - paths: - cc: /software/depot/gcc-9.3.0/bin/gcc - cxx: /software/depot/gcc-9.3.0/bin/g++ - f77: /software/depot/gcc-9.3.0/bin/gfortran - fc: /software/depot/gcc-9.3.0/bin/gfortran - flags: {} - operating_system: centos7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] diff --git a/configs/sites/tier1/sandy/config.yaml b/configs/sites/tier1/sandy/config.yaml deleted file mode 100644 index 84fe05ab9..000000000 --- a/configs/sites/tier1/sandy/config.yaml +++ /dev/null @@ -1,2 +0,0 @@ -config: - build_jobs: 8 diff --git a/configs/sites/tier1/sandy/modules.yaml b/configs/sites/tier1/sandy/modules.yaml deleted file mode 100644 index aeb254c2e..000000000 --- a/configs/sites/tier1/sandy/modules.yaml +++ /dev/null @@ -1,8 +0,0 @@ -modules: - default: - enable:: - - tcl - tcl: - include: - # List of packages for which we need modules that are blacklisted by default - - python diff --git a/configs/sites/tier1/sandy/packages.yaml b/configs/sites/tier1/sandy/packages.yaml deleted file mode 100644 index 1af1ab029..000000000 --- a/configs/sites/tier1/sandy/packages.yaml +++ /dev/null @@ -1,113 +0,0 @@ -packages: - all: - compiler:: [gcc@9.3.0] - providers: - mpi:: [openmpi@4.0.5] - -### MPI, Python, MKL - mpi: - buildable: False - openmpi: - externals: - - spec: openmpi@4.0.5%gcc@9.3.0 - prefix: /software7/depot/openmpi-4.0.5 - -### All other external packages listed alphabetically - autoconf: - externals: - - spec: autoconf@2.69 - prefix: /usr - automake: - externals: - - spec: automake@1.16.1 - prefix: /usr - binutils: - externals: - - spec: binutils@2.30.117 - prefix: /usr - bison: - externals: - - spec: bison@3.0.4 - prefix: /usr - coreutils: - externals: - - spec: coreutils@8.30 - prefix: /usr - curl: - externals: - - spec: curl@7.61.1 - prefix: /usr - diffutils: - externals: - - spec: diffutils@3.6 - prefix: /usr - findutils: - externals: - - spec: findutils@4.6.0 - prefix: /usr - flex: - externals: - - spec: flex@2.6.1+lex - prefix: /usr - gawk: - externals: - - spec: gawk@4.2.1 - prefix: /usr - git: - externals: - - spec: git@2.39.2+tcltk - prefix: /software8/depot/git-2.39.2 - git-lfs: - externals: - - spec: git-lfs@3.3.0 - prefix: /software8/depot/git-2.39.2 - gmake: - externals: - - spec: gmake@4.2.1 - prefix: /usr - groff: - externals: - - spec: groff@1.22.3 - prefix: /usr - libtool: - externals: - - spec: libtool@2.4.6 - prefix: /usr - m4: - externals: - - spec: m4@1.4.18 - prefix: /usr - openssh: - externals: - - spec: openssh@8.0p1 - prefix: /usr - # Don't use external openssl, too old - #openssl: - # externals: - # - spec: openssl@1.1.1k - # prefix: /usr - # Can't use with py-xnrl - #perl: - # externals: - # - spec: perl@5.26.3~cpanm+shared+threads - # prefix: /usr - pkgconf: - externals: - - spec: pkgconf@1.4.2 - prefix: /usr - subversion: - externals: - - spec: subversion@1.10.2 - prefix: /usr - tar: - externals: - - spec: tar@1.30 - prefix: /usr - texinfo: - externals: - - spec: texinfo@6.5 - prefix: /usr - wget: - externals: - - spec: wget@1.19.5 - prefix: /usr diff --git a/configs/templates/neptune-dev/spack.yaml b/configs/templates/neptune-dev/spack.yaml index 564cf6d86..9a9e3dfec 100644 --- a/configs/templates/neptune-dev/spack.yaml +++ b/configs/templates/neptune-dev/spack.yaml @@ -8,8 +8,8 @@ spack: definitions: - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel', '%oneapi'] - packages: - - neptune-env +espc ^esmf@8.7.0b11 snapshot=b11 - - neptune-python-env +xnrl ^esmf@8.7.0b11 snapshot=b11 + - neptune-env +espc ^esmf@8.7.0 + - neptune-python-env +xnrl ^esmf@8.7.0 specs: - matrix: diff --git a/configs/templates/skylab-dev/spack.yaml b/configs/templates/skylab-dev/spack.yaml index 4616138ca..be901ec7e 100644 --- a/configs/templates/skylab-dev/spack.yaml +++ b/configs/templates/skylab-dev/spack.yaml @@ -14,11 +14,11 @@ spack: - jedi-fv3-env - jedi-geos-env ^esmf@=8.6.1 - jedi-mpas-env - - jedi-neptune-env ^esmf@=8.7.0b11 snapshot=b11 + - jedi-neptune-env ^esmf@=8.7.0 - jedi-ufs-env ^esmf@=8.6.1 - jedi-um-env - - neptune-env ^esmf@=8.7.0b11 snapshot=b11 - - neptune-python-env ^esmf@=8.7.0b11 snapshot=b11 + - neptune-env ^esmf@=8.7.0 + - neptune-python-env ^esmf@=8.7.0 - soca-env # Various crtm tags (list all to avoid duplicate packages) @@ -27,7 +27,7 @@ spack: # Various esmf tags (list all to avoid duplicate packages) - esmf@=8.6.1 snapshot=none - - esmf@=8.7.0b11 snapshot=b11 + - esmf@=8.7.0 snapshot=none specs: - matrix: diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index 608b4462a..370a03e7c 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -17,12 +17,12 @@ spack: - jedi-fv3-env - jedi-geos-env ^esmf@=8.6.1 - jedi-mpas-env - - jedi-neptune-env ^esmf@=8.7.0b11 snapshot=b11 + - jedi-neptune-env ^esmf@=8.7.0 - jedi-tools-env - jedi-ufs-env ^esmf@=8.6.1 - jedi-um-env - - neptune-env ^esmf@=8.7.0b11 snapshot=b11 - - neptune-python-env ^esmf@=8.7.0b11 snapshot=b11 + - neptune-env ^esmf@=8.7.0 + - neptune-python-env ^esmf@=8.7.0 - soca-env - ufs-srw-app-env ^esmf@=8.6.1 - ufs-weather-model-env ^esmf@=8.6.1 @@ -33,7 +33,7 @@ spack: # Various esmf tags (list all to avoid duplicate packages) - esmf@=8.6.1 snapshot=none - - esmf@=8.7.0b11 snapshot=b11 + - esmf@=8.7.0 snapshot=none # MADIS for WCOSS2 decoders. - madis@4.5 diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 138fa604c..1fcf8ab7b 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -65,7 +65,7 @@ Pre-configured sites (tier 1) +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ | | Narwhal | GCC, Intel, oneAPI | ``/p/app/projects/NEPTUNE/spack-stack/`` | NRL | | U.S. Navy (HPCMP) +-----------------------+--------------------+--------------------------------------------------------+-----------------+ -| | Nautilus | Intel | ``/p/app/projects/NEPTUNE/spack-stack/`` | NRL | +| | Nautilus | GCC, Intel, oneAPI | ``/p/app/projects/NEPTUNE/spack-stack/`` | NRL | +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ | Univ. of Wisconsin | S4 | Intel | ``/data/prod/jedi/spack-stack/`` | JCSDA | +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ @@ -167,6 +167,30 @@ The following is required for building new spack environments with Intel on this module unload cray-libsci module load cray-libsci/23.05.1.4 +The following is required for building new spack environments with Intel oneAPI on this platform.. Don't use ``module purge`` on Narwhal! + +.. code-block:: console + + umask 0022 + module unload PrgEnv-cray + module load PrgEnv-intel/8.4.0 + module unload intel + module load intel/2024.2 + module unload cray-mpich + module unload craype-network-ofi + # Warning. Do not load craype-network-ucx + # or cray-mpich-ucx/8.1.21! + # There is a bug in the modulefile that prevents + # spack from setting the environment for its + # build steps when the module is already + # loaded. Instead, let spack load it when the + # package requires it. + #module load craype-network-ucx + #module load cray-mpich-ucx/8.1.21 + module load libfabric/1.12.1.2.2.1 + module unload cray-libsci + module load cray-libsci/23.05.1.4 + The following is required for building new spack environments with GNU on this platform.. Don't use ``module purge`` on Narwhal! .. code-block:: console diff --git a/spack b/spack index 6f9a6ba4e..b9f2dd225 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 6f9a6ba4e5850494ffb7e7d592fe982036c32fb9 +Subproject commit b9f2dd2254cfb1086040f1a100746bd22cb12702 diff --git a/spack-ext/repos/spack-stack/packages/neptune-env/package.py b/spack-ext/repos/spack-stack/packages/neptune-env/package.py index ddd732692..ff7aef9d4 100644 --- a/spack-ext/repos/spack-stack/packages/neptune-env/package.py +++ b/spack-ext/repos/spack-stack/packages/neptune-env/package.py @@ -47,5 +47,6 @@ class NeptuneEnv(BundlePackage): # Basic Python dependencies that are always needed depends_on("py-f90nml", type="run") depends_on("py-python-dateutil", type="run") + depends_on("py-pyyaml", type="run") # There is no need for install() since there is no code. diff --git a/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py b/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py index 5b6f243b8..2e20105b8 100644 --- a/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py +++ b/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py @@ -22,6 +22,8 @@ class NeptunePythonEnv(BundlePackage): variant("xnrl", default=False, description="Build non-pulic XNRL") depends_on("neptune-env", type="run") + # Enable the Python variant for ESMF + depends_on("esmf +python", type="run") depends_on("py-h5py", type="run") depends_on("py-netcdf4", type="run") From 885db56fa09c6d8b935703e2b06d99d98b23a36d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 21 Oct 2024 16:46:42 -0600 Subject: [PATCH 05/33] [skip ci] Bug fix for wrong local-source paths on Narwhal and Nautilus (#1353) --- configs/sites/tier1/narwhal/mirrors.yaml | 4 ++-- configs/sites/tier1/nautilus/mirrors.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/sites/tier1/narwhal/mirrors.yaml b/configs/sites/tier1/narwhal/mirrors.yaml index ec3db4a00..2b67bea03 100644 --- a/configs/sites/tier1/narwhal/mirrors.yaml +++ b/configs/sites/tier1/narwhal/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: local-source: fetch: - url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache + url: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache + url: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache access_pair: - null - null diff --git a/configs/sites/tier1/nautilus/mirrors.yaml b/configs/sites/tier1/nautilus/mirrors.yaml index ec3db4a00..2b67bea03 100644 --- a/configs/sites/tier1/nautilus/mirrors.yaml +++ b/configs/sites/tier1/nautilus/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: local-source: fetch: - url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache + url: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache + url: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache access_pair: - null - null From b59c4d3895c18d4342b80e700e92dce4eeaaf4dc Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 22 Oct 2024 09:23:04 -0600 Subject: [PATCH 06/33] [skip ci] Delete empty configs/common/packages (#1354) This file got merged mistakenly, it doesn't have a `.yaml` suffix and is empty. --- configs/common/packages | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 configs/common/packages diff --git a/configs/common/packages b/configs/common/packages deleted file mode 100644 index e69de29bb..000000000 From 2c0dda1d65925fc5907922ee1640eccdef347eea Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 23 Oct 2024 15:00:39 -0700 Subject: [PATCH 07/33] Update default sigio & sfcio (#1358) * point to testing submodule * update pointer to testing submodule * default sfcio->1.4.2 & sigio->2.3.3 * update pointer to testing submodule * revert submodule/pointer --- configs/common/packages.yaml | 4 ++-- spack | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 99f17b060..7248b1b61 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -260,11 +260,11 @@ packages: scotch: require: '@7.0.4 +mpi+metis~shared~threads~mpi_thread+noarch' sfcio: - require: '@1.4.1' + require: '@1.4.2' shumlib: require: '@macos_clang_linux_intel_port' sigio: - require: '@2.3.2' + require: '@2.3.3' sp: require: '@2.5.0 precision=4,d,8' udunits: diff --git a/spack b/spack index b9f2dd225..990f0b84b 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit b9f2dd2254cfb1086040f1a100746bd22cb12702 +Subproject commit 990f0b84b7f6096c0542d04784c291666cdc2b7b From 72ec9f8e5d59d03f234a72dbd77fbc7e2f35f702 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 4 Nov 2024 07:26:38 -0700 Subject: [PATCH 08/33] Update .gitmodules and submodule pointer for spack for code review and testing --- .gitmodules | 6 ++++-- spack | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index a69f3e4f1..8db64f136 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,9 @@ [submodule "spack"] path = spack - url = https://github.com/jcsda/spack - branch = spack-stack-dev + #url = https://github.com/jcsda/spack + #branch = spack-stack-dev + url = https://github.com/climbfuji/spack + branch = feature/update_from_spack_dev_20241031 [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index 990f0b84b..2f31a163f 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 990f0b84b7f6096c0542d04784c291666cdc2b7b +Subproject commit 2f31a163f862793c6e94ee76d13640366f3c1834 From 15a91da195b9b954767aab2b6e932ea0a9239a28 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 4 Nov 2024 07:26:59 -0700 Subject: [PATCH 09/33] On blackpearl, only use one build job for spack --- configs/sites/tier2/blackpearl/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/sites/tier2/blackpearl/config.yaml b/configs/sites/tier2/blackpearl/config.yaml index 77a2e80ad..0eb7669fb 100644 --- a/configs/sites/tier2/blackpearl/config.yaml +++ b/configs/sites/tier2/blackpearl/config.yaml @@ -1,2 +1,2 @@ config: - build_jobs: 2 + build_jobs: 1 From 9a2e3048f8e75cf423a5f543cf3e3da8a86a3eab Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 4 Nov 2024 07:27:26 -0700 Subject: [PATCH 10/33] Bug fix in configs/templates/unified-dev/spack.yaml: gsi-env does not depend on ESMF --- configs/templates/unified-dev/spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index 370a03e7c..744789778 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -13,7 +13,7 @@ spack: - geos-gcm-env ^esmf@=8.6.1 - global-workflow-env ^esmf@=8.6.1 - gmao-swell-env - - gsi-env ^esmf@=8.6.1 + - gsi-env - jedi-fv3-env - jedi-geos-env ^esmf@=8.6.1 - jedi-mpas-env From 321d42f7b83363e6fd23127729b0b4f588b8339f Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 4 Nov 2024 07:28:13 -0700 Subject: [PATCH 11/33] Update configs/common/packages.yaml: pin gmake to either 4.2.1 or 4.4.1, and remove noavx512 variant for openblas --- configs/common/packages.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 7248b1b61..011e5027c 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -102,6 +102,10 @@ packages: # Note: Uncommenting this entry will break # the container builds. #require: '@2.11.0' + # To avoid duplicate packages being built + gmake: + require: + - one_of: ['@=4.2.1', '@=4.4.1'] grib-util: require: '@1.4.0' gsibec: @@ -170,7 +174,7 @@ packages: odc: require: '@1.5.2 ~fortran' openblas: - require: '@0.3.24 +noavx512' + require: '@0.3.24' openmpi: require: '~internal-hwloc +two_level_namespace' openssl: From 8a651e38e36d5017c74e73db3da5f818039388e5 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 4 Nov 2024 08:43:58 -0700 Subject: [PATCH 12/33] Pin meson to 1.5.1 or higher to avoid duplicate packages being built --- configs/common/packages.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 011e5027c..168ccf342 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -102,7 +102,7 @@ packages: # Note: Uncommenting this entry will break # the container builds. #require: '@2.11.0' - # To avoid duplicate packages being built + # To avoid duplicate packages being built gmake: require: - one_of: ['@=4.2.1', '@=4.4.1'] @@ -142,6 +142,10 @@ packages: mapl: require: '@2.46.3 ~shared ~f2py' variants: '+pflogger' + # To avoid duplicate packages being built + meson: + require: + - '@1.5.1:' # If making changes here, also check the Discover site configs and the CI workflows met: require: '@11.1.1 +python +grib2' From 46705640588a386731594b64f508f477842f7711 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 4 Nov 2024 13:39:51 -0700 Subject: [PATCH 13/33] Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 2f31a163f..7c25b422c 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 2f31a163f862793c6e94ee76d13640366f3c1834 +Subproject commit 7c25b422cfeaa1e9d64ccc5cc447bcce7ec736ef From e347cc36450eda858246f4355ae57fb1c0a30a62 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 5 Nov 2024 12:48:47 -0700 Subject: [PATCH 14/33] Update openmpi from 5.0.3 to 5.0.5 in several site/container configs --- configs/containers/docker-ubuntu-gcc-openmpi.yaml | 2 +- configs/sites/tier2/blackpearl/packages_gcc.yaml | 2 +- configs/sites/tier2/bounty/packages_gcc.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index f0e495073..526d47589 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -28,7 +28,7 @@ spack: require: '%gcc' target: [x86_64] providers: - mpi: [openmpi@5.0.3] + mpi: [openmpi@5.0.5] compiler: [gcc@12.3.0] gcc: buildable: false diff --git a/configs/sites/tier2/blackpearl/packages_gcc.yaml b/configs/sites/tier2/blackpearl/packages_gcc.yaml index a389fbcfc..730601342 100644 --- a/configs/sites/tier2/blackpearl/packages_gcc.yaml +++ b/configs/sites/tier2/blackpearl/packages_gcc.yaml @@ -2,4 +2,4 @@ packages: all: compiler:: [gcc@13.3.0] providers: - mpi:: [openmpi@5.0.3] + mpi:: [openmpi@5.0.5] diff --git a/configs/sites/tier2/bounty/packages_gcc.yaml b/configs/sites/tier2/bounty/packages_gcc.yaml index a91408366..0ddd36693 100644 --- a/configs/sites/tier2/bounty/packages_gcc.yaml +++ b/configs/sites/tier2/bounty/packages_gcc.yaml @@ -2,4 +2,4 @@ packages: all: compiler:: [gcc@12.3.0] providers: - mpi:: [openmpi@5.0.3] + mpi:: [openmpi@5.0.5] From f51d45a0b133bc9f7dbdee03c17453d86a30f262 Mon Sep 17 00:00:00 2001 From: Stephen Herbener <32968781+srherbener@users.noreply.github.com> Date: Tue, 5 Nov 2024 12:49:43 -0700 Subject: [PATCH 15/33] Updated macos notes to reflect that apple-clang@15.x compilers are now fully supported (#1365) * Updated macos notes to reflect that apple-clang@15.x compilers are now fully supported. * Removed "known issue" about the apple-clang@15.x versions not being fully supported. --- doc/source/KnownIssues.rst | 4 ---- doc/source/NewSiteConfigs.rst | 7 +++---- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/doc/source/KnownIssues.rst b/doc/source/KnownIssues.rst index 2e2c2171a..6d411af8a 100644 --- a/doc/source/KnownIssues.rst +++ b/doc/source/KnownIssues.rst @@ -109,10 +109,6 @@ macOS Can happen when trying to use the raster plotting scripts in ``fv3-jedi-tools``. In that case, exporting ``DYLD_LIBRARY_PATH=/usr/lib/:$DYLD_LIBRARY_PATH`` can help. If ``git`` commands fail after this, you might need to verify where ``which git`` points to (Homebrew vs module) and unload the ``git`` module. -6. Not all versions of ``apple-clang@15.0.0`` are currently supported. - - If you are running macOS Sonoma 14.5 you may have a recent version of clang that does not yet build spack packages. On the console you should run ``clang --version`` and verify that the underlying clang compiler is ``clang-1500.1.0.2.5`` which is the version provided with Command Line Tools for XCode 15.1. You can downgrade your Command Line Tool packages by first running ``sudo rm -rf /Library/Developer/CommandLineTools`` then you can download and install the Command Line Tools 15.1 APK from `developer.apple.com `_. The Apple Developer site requires an iCloud login. - ============================== Ubuntu ============================== diff --git a/doc/source/NewSiteConfigs.rst b/doc/source/NewSiteConfigs.rst index 4866137d4..04ec87431 100644 --- a/doc/source/NewSiteConfigs.rst +++ b/doc/source/NewSiteConfigs.rst @@ -32,8 +32,7 @@ The instructions below are for GNU (`gcc`), since this is the easiest and best s We have noted problems on some - not all - platforms with ``intel@2021.5.0`` when we switched from ``zlib`` to ``zlib-ng`` in spack-stack-1.7.0. These issues went away when using a different version of the compiler (anything between 2021.3.0 and 2021.11.0). It is therefore recommended to avoid using ``intel@2021.5.0`` unless it is the only option. .. [#fn2] - Note that ``apple-clang@14.x`` compiler versions are fully supported, and ``apple-clang@15.0.0`` will work but requires the :ref:`workaround noted below`. - Also, when using ``apple-clang@15.0.0`` you must use Command Line Tools version 15.1, and the Command Line Tools versions 15.3 and newer are not yet supported. + Note that ``apple-clang@14.x`` and ``apple-clang@15.x`` compiler versions are fully supported, and when using ``apple-clang@15.x`` the :ref:`workaround noted below` is required. .. [#fn3] Support for Nvidia compilers is experimental and limited to a subset of packages. Please refer to :numref:`Section %s ` below. @@ -285,8 +284,8 @@ Remember to activate the ``lua`` module environment and have MacTeX in your sear .. _apple-clang-15-workaround: .. note:: - When using apple-clang@15.0.0 (or newer) compilers, you need to manually add the following ldflags spec in the `site/compilers.yaml` file. - There are known issues with new features in the Apple linker/loader that comes with the 15.0.0 compiler set, and this change tells the linker/loader to use its legacy features which work fine. + When using apple-clang@15.x (or newer) compilers, you need to manually add the following ldflags spec in the `site/compilers.yaml` file. + There are known issues with new features in the Apple linker/loader that comes with the 15.x compiler set, and this change tells the linker/loader to use its legacy features which work fine. .. code-block:: yaml :emphasize-lines: 9,10 From d57737d0a95ace943b68cdc199545205b860152e Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 6 Nov 2024 08:53:31 -0700 Subject: [PATCH 16/33] Update GitHub actions workflows: make macOS workflow similar to Ubuntu workflows --- .github/workflows/macos-ci-aarch64.yaml | 63 +++++++++++++++---- .github/workflows/ubuntu-ci-x86_64-gnu.yaml | 18 ++++-- .github/workflows/ubuntu-ci-x86_64-intel.yaml | 11 ++-- .../workflows/ubuntu-ci-x86_64-oneapi.yaml | 30 ++++----- 4 files changed, 82 insertions(+), 40 deletions(-) diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml index e905915f2..0146b0d51 100644 --- a/.github/workflows/macos-ci-aarch64.yaml +++ b/.github/workflows/macos-ci-aarch64.yaml @@ -35,7 +35,7 @@ jobs: mkdir -p /Users/ec2-user/spack-stack/build-cache mkdir -p /Users/ec2-user/spack-stack/source-cache - - name: create-env + - name: create-buildcache run: | # Get day of week to decide whether to use build caches or not DOW=$(date +%u) @@ -56,15 +56,13 @@ jobs: source ./setup.sh # Important! export SPACK_PYTHON=/usr/bin/python3 - export ENVNAME=ue-apple-clang-14.0.3 - export ENVDIR=$PWD/envs/${ENVNAME} - spack clean -ab spack bootstrap now - + # + export ENVNAME=ue-apple-clang-14.0.3-buildcache + export ENVDIR=$PWD/envs/${ENVNAME} spack stack create env --site macos.default --template unified-dev --name ${ENVNAME} --compiler apple-clang spack env activate ${ENVDIR} - spack add ${{ inputs.specs || '' }} export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site" # Find external packages @@ -86,8 +84,11 @@ jobs: export -n SPACK_SYSTEM_CONFIG_PATH + # For buildcaches + spack config add config:install_tree:padded_length:200 + # Set compiler and MPI specs - spack config add "packages:all:providers:mpi:[openmpi@5.0.3]" + spack config add "packages:all:providers:mpi:[openmpi@5.0.5]" spack config add "packages:all:compiler:[apple-clang@14.0.3]" # Add additional variants for MET packages, different from config/common/packages.yaml @@ -97,8 +98,8 @@ jobs: # *DH # Concretize and check for duplicates - spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.3 - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.3 -i fms -i crtm -i esmf -i mapl + spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.3-buildcache + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.3-buildcache -i fms -i crtm -i esmf -i mapl # Add and update source cache spack mirror add local-source file:///Users/ec2-user/spack-stack/source-cache/ @@ -128,6 +129,11 @@ jobs: spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.apple-clang-14.0.3.jedi-base-env spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ jedi-base-env + # jedi-ufs-env + echo "jedi-ufs-env ..." + spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.apple-clang-14.0.3.jedi-ufs-env + spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ jedi-ufs-env + # the rest echo "unified-env ..." spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.3.unified-env @@ -138,12 +144,44 @@ jobs: spack mirror rm local-binary fi + # Remove buildcache config settings + spack config remove config:install_tree:padded_length + + # Next steps: synchronize source and build cache to a central/combined mirror? + + # Cleanup + spack clean -a + spack env deactivate + + - name: create-env + run: | + # Set up spack-stack + source ./setup.sh + # Important! + export SPACK_PYTHON=/usr/bin/python3 + export BUILDCACHE_ENVNAME=ue-apple-clang-14.0.3-builddcache + export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME} + export ENVNAME=ue-apple-clang-14.0.3 + export ENVDIR=$PWD/envs/${ENVNAME} + rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/ + spack env activate ${ENVDIR} + + # Concretize and check for duplicates + spack concretize --force 2>&1 | tee log.concretize.apple-clang-14.0.3 + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.3 -i fms -i crtm -i esmf -i mapl + + # Add binary cache back in + spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/ + echo "Packages in combined spack build caches:" + spack buildcache list + + # Install from cache + spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.3.unified-env + # Create modules spack clean -a spack module lmod refresh -y spack stack setup-meta-modules - - # Next steps: synchronize source and build cache to a central/combined mirror? spack env deactivate # Test environment chaining @@ -160,7 +198,6 @@ jobs: - name: test-env run: | - # Set up homebrew and lmod support eval "$(/opt/homebrew/bin/brew shellenv)" source /opt/homebrew/opt/lmod/init/profile @@ -170,7 +207,7 @@ jobs: module use ${ENVDIR}/install/modulefiles/Core module load stack-apple-clang/14.0.3 - module load stack-openmpi/5.0.3 + module load stack-openmpi/5.0.5 module load stack-python/3.11.7 module available diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml index 3b12d1c11..5452a9ab5 100644 --- a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml @@ -122,6 +122,11 @@ jobs: spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-base-env spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env + # jedi-ufs-env + echo "jedi-ufs-env ..." + spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-ufs-env + spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env + # the rest echo "${TEMPLATE} ..." spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.gnu-11.4.0-buildcache.${TEMPLATE} @@ -145,6 +150,7 @@ jobs: - name: create-env run: | + # Set up spack-stack source ./setup.sh export BUILDCACHE_ENVNAME=ue-gcc-11.4.0-buildcache export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME} @@ -153,20 +159,22 @@ jobs: rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/ spack env activate ${ENVDIR} - # Concretize + # Concretize and check for duplicates spack concretize --force 2>&1 | tee log.concretize.gnu-11.4.0 ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.gnu-11.4.0 -i fms -i crtm -i esmf -i mapl - # Add binary cache back in and reindex it + # Add binary cache back in spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ echo "Packages in combined spack build caches:" spack buildcache list - # base-env - echo "base-env ..." + # Install from cache spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.gnu-11.4.0.unified-env + # Check shared libraries ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check + + # Create modules spack clean -a spack module tcl refresh -y spack stack setup-meta-modules @@ -201,7 +209,7 @@ jobs: module use ${ENVDIR}/install/modulefiles/Core module load stack-gcc/11.4.0 - module load stack-openmpi/5.0.3 + module load stack-openmpi/5.0.5 module load stack-python/3.11.7 module available diff --git a/.github/workflows/ubuntu-ci-x86_64-intel.yaml b/.github/workflows/ubuntu-ci-x86_64-intel.yaml index eed14ce5b..513dbb4d2 100644 --- a/.github/workflows/ubuntu-ci-x86_64-intel.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-intel.yaml @@ -183,6 +183,7 @@ jobs: - name: create-env run: | + # Set up spack-stack source ./setup.sh export BUILDCACHE_ENVNAME=ue-intel-2021.10.0-buildcache export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME} @@ -191,20 +192,22 @@ jobs: rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/ spack env activate ${ENVDIR} - # Concretize + # Concretize and check for duplicates spack concretize --force 2>&1 | tee log.concretize.intel-2021.10.0 ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.intel-2021.10.0 -i fms -i crtm -i esmf -i mapl - # Add binary cache back in and reindex it + # Add binary cache back in spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ echo "Packages in combined spack build caches:" spack buildcache list - # base-env - echo "base-env ..." + # Install from cache spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.intel-2021.10.0.unified-env + # Check shared libraries ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check + + # Create modules spack clean -a spack module tcl refresh -y spack stack setup-meta-modules diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml index 0b4f7d337..263fbe0c4 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml @@ -168,6 +168,11 @@ jobs: spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.jedi-base-env spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env + # jedi-ufs-env + echo "jedi-ufs-env ..." + spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.jedi-ufs-env + spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env + # the rest echo "unified-env ..." spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.unified-env @@ -189,6 +194,7 @@ jobs: - name: create-env run: | + # Set up spack-stack source ./setup.sh export BUILDCACHE_ENVNAME=ue-oneapi-2024.2.0-buildcache export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME} @@ -197,39 +203,27 @@ jobs: rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/ spack env activate ${ENVDIR} - # Concretize + # Concretize and check for duplicates spack concretize --force 2>&1 | tee log.concretize.oneapi-2024.2.0 ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.oneapi-2024.2.0 -i fms -i crtm -i esmf -i mapl - # Add binary cache back in and reindex it + # Add binary cache back in spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ echo "Packages in combined spack build caches:" spack buildcache list - # base-env - echo "base-env ..." + # Install from cache spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-2024.2.0.unified-env + # Check shared libraries ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check + + # Create modules spack clean -a spack module tcl refresh -y spack stack setup-meta-modules spack env deactivate - ### # Test environment chaining - ### echo "Test environment chaining" - ### spack stack create env --name chaintest --template empty --site linux.default --upstream ${ENVDIR}/install --compiler intel - ### # Retain config from upstream so we don't have to rebuild: - ### cp -r ${ENVDIR}/{site,common} $PWD/envs/chaintest/. - ### spack env activate ${PWD}/envs/chaintest - ### # Pin gmake to avoid duplicate packages (excluded gmake from spack external find above) - ### spack config add "packages:gmake:require:'@:4.2'" - ### spack add nccmp@1.9.0.1%oneapi - ### spack concretize | tee envs/chaintest/log.concretize - ### unwanted_duplicates=$(( cat envs/chaintest/log.concretize | grep -E '^ - ' | grep -Fv 'nccmp@1.9.0.1' || true ) | wc -l) - ### if [ ${unwanted_duplicates} -gt 0 ]; then echo "Environment chaining test failed"; exit 1; fi - ### spack env deactivate - - name: test-env run: | source /etc/profile.d/modules.sh From f24ccd5c154c5ef5cfb75f77c3936512ae6a0184 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 6 Nov 2024 08:53:49 -0700 Subject: [PATCH 17/33] Update PrgEnv in configs/sites/tier1/narwhal/compilers.yaml --- configs/sites/tier1/narwhal/compilers.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/sites/tier1/narwhal/compilers.yaml b/configs/sites/tier1/narwhal/compilers.yaml index 838a8a1dd..ec9b170c0 100644 --- a/configs/sites/tier1/narwhal/compilers.yaml +++ b/configs/sites/tier1/narwhal/compilers.yaml @@ -9,7 +9,7 @@ compilers:: flags: {} operating_system: sles15 modules: - - PrgEnv-intel/8.3.3 + - PrgEnv-intel/8.4.0 - intel-classic/2023.2.0 - cray-libsci/23.05.1.4 - libfabric/1.12.1.2.2.1 @@ -55,7 +55,7 @@ compilers:: flags: {} operating_system: sles15 modules: - - PrgEnv-gnu/8.3.3 + - PrgEnv-gnu/8.4.0 - gcc/10.3.0 - cray-libsci/23.05.1.4 - libfabric/1.12.1.2.2.1 From 38cf079e47fe55199fd3f4e461390a213a953483 Mon Sep 17 00:00:00 2001 From: Stephen Herbener <32968781+srherbener@users.noreply.github.com> Date: Wed, 6 Nov 2024 11:36:41 -0700 Subject: [PATCH 18/33] Add instructions for configuring a spack-stack based python environment (#1367) Instructions are added to the UsingSpackEnvironments.rst showing how to configure and use a consistent python virtual environment based on spack-stack. --- doc/source/UsingSpackEnvironments.rst | 86 +++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/doc/source/UsingSpackEnvironments.rst b/doc/source/UsingSpackEnvironments.rst index a22dd9021..21d279c0b 100644 --- a/doc/source/UsingSpackEnvironments.rst +++ b/doc/source/UsingSpackEnvironments.rst @@ -5,6 +5,17 @@ Using spack-stack environments The following tutorial assumes you have a functioning spack-stack environment installed local to your system. This environment is provided on platforms described in :numref:`Section %s `. If you intend to run spack-stack on your developer machine or on a new platform, you can create an environment using the steps described in :numref:`Section %s `. +There are three steps in setting up a usable development environment. +The first is to load the spack-stack environment and the second is to create a python virtual environment that is based on the python executable included within the spack-stack installation. +The reason for the python virtual environment is to ensure that python based applications are utilizing the spack-stack python modules in a consistent manner. +The third step is to configure your build system to use the python virtual environment created in the second step. + +When using a spack-stack environment please utilize the spack-stack installed python modules as much as possible to help maintain the consistency mentioned above. +Note that after loading the spack-stack environment, all of the spack-stack installed python modules have been added to :code:`PYTHONPATH` so they are immediately accessable in your spack-stack based python virtual environment. + +Load the spack-stack environment +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + Spack environments are used by loading the modulefiles generated at the end of the installation process. These modules control the unix environment and allow CMake, ecbuild, and other build toolchains to resolve the version of software intended for the compilation task. The ``spack`` command itself is not needed in this setup, hence the instructions for creating new environments (``source setup.sh`` etc.) can be ignored. The following is sufficient for loading the modules, allowing them to be used while compiling and running user code. .. note:: @@ -28,3 +39,78 @@ Now list all available modules via ``module available``. You may be required to .. note:: When using ``lua`` modules, loading a different module will automatically switch the dependency modules. This is not the case for ``tcl`` modules. For the latter, it is recommended to start over with a clean shell and repeat the above steps. + +Create and activate a python virtual environment +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +It is important that the creation of the python virtual environment be based on the python executable from the spack-stack installation. +This ensures consistency for python applications between the python executable and the spack-stack installed python packages (eg., numpy). +Without this consistency, it is easy for the wrong underlying library versions to get dynamically loaded and cause problems with applications crashing. + +After the :code:`module load stack-python-name/python-version` command is run, the environment variable :code:`python_ROOT` will be set to the path where the spack-stack installed python version is located. +The :code:`python_ROOT` variable can be used to ensure that you get the proper virtual environment set as shown here: + +.. code-block:: console + + ${python_ROOT}/bin/python3 -m venv + +Once the virtual environment is set, it must be activated: + +.. code-block:: console + + source /bin/activate + +and after activation the spack-stack python executable will be the first one in your PATH. +The implication of this is that you should activate the python virtual enviroment as the last step in setting up your environment to ensure that the path to the virtual environment python remains first in your PATH. Here is an example of the whole process: + +.. code-block:: console + + # start from clean slate + module purge + + # load the base packages from the spack-stack environment + module use $SPACK_STACK_GNU_ENV/install/modulefiles/Core + module load stack-gcc/12.2.0 + module load stack-openmpi/4.1.4 + module load stack-python/3.11.7 + + # load the additional environments required for your + # target application + module load jedi-fv3-env + module load ewok-env + module load soca-env + + # Create and activate the spack-stack based python + # virtual environment + # Note that you only need to create the virtual environment + # the first time. Once created you only need to activate + # the virtual environment. + cd $HOME/projects/jedi + ${python_ROOT}/bin/python3 -m venv jedi_py_venv # first time only + source jedi_py_venv/bin/activate + +Configure build system to utilize the python virtual environment +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Configuring your application build system to use the python virtual environment will continue the goal of consistency mentioned above where all python scripts and packages within the target application are based on the spack-stack built python executable and packages. + +There are a variety of build systems in use, and CMake is quite commonly used so CMake will be used as an example for this step. +The CMake variable :code:`Python3_FIND_STRATEGY` can be used in conjunction with the python virtual environment to direct CMake to find and use the desired python virtual environment. +By default CMake chooses the latest python installation regardless of which comes first in your PATH. +By setting :code:`Python3_FIND_STRATEGY=LOCATION`, CMake will instead find and use the first python installation found in your PATH. +This is the reason for making the spack-stack based python virtual environment first in PATH in the step above. + +:code:`Python3_FIND_STRATEGY` can be set in two ways: the first in the project's top-level CMakeLists.txt file and the second on the cmake (or ecbuild) command line. +Here are examples of both methods: + +.. code-block:: console + + # In CMakeLists.txt + set( Python3_FIND_STRATEGY LOCATION ) + +.. code-block:: console + + # On the command line + cmake -DPython3_FIND_STRATEGY=LOCATION ... + + From cc080b80040860954965e08eb21e81d44c4c76ba Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 7 Nov 2024 09:40:29 -0700 Subject: [PATCH 19/33] Fix typo in .github/workflows/macos-ci-aarch64.yaml --- .github/workflows/macos-ci-aarch64.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml index 0146b0d51..43a8e9d6c 100644 --- a/.github/workflows/macos-ci-aarch64.yaml +++ b/.github/workflows/macos-ci-aarch64.yaml @@ -159,7 +159,7 @@ jobs: source ./setup.sh # Important! export SPACK_PYTHON=/usr/bin/python3 - export BUILDCACHE_ENVNAME=ue-apple-clang-14.0.3-builddcache + export BUILDCACHE_ENVNAME=ue-apple-clang-14.0.3-buildcache export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME} export ENVNAME=ue-apple-clang-14.0.3 export ENVDIR=$PWD/envs/${ENVNAME} From 9bd2e34d92fd9dbde96b683194bc9738ead9666a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 7 Nov 2024 09:40:53 -0700 Subject: [PATCH 20/33] Set correct variant for intel-oneapi-mpi in .github/workflows/ubuntu-ci-x86_64-intel.yaml --- .github/workflows/ubuntu-ci-x86_64-intel.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu-ci-x86_64-intel.yaml b/.github/workflows/ubuntu-ci-x86_64-intel.yaml index 513dbb4d2..f67e7e99a 100644 --- a/.github/workflows/ubuntu-ci-x86_64-intel.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-intel.yaml @@ -93,7 +93,7 @@ jobs: echo " intel-oneapi-mpi:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml echo " buildable: false" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml echo " externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 +classic-names" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml echo " prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml # Add external Intel MKL and oneAPI runtime @@ -162,6 +162,11 @@ jobs: spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.intel-2021.10.0-buildcache.jedi-base-env spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env + # jedi-ufs-env + echo "jedi-ufs-env ..." + spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.intel-2021.10.0-buildcache.jedi-ufs-env + spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env + # the rest echo "unified-env ..." spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.intel-2021.10.0-buildcache.unified-env From a2fab24380e122467c6c91aa8bf0acc72c51832d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 7 Nov 2024 13:15:40 -0700 Subject: [PATCH 21/33] Remove deprecated '-a' from all 'spack buildcache create' in GitHub workflows --- .github/workflows/macos-ci-aarch64.yaml | 8 ++++---- .github/workflows/ubuntu-ci-x86_64-gnu.yaml | 8 ++++---- .github/workflows/ubuntu-ci-x86_64-intel.yaml | 8 ++++---- .github/workflows/ubuntu-ci-x86_64-oneapi.yaml | 8 ++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml index 43a8e9d6c..cb4c6c219 100644 --- a/.github/workflows/macos-ci-aarch64.yaml +++ b/.github/workflows/macos-ci-aarch64.yaml @@ -122,22 +122,22 @@ jobs: # base-env echo "base-env ..." spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.apple-clang-14.0.3.base-env - spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ base-env + spack buildcache create -u /Users/ec2-user/spack-stack/build-cache/ base-env # jedi-base-env echo "jedi-base-env ..." spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.apple-clang-14.0.3.jedi-base-env - spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ jedi-base-env + spack buildcache create -u /Users/ec2-user/spack-stack/build-cache/ jedi-base-env # jedi-ufs-env echo "jedi-ufs-env ..." spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.apple-clang-14.0.3.jedi-ufs-env - spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ jedi-ufs-env + spack buildcache create -u /Users/ec2-user/spack-stack/build-cache/ jedi-ufs-env # the rest echo "unified-env ..." spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.3.unified-env - spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ + spack buildcache create -u /Users/ec2-user/spack-stack/build-cache/ # Remove binary cache for next round of concretization if [ "$USE_BINARY_CACHE" = true ] ; then diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml index 5452a9ab5..1a73ad69b 100644 --- a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml @@ -115,22 +115,22 @@ jobs: # base-env echo "base-env ..." spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.base-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env # jedi-base-env echo "jedi-base-env ..." spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-base-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env # jedi-ufs-env echo "jedi-ufs-env ..." spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-ufs-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env # the rest echo "${TEMPLATE} ..." spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.gnu-11.4.0-buildcache.${TEMPLATE} - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ # Remove binary cache for next round of concretization if [ "$USE_BINARY_CACHE" = true ] ; then diff --git a/.github/workflows/ubuntu-ci-x86_64-intel.yaml b/.github/workflows/ubuntu-ci-x86_64-intel.yaml index f67e7e99a..f391ce901 100644 --- a/.github/workflows/ubuntu-ci-x86_64-intel.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-intel.yaml @@ -155,22 +155,22 @@ jobs: # base-env echo "base-env ..." spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.intel-2021.10.0-buildcache.base-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env # jedi-base-env echo "jedi-base-env ..." spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.intel-2021.10.0-buildcache.jedi-base-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env # jedi-ufs-env echo "jedi-ufs-env ..." spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.intel-2021.10.0-buildcache.jedi-ufs-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env # the rest echo "unified-env ..." spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.intel-2021.10.0-buildcache.unified-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ # Remove binary cache for next round of concretization if [ "$USE_BINARY_CACHE" = true ] ; then diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml index 263fbe0c4..976813438 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml @@ -161,22 +161,22 @@ jobs: # base-env echo "base-env ..." spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.base-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env # jedi-base-env echo "jedi-base-env ..." spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.jedi-base-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env # jedi-ufs-env echo "jedi-ufs-env ..." spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.jedi-ufs-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env # the rest echo "unified-env ..." spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.unified-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ # Remove binary cache for next round of concretization if [ "$USE_BINARY_CACHE" = true ] ; then From 9b86458660304ed8cd01a3231f4d560f93a4d5bd Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 7 Nov 2024 20:49:42 -0700 Subject: [PATCH 22/33] Trigger CI From c84ae63e41a884dd328fec342394f62211035c2a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 8 Nov 2024 08:01:17 -0700 Subject: [PATCH 23/33] Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 7c25b422c..a3dd052ff 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 7c25b422cfeaa1e9d64ccc5cc447bcce7ec736ef +Subproject commit a3dd052fffce42379cfe814c5550cae59539d895 From 260b6f946df0ba4a90c739ba9a25bb0679e61dc2 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Nov 2024 11:38:02 -0700 Subject: [PATCH 24/33] Add +classic-names variant for intel-oneapi-mpi%intel in all site configs that need it --- configs/sites/tier1/atlantis/packages_intel.yaml | 2 +- configs/sites/tier1/aws-pcluster/packages_intel.yaml | 2 +- configs/sites/tier1/discover-scu16/packages_intel.yaml | 2 +- configs/sites/tier1/discover-scu17/packages_intel.yaml | 2 +- configs/sites/tier1/hera/packages_intel.yaml | 2 +- configs/sites/tier1/hercules/packages_intel.yaml | 2 +- configs/sites/tier1/jet/packages_intel.yaml | 2 +- configs/sites/tier1/noaa-aws/packages_intel.yaml | 2 +- configs/sites/tier1/noaa-azure/packages_intel.yaml | 2 +- configs/sites/tier1/noaa-gcloud/packages_intel.yaml | 2 +- configs/sites/tier1/orion/packages_intel.yaml | 2 +- configs/sites/tier1/s4/packages.yaml | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/configs/sites/tier1/atlantis/packages_intel.yaml b/configs/sites/tier1/atlantis/packages_intel.yaml index 4271f3f10..e6eb565f3 100644 --- a/configs/sites/tier1/atlantis/packages_intel.yaml +++ b/configs/sites/tier1/atlantis/packages_intel.yaml @@ -7,7 +7,7 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.6.0%intel@2021.6.0 + - spec: intel-oneapi-mpi@2021.6.0%intel@2021.6.0 +classic-names prefix: /cm/shared/apps/intel/oneapi modules: - mpi/2021.6.0 diff --git a/configs/sites/tier1/aws-pcluster/packages_intel.yaml b/configs/sites/tier1/aws-pcluster/packages_intel.yaml index 7fb3d8477..e0f255a05 100644 --- a/configs/sites/tier1/aws-pcluster/packages_intel.yaml +++ b/configs/sites/tier1/aws-pcluster/packages_intel.yaml @@ -11,7 +11,7 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 + - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 +classic-names prefix: /opt/intel/oneapi modules: - libfabric-aws/1.19.0amzn4.0 diff --git a/configs/sites/tier1/discover-scu16/packages_intel.yaml b/configs/sites/tier1/discover-scu16/packages_intel.yaml index 8bd48da65..a0f857ac8 100644 --- a/configs/sites/tier1/discover-scu16/packages_intel.yaml +++ b/configs/sites/tier1/discover-scu16/packages_intel.yaml @@ -7,7 +7,7 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.6.0%intel@2021.6.0 + - spec: intel-oneapi-mpi@2021.6.0%intel@2021.6.0 +classic-names prefix: /usr/local/intel/oneapi/2021 modules: - mpi/impi/2021.6.0 diff --git a/configs/sites/tier1/discover-scu17/packages_intel.yaml b/configs/sites/tier1/discover-scu17/packages_intel.yaml index d971e2387..e28f13a05 100644 --- a/configs/sites/tier1/discover-scu17/packages_intel.yaml +++ b/configs/sites/tier1/discover-scu17/packages_intel.yaml @@ -7,7 +7,7 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.10.0%intel@=2021.10.0 + - spec: intel-oneapi-mpi@2021.10.0%intel@=2021.10.0 +classic-names prefix: /usr/local/intel/oneapi/2021 modules: - mpi/impi/2021.10.0 diff --git a/configs/sites/tier1/hera/packages_intel.yaml b/configs/sites/tier1/hera/packages_intel.yaml index 5ac52dd8b..34b77b76d 100644 --- a/configs/sites/tier1/hera/packages_intel.yaml +++ b/configs/sites/tier1/hera/packages_intel.yaml @@ -11,7 +11,7 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.5.1%intel@2021.5.0 + - spec: intel-oneapi-mpi@2021.5.1%intel@2021.5.0 +classic-names modules: - impi/2022.1.2 prefix: /apps/oneapi diff --git a/configs/sites/tier1/hercules/packages_intel.yaml b/configs/sites/tier1/hercules/packages_intel.yaml index eb4a22dc7..b58287ac3 100644 --- a/configs/sites/tier1/hercules/packages_intel.yaml +++ b/configs/sites/tier1/hercules/packages_intel.yaml @@ -11,7 +11,7 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.9.0%intel@2021.9.0 + - spec: intel-oneapi-mpi@2021.9.0%intel@2021.9.0 +classic-names modules: - intel-oneapi-mpi/2021.9.0 intel-oneapi-mkl: diff --git a/configs/sites/tier1/jet/packages_intel.yaml b/configs/sites/tier1/jet/packages_intel.yaml index 5e0b82aaf..e04c4fa67 100644 --- a/configs/sites/tier1/jet/packages_intel.yaml +++ b/configs/sites/tier1/jet/packages_intel.yaml @@ -7,7 +7,7 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.5.1%intel@2021.5.0 + - spec: intel-oneapi-mpi@2021.5.1%intel@2021.5.0 +classic-names modules: - impi/2022.1.2 prefix: /apps/oneapi diff --git a/configs/sites/tier1/noaa-aws/packages_intel.yaml b/configs/sites/tier1/noaa-aws/packages_intel.yaml index 1ffe7cf2f..5cc0e66d7 100644 --- a/configs/sites/tier1/noaa-aws/packages_intel.yaml +++ b/configs/sites/tier1/noaa-aws/packages_intel.yaml @@ -11,7 +11,7 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 + - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 +classic-names prefix: /apps/oneapi modules: - impi/2023.2.0 diff --git a/configs/sites/tier1/noaa-azure/packages_intel.yaml b/configs/sites/tier1/noaa-azure/packages_intel.yaml index 1ffe7cf2f..5cc0e66d7 100644 --- a/configs/sites/tier1/noaa-azure/packages_intel.yaml +++ b/configs/sites/tier1/noaa-azure/packages_intel.yaml @@ -11,7 +11,7 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 + - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 +classic-names prefix: /apps/oneapi modules: - impi/2023.2.0 diff --git a/configs/sites/tier1/noaa-gcloud/packages_intel.yaml b/configs/sites/tier1/noaa-gcloud/packages_intel.yaml index 1ffe7cf2f..5cc0e66d7 100644 --- a/configs/sites/tier1/noaa-gcloud/packages_intel.yaml +++ b/configs/sites/tier1/noaa-gcloud/packages_intel.yaml @@ -11,7 +11,7 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 + - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 +classic-names prefix: /apps/oneapi modules: - impi/2023.2.0 diff --git a/configs/sites/tier1/orion/packages_intel.yaml b/configs/sites/tier1/orion/packages_intel.yaml index eb4a22dc7..b58287ac3 100644 --- a/configs/sites/tier1/orion/packages_intel.yaml +++ b/configs/sites/tier1/orion/packages_intel.yaml @@ -11,7 +11,7 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.9.0%intel@2021.9.0 + - spec: intel-oneapi-mpi@2021.9.0%intel@2021.9.0 +classic-names modules: - intel-oneapi-mpi/2021.9.0 intel-oneapi-mkl: diff --git a/configs/sites/tier1/s4/packages.yaml b/configs/sites/tier1/s4/packages.yaml index a2457222e..ed2478503 100644 --- a/configs/sites/tier1/s4/packages.yaml +++ b/configs/sites/tier1/s4/packages.yaml @@ -13,7 +13,7 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 + - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 +classic-names prefix: /opt/intel/oneapi/2023.2 modules: - intel/2023.2 From 540360aac9e4d8c3c98635c2f07fcade4ad90c25 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Nov 2024 20:40:42 -0700 Subject: [PATCH 25/33] In spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py, use information from preferred compiler to set correct MPI compiler wrapper command in MPI meta module (especially for Intel oneAPI) --- .../spack-stack/stack/meta_modules.py | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py index e2d75b651..a5ce92173 100755 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py @@ -361,7 +361,10 @@ def setup_meta_modules(): # Determine the preferred compiler and sort the flattened list of compilers # such that the preferred compiler comes last. This is so that all other # compilers populate the MODULEPATHS_SAVE list before the preferred compiler - # takes it and adds it to the stack-COMPILER metamodule. + # takes it and adds it to the stack-COMPILER metamodule. Likewise, we need + # to save the list of compiler substitutions from the preferred compiler + # so that we have access to it when we build the MPI meta module. Note that + # by definition, only the preferred compiler can be used for MPI dependencies. try: preferred_compilers = spack.config.get("packages")["all"]["prefer"] except: @@ -386,6 +389,8 @@ def custom_sort_key(entry): compiler_config = spack.config.get("compilers") # Collect and save modulepaths for the preferred compiler MODULEPATHS_SAVE = [] + # Initialize saved substitutes to None (populate for preferred compiler later) + SUBSTITUTES_SAVE = None for compiler_identifier in sorted_flattened_compiler_list: (compiler_name, compiler_version) = compiler_identifier.replace("@=","@").split("@") # Loop through all configured compilers and find the correct match @@ -523,6 +528,10 @@ def custom_sort_key(entry): with open(compiler_module_file, "w") as f: f.write(module_content) logging.info(" ... writing {}".format(compiler_module_file)) + # If this is the last compiler in the list (i.e. the preferred compiler), + # then save the substitutes for later use for building the MPI meta module. + if compiler_identifier == sorted_flattened_compiler_list[-1]: + SUBSTITUTES_SAVE = substitutes del MODULEPATHS_SAVE # Create mpi modules @@ -685,7 +694,18 @@ def custom_sort_key(entry): ) # Compiler wrapper environment variables - if "intel" in mpi_name: + if "intel" in mpi_name and compiler_name == "oneapi": + substitutes["MPICC"] = os.path.join("mpiicx") + substitutes["MPICXX"] = os.path.join("mpiicpx") + if "ifx" in SUBSTITUTES_SAVE["FC"] and not "ifort" in SUBSTITUTES_SAVE["FC"]: + substitutes["MPIF77"] = os.path.join("mpiifx") + substitutes["MPIF90"] = os.path.join("mpiifx") + elif not "ifx" in SUBSTITUTES_SAVE["FC"] and "ifort" in SUBSTITUTES_SAVE["FC"]: + substitutes["MPIF77"] = os.path.join("mpiifort") + substitutes["MPIF90"] = os.path.join("mpiifort") + else: + raise Exception(f"For {mpi_name}, cannot determine MPI wrapper from FC={SUBSTITUTES_SAVE['FC']}") + elif "intel" in mpi_name and compiler_name == "intel": substitutes["MPICC"] = os.path.join("mpiicc") substitutes["MPICXX"] = os.path.join("mpiicpc") substitutes["MPIF77"] = os.path.join("mpiifort") @@ -772,6 +792,7 @@ def custom_sort_key(entry): with open(mpi_module_file, "w") as f: f.write(module_content) logging.info(" ... writing {}".format(mpi_module_file)) + del SUBSTITUTES_SAVE # Create python modules. Need to accommodate both external # Python distributions and spack-built Python distributions. From 5acd9d4b58e1c95137fb96b3389105e4d957d54b Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 12 Nov 2024 05:42:42 -0700 Subject: [PATCH 26/33] Add debug printout of MPI meta module, add .github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml --- .github/workflows/ubuntu-ci-x86_64-gnu.yaml | 3 + .github/workflows/ubuntu-ci-x86_64-intel.yaml | 3 + .../ubuntu-ci-x86_64-oneapi-ifx.yaml | 248 ++++++++++++++++++ .../workflows/ubuntu-ci-x86_64-oneapi.yaml | 3 + 4 files changed, 257 insertions(+) create mode 100644 .github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml index 1a73ad69b..63a75cc7d 100644 --- a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml @@ -217,3 +217,6 @@ jobs: module load ewok-env module load soca-env module list + + echo "Contents of MPI metamodule:" + cat ${ENVDIR}/install/modulefiles/gcc/11.4.0/stack-openmpi/5.0.5 diff --git a/.github/workflows/ubuntu-ci-x86_64-intel.yaml b/.github/workflows/ubuntu-ci-x86_64-intel.yaml index f391ce901..c53cc6912 100644 --- a/.github/workflows/ubuntu-ci-x86_64-intel.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-intel.yaml @@ -249,3 +249,6 @@ jobs: module load ewok-env module load soca-env module list + + echo "Contents of MPI metamodule:" + cat ${ENVDIR}/install/modulefiles/intel/2021.10.0/stack-intel-oneapi-mpi/2021.10.0 diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml new file mode 100644 index 000000000..b73d3704f --- /dev/null +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml @@ -0,0 +1,248 @@ +name: ubuntu-ci-c6a-x86_64-oneapi-ifx-build +on: + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + ubuntu-ci-c6a-x86_64-oneapi-ifx-build: + runs-on: [ubuntu-ci-c6a-x86_64] + + steps: + - name: cleanup + run: | + pwd + ls -lart + set +e + find ./* -type d -exec chmod u+xw {} \; + set -e + rm -fr * + + - name: checkout + uses: actions/checkout@v4 + with: + submodules: true + + - name: prepare-directories + run: | + mkdir -p /home/ubuntu/spack-stack/build-cache/ + mkdir -p /home/ubuntu/spack-stack/source-cache/ + + - name: create-buildcache + run: | + # Get day of week to decide whether to use build caches or not + DOW=$(date +%u) + # Monday is 1 ... Sunday is 7 + if [[ $DOW == 7 ]]; then + export USE_BINARY_CACHE=false + echo "Ignore existing binary cache for creating buildcache environment" + else + export USE_BINARY_CACHE=true + echo "Use existing binary cache for creating buildcache environment" + fi + + # Set up spack-stack + source ./setup.sh + export ENVNAME=ue-oneapi-ifx-2024.2.0-buildcache + export ENVDIR=$PWD/envs/${ENVNAME} + spack stack create env --site linux.default --template unified-dev --name ${ENVNAME} --compiler oneapi + spack env activate ${ENVDIR} + export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site" + + # Find external packages + spack external find --scope system \ + --exclude bison --exclude openssl \ + --exclude curl --exclude python \ + --exclude gmake + spack external find --scope system sed + spack external find --scope system perl + spack external find --scope system wget + spack external find --scope system texlive + spack external find --scope system mysql + + # Find compilers + spack compiler find --scope system + + # For Intel oneAPI + echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo "- compiler:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo " spec: oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo " paths:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo " cc: /opt/intel/oneapi/compiler/2024.2/bin/icx" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo " cxx: /opt/intel/oneapi/compiler/2024.2/bin/icpx" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo " f77: /opt/intel/oneapi/compiler/2024.2/bin/ifx" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo " fc: /opt/intel/oneapi/compiler/2024.2/bin/ifx" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo " flags: {}" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo " operating_system: ubuntu22.04" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo " target: x86_64" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo " modules: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo " environment: {}" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo " extra_rpaths: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + + # Need to find external Intel MPI and annotate with the + # correct compiler, no way to do that with spack commands. + echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " intel-oneapi-mpi:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " buildable: false" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " - spec: intel-oneapi-mpi@2021.13%oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + + # Add external Intel MKL and oneAPI runtime + echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " intel-oneapi-mkl:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " intel-oneapi-runtime:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " - spec: intel-oneapi-runtime@2024.2.0%oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + + export -n SPACK_SYSTEM_CONFIG_PATH + + # For buildcaches + spack config add config:install_tree:padded_length:200 + + # Set compiler and MPI specs + spack config add "packages:all:providers:mpi:[intel-oneapi-mpi@2021.13]" + spack config add "packages:all:compiler:[oneapi@2024.2.0,gcc@11.4.0]" + + # Add additional variants for MET packages, different from config/common/packages.yaml + spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" + + # Switch providers for blas, lapack, fftw-api + spack config add "packages:all:providers:mpi:[intel-oneapi-mpi]" + spack config add "packages:all:providers:blas:[intel-oneapi-mkl]" + spack config add "packages:all:providers:fftw-api:[intel-oneapi-mkl]" + spack config add "packages:all:providers:lapack:[intel-oneapi-mkl]" + spack config add "packages:ectrans:require:'+mkl ~fftw'" + spack config add "packages:gsibec:require:'+mkl'" + spack config add "packages:py-numpy:require:['^intel-oneapi-mkl']" + + # Pin gmake to avoid duplicate packages (excluded gmake from spack external find above) + spack config add "packages:gmake:require:'@:4.2'" + + # Remove wgrib2 from all virtual packages, since it doesn't build with oneAPI + sed -i 's/depends_on("wgrib2")/#depends_on("wgrib2")/g' `grep -lRie wgrib2 spack-ext/` + + # Don't generate ecflow module when using external package + spack config add "modules:default:tcl:exclude:[ecflow]" + + # Concretize and check for duplicates + spack concretize 2>&1 | tee log.concretize.oneapi-ifx-2024.2.0-buildcache + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.oneapi-ifx-2024.2.0-buildcache -i fms -i crtm -i esmf -i mapl + + # Add and update source cache + spack mirror add local-source file:///home/ubuntu/spack-stack/source-cache/ + spack mirror create -a -d /home/ubuntu/spack-stack/source-cache/ + + # Add binary cache if requested + if [ "$USE_BINARY_CACHE" = true ] ; then + set +e + spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ + spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary) + set +e + echo "Packages in spack binary cache:" + spack buildcache list + fi + + # Break installation up in pieces and create build caches in between + # This allows us to "spin up" builds that altogether take longer than + # six hours, and/or fail later in the build process. + + # base-env + echo "base-env ..." + spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.oneapi-ifx-2024.2.0-buildcache.base-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env + + # jedi-base-env + echo "jedi-base-env ..." + spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.oneapi-ifx-2024.2.0-buildcache.jedi-base-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env + + # jedi-ufs-env + echo "jedi-ufs-env ..." + spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.oneapi-ifx-2024.2.0-buildcache.jedi-ufs-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env + + # the rest + echo "unified-env ..." + spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-ifx-2024.2.0-buildcache.unified-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ + + # Remove binary cache for next round of concretization + if [ "$USE_BINARY_CACHE" = true ] ; then + spack mirror rm local-binary + fi + + # Remove buildcache config settings + spack config remove config:install_tree:padded_length + + # Next steps: synchronize source and build cache to a central/combined mirror? + + # Cleanup + spack clean -a + spack env deactivate + + - name: create-env + run: | + # Set up spack-stack + source ./setup.sh + export BUILDCACHE_ENVNAME=ue-oneapi-ifx-2024.2.0-buildcache + export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME} + export ENVNAME=ue-oneapi-ifx-2024.2.0 + export ENVDIR=$PWD/envs/${ENVNAME} + rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/ + spack env activate ${ENVDIR} + + # Concretize and check for duplicates + spack concretize --force 2>&1 | tee log.concretize.oneapi-ifx-2024.2.0 + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.oneapi-ifx-2024.2.0 -i fms -i crtm -i esmf -i mapl + + # Add binary cache back in + spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ + echo "Packages in combined spack build caches:" + spack buildcache list + + # Install from cache + spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-ifx-2024.2.0.unified-env + + # Check shared libraries + ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check + + # Create modules + spack clean -a + spack module tcl refresh -y + spack stack setup-meta-modules + spack env deactivate + + - name: test-env + run: | + source /etc/profile.d/modules.sh + module use /home/ubuntu/spack-stack/modulefiles + + export ENVNAME=ue-oneapi-ifx-2024.2.0 + export ENVDIR=$PWD/envs/${ENVNAME} + ls -l ${ENVDIR}/install/modulefiles/Core + + module use ${ENVDIR}/install/modulefiles/Core + module load stack-oneapi/2024.2.0 + module load stack-intel-oneapi-mpi/2021.13 + module load stack-python/3.11.7 + module available + + module load jedi-ufs-env + module load ewok-env + module load soca-env + module list + + echo "Contents of MPI metamodule:" + cat ${ENVDIR}/install/modulefiles/oneapi/2024.2.0/stack-intel-oneapi-mpi/2021.13 diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml index 976813438..fdec50606 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml @@ -243,3 +243,6 @@ jobs: module load ewok-env module load soca-env module list + + echo "Contents of MPI metamodule:" + cat ${ENVDIR}/install/modulefiles/oneapi/2024.2.0/stack-intel-oneapi-mpi/2021.13 From c8dcfb4012b44b87bada83886bed9cb4df3b2383 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Wed, 13 Nov 2024 05:20:29 -0800 Subject: [PATCH 27/33] Enable scotch+esmumps by default (#1380) --- configs/common/packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 7248b1b61..ea8049c6d 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -258,7 +258,7 @@ packages: qt: require: '@5' scotch: - require: '@7.0.4 +mpi+metis~shared~threads~mpi_thread+noarch' + require: '@7.0.4 +mpi+metis~shared~threads~mpi_thread+noarch+esmumps' sfcio: require: '@1.4.2' shumlib: From efd7126e6f18acb5d7f775aae1261a6ce536d9d6 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 13 Nov 2024 15:24:24 -0700 Subject: [PATCH 28/33] Revert .gitmodules and update submodule pointer for spack --- .gitmodules | 6 ++---- spack | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index 8db64f136..a69f3e4f1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,7 @@ [submodule "spack"] path = spack - #url = https://github.com/jcsda/spack - #branch = spack-stack-dev - url = https://github.com/climbfuji/spack - branch = feature/update_from_spack_dev_20241031 + url = https://github.com/jcsda/spack + branch = spack-stack-dev [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index a3dd052ff..b6ef265ec 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit a3dd052fffce42379cfe814c5550cae59539d895 +Subproject commit b6ef265ec4582ddca131534a9d314e233919ae69 From 9fbd7bb8991e61517e28075f0bd73c91b0ad98a6 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 14 Nov 2024 04:46:49 -0700 Subject: [PATCH 29/33] Update spack-stack-dev from spack develop as of 2024/10/31 (#1363) * Update submodule pointer for spack for the changes in Update spack-stack-dev from spack develop as of 2024/10/31 spack#482 (update from spack develop as of 2024/10/31) * Make macOS CI test look like the UBuntu CI tests (split into two steps: first create buildcache environment, then install test environment from the buildcache) * Remove deprecated -a flag from spack buildcache push for all GitHub actions workflows * Add +classic-names to the intel-oneapi-mpi external package definition in GitHub actions. May need to do the same for each of the preconfigured sites when using icc+icpc+ifort. Without +classic-names, it works for icx+icpx+ifort and for ``icx+icpx`+`ifx`. * Updates to configs/common/packages.yaml to remove deprecated variants and pin certain packages to avoid duplicate packages * Update tier2 site config for blackpeark and bump PrgEnv for Narwhal * Bug fix: gsi-env does not depend on esmf, for some reason the concretizer didn't realize this until now * Pin gdal@3.8.5 for nco template (cxx issues) --------- Co-authored-by: Alex Richert --- .github/workflows/macos-ci-aarch64.yaml | 69 ++++++++++++++----- .github/workflows/ubuntu-ci-x86_64-gnu.yaml | 24 ++++--- .github/workflows/ubuntu-ci-x86_64-intel.yaml | 24 ++++--- .../workflows/ubuntu-ci-x86_64-oneapi.yaml | 36 ++++------ configs/common/packages.yaml | 10 ++- .../containers/docker-ubuntu-gcc-openmpi.yaml | 2 +- configs/sites/tier1/narwhal/compilers.yaml | 4 +- configs/sites/tier2/blackpearl/config.yaml | 2 +- .../sites/tier2/blackpearl/packages_gcc.yaml | 2 +- configs/sites/tier2/bounty/packages_gcc.yaml | 2 +- configs/templates/nco/spack.yaml | 2 +- configs/templates/unified-dev/spack.yaml | 2 +- spack | 2 +- 13 files changed, 118 insertions(+), 63 deletions(-) diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml index e905915f2..cb4c6c219 100644 --- a/.github/workflows/macos-ci-aarch64.yaml +++ b/.github/workflows/macos-ci-aarch64.yaml @@ -35,7 +35,7 @@ jobs: mkdir -p /Users/ec2-user/spack-stack/build-cache mkdir -p /Users/ec2-user/spack-stack/source-cache - - name: create-env + - name: create-buildcache run: | # Get day of week to decide whether to use build caches or not DOW=$(date +%u) @@ -56,15 +56,13 @@ jobs: source ./setup.sh # Important! export SPACK_PYTHON=/usr/bin/python3 - export ENVNAME=ue-apple-clang-14.0.3 - export ENVDIR=$PWD/envs/${ENVNAME} - spack clean -ab spack bootstrap now - + # + export ENVNAME=ue-apple-clang-14.0.3-buildcache + export ENVDIR=$PWD/envs/${ENVNAME} spack stack create env --site macos.default --template unified-dev --name ${ENVNAME} --compiler apple-clang spack env activate ${ENVDIR} - spack add ${{ inputs.specs || '' }} export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site" # Find external packages @@ -86,8 +84,11 @@ jobs: export -n SPACK_SYSTEM_CONFIG_PATH + # For buildcaches + spack config add config:install_tree:padded_length:200 + # Set compiler and MPI specs - spack config add "packages:all:providers:mpi:[openmpi@5.0.3]" + spack config add "packages:all:providers:mpi:[openmpi@5.0.5]" spack config add "packages:all:compiler:[apple-clang@14.0.3]" # Add additional variants for MET packages, different from config/common/packages.yaml @@ -97,8 +98,8 @@ jobs: # *DH # Concretize and check for duplicates - spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.3 - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.3 -i fms -i crtm -i esmf -i mapl + spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.3-buildcache + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.3-buildcache -i fms -i crtm -i esmf -i mapl # Add and update source cache spack mirror add local-source file:///Users/ec2-user/spack-stack/source-cache/ @@ -121,29 +122,66 @@ jobs: # base-env echo "base-env ..." spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.apple-clang-14.0.3.base-env - spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ base-env + spack buildcache create -u /Users/ec2-user/spack-stack/build-cache/ base-env # jedi-base-env echo "jedi-base-env ..." spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.apple-clang-14.0.3.jedi-base-env - spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ jedi-base-env + spack buildcache create -u /Users/ec2-user/spack-stack/build-cache/ jedi-base-env + + # jedi-ufs-env + echo "jedi-ufs-env ..." + spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.apple-clang-14.0.3.jedi-ufs-env + spack buildcache create -u /Users/ec2-user/spack-stack/build-cache/ jedi-ufs-env # the rest echo "unified-env ..." spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.3.unified-env - spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ + spack buildcache create -u /Users/ec2-user/spack-stack/build-cache/ # Remove binary cache for next round of concretization if [ "$USE_BINARY_CACHE" = true ] ; then spack mirror rm local-binary fi + # Remove buildcache config settings + spack config remove config:install_tree:padded_length + + # Next steps: synchronize source and build cache to a central/combined mirror? + + # Cleanup + spack clean -a + spack env deactivate + + - name: create-env + run: | + # Set up spack-stack + source ./setup.sh + # Important! + export SPACK_PYTHON=/usr/bin/python3 + export BUILDCACHE_ENVNAME=ue-apple-clang-14.0.3-buildcache + export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME} + export ENVNAME=ue-apple-clang-14.0.3 + export ENVDIR=$PWD/envs/${ENVNAME} + rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/ + spack env activate ${ENVDIR} + + # Concretize and check for duplicates + spack concretize --force 2>&1 | tee log.concretize.apple-clang-14.0.3 + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.3 -i fms -i crtm -i esmf -i mapl + + # Add binary cache back in + spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/ + echo "Packages in combined spack build caches:" + spack buildcache list + + # Install from cache + spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.3.unified-env + # Create modules spack clean -a spack module lmod refresh -y spack stack setup-meta-modules - - # Next steps: synchronize source and build cache to a central/combined mirror? spack env deactivate # Test environment chaining @@ -160,7 +198,6 @@ jobs: - name: test-env run: | - # Set up homebrew and lmod support eval "$(/opt/homebrew/bin/brew shellenv)" source /opt/homebrew/opt/lmod/init/profile @@ -170,7 +207,7 @@ jobs: module use ${ENVDIR}/install/modulefiles/Core module load stack-apple-clang/14.0.3 - module load stack-openmpi/5.0.3 + module load stack-openmpi/5.0.5 module load stack-python/3.11.7 module available diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml index 3b12d1c11..1a73ad69b 100644 --- a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml @@ -115,17 +115,22 @@ jobs: # base-env echo "base-env ..." spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.base-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env # jedi-base-env echo "jedi-base-env ..." spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-base-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env + + # jedi-ufs-env + echo "jedi-ufs-env ..." + spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-ufs-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env # the rest echo "${TEMPLATE} ..." spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.gnu-11.4.0-buildcache.${TEMPLATE} - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ # Remove binary cache for next round of concretization if [ "$USE_BINARY_CACHE" = true ] ; then @@ -145,6 +150,7 @@ jobs: - name: create-env run: | + # Set up spack-stack source ./setup.sh export BUILDCACHE_ENVNAME=ue-gcc-11.4.0-buildcache export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME} @@ -153,20 +159,22 @@ jobs: rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/ spack env activate ${ENVDIR} - # Concretize + # Concretize and check for duplicates spack concretize --force 2>&1 | tee log.concretize.gnu-11.4.0 ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.gnu-11.4.0 -i fms -i crtm -i esmf -i mapl - # Add binary cache back in and reindex it + # Add binary cache back in spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ echo "Packages in combined spack build caches:" spack buildcache list - # base-env - echo "base-env ..." + # Install from cache spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.gnu-11.4.0.unified-env + # Check shared libraries ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check + + # Create modules spack clean -a spack module tcl refresh -y spack stack setup-meta-modules @@ -201,7 +209,7 @@ jobs: module use ${ENVDIR}/install/modulefiles/Core module load stack-gcc/11.4.0 - module load stack-openmpi/5.0.3 + module load stack-openmpi/5.0.5 module load stack-python/3.11.7 module available diff --git a/.github/workflows/ubuntu-ci-x86_64-intel.yaml b/.github/workflows/ubuntu-ci-x86_64-intel.yaml index eed14ce5b..f391ce901 100644 --- a/.github/workflows/ubuntu-ci-x86_64-intel.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-intel.yaml @@ -93,7 +93,7 @@ jobs: echo " intel-oneapi-mpi:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml echo " buildable: false" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml echo " externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 +classic-names" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml echo " prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml # Add external Intel MKL and oneAPI runtime @@ -155,17 +155,22 @@ jobs: # base-env echo "base-env ..." spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.intel-2021.10.0-buildcache.base-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env # jedi-base-env echo "jedi-base-env ..." spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.intel-2021.10.0-buildcache.jedi-base-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env + + # jedi-ufs-env + echo "jedi-ufs-env ..." + spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.intel-2021.10.0-buildcache.jedi-ufs-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env # the rest echo "unified-env ..." spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.intel-2021.10.0-buildcache.unified-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ # Remove binary cache for next round of concretization if [ "$USE_BINARY_CACHE" = true ] ; then @@ -183,6 +188,7 @@ jobs: - name: create-env run: | + # Set up spack-stack source ./setup.sh export BUILDCACHE_ENVNAME=ue-intel-2021.10.0-buildcache export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME} @@ -191,20 +197,22 @@ jobs: rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/ spack env activate ${ENVDIR} - # Concretize + # Concretize and check for duplicates spack concretize --force 2>&1 | tee log.concretize.intel-2021.10.0 ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.intel-2021.10.0 -i fms -i crtm -i esmf -i mapl - # Add binary cache back in and reindex it + # Add binary cache back in spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ echo "Packages in combined spack build caches:" spack buildcache list - # base-env - echo "base-env ..." + # Install from cache spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.intel-2021.10.0.unified-env + # Check shared libraries ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check + + # Create modules spack clean -a spack module tcl refresh -y spack stack setup-meta-modules diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml index 0b4f7d337..976813438 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml @@ -161,17 +161,22 @@ jobs: # base-env echo "base-env ..." spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.base-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env # jedi-base-env echo "jedi-base-env ..." spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.jedi-base-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env + + # jedi-ufs-env + echo "jedi-ufs-env ..." + spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.jedi-ufs-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env # the rest echo "unified-env ..." spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.unified-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ # Remove binary cache for next round of concretization if [ "$USE_BINARY_CACHE" = true ] ; then @@ -189,6 +194,7 @@ jobs: - name: create-env run: | + # Set up spack-stack source ./setup.sh export BUILDCACHE_ENVNAME=ue-oneapi-2024.2.0-buildcache export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME} @@ -197,39 +203,27 @@ jobs: rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/ spack env activate ${ENVDIR} - # Concretize + # Concretize and check for duplicates spack concretize --force 2>&1 | tee log.concretize.oneapi-2024.2.0 ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.oneapi-2024.2.0 -i fms -i crtm -i esmf -i mapl - # Add binary cache back in and reindex it + # Add binary cache back in spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ echo "Packages in combined spack build caches:" spack buildcache list - # base-env - echo "base-env ..." + # Install from cache spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-2024.2.0.unified-env + # Check shared libraries ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check + + # Create modules spack clean -a spack module tcl refresh -y spack stack setup-meta-modules spack env deactivate - ### # Test environment chaining - ### echo "Test environment chaining" - ### spack stack create env --name chaintest --template empty --site linux.default --upstream ${ENVDIR}/install --compiler intel - ### # Retain config from upstream so we don't have to rebuild: - ### cp -r ${ENVDIR}/{site,common} $PWD/envs/chaintest/. - ### spack env activate ${PWD}/envs/chaintest - ### # Pin gmake to avoid duplicate packages (excluded gmake from spack external find above) - ### spack config add "packages:gmake:require:'@:4.2'" - ### spack add nccmp@1.9.0.1%oneapi - ### spack concretize | tee envs/chaintest/log.concretize - ### unwanted_duplicates=$(( cat envs/chaintest/log.concretize | grep -E '^ - ' | grep -Fv 'nccmp@1.9.0.1' || true ) | wc -l) - ### if [ ${unwanted_duplicates} -gt 0 ]; then echo "Environment chaining test failed"; exit 1; fi - ### spack env deactivate - - name: test-env run: | source /etc/profile.d/modules.sh diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index ea8049c6d..1c40bd5e8 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -102,6 +102,10 @@ packages: # Note: Uncommenting this entry will break # the container builds. #require: '@2.11.0' + # To avoid duplicate packages being built + gmake: + require: + - one_of: ['@=4.2.1', '@=4.4.1'] grib-util: require: '@1.4.0' gsibec: @@ -138,6 +142,10 @@ packages: mapl: require: '@2.46.3 ~shared ~f2py' variants: '+pflogger' + # To avoid duplicate packages being built + meson: + require: + - '@1.5.1:' # If making changes here, also check the Discover site configs and the CI workflows met: require: '@11.1.1 +python +grib2' @@ -170,7 +178,7 @@ packages: odc: require: '@1.5.2 ~fortran' openblas: - require: '@0.3.24 +noavx512' + require: '@0.3.24' openmpi: require: '~internal-hwloc +two_level_namespace' openssl: diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index f0e495073..526d47589 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -28,7 +28,7 @@ spack: require: '%gcc' target: [x86_64] providers: - mpi: [openmpi@5.0.3] + mpi: [openmpi@5.0.5] compiler: [gcc@12.3.0] gcc: buildable: false diff --git a/configs/sites/tier1/narwhal/compilers.yaml b/configs/sites/tier1/narwhal/compilers.yaml index 838a8a1dd..ec9b170c0 100644 --- a/configs/sites/tier1/narwhal/compilers.yaml +++ b/configs/sites/tier1/narwhal/compilers.yaml @@ -9,7 +9,7 @@ compilers:: flags: {} operating_system: sles15 modules: - - PrgEnv-intel/8.3.3 + - PrgEnv-intel/8.4.0 - intel-classic/2023.2.0 - cray-libsci/23.05.1.4 - libfabric/1.12.1.2.2.1 @@ -55,7 +55,7 @@ compilers:: flags: {} operating_system: sles15 modules: - - PrgEnv-gnu/8.3.3 + - PrgEnv-gnu/8.4.0 - gcc/10.3.0 - cray-libsci/23.05.1.4 - libfabric/1.12.1.2.2.1 diff --git a/configs/sites/tier2/blackpearl/config.yaml b/configs/sites/tier2/blackpearl/config.yaml index 77a2e80ad..0eb7669fb 100644 --- a/configs/sites/tier2/blackpearl/config.yaml +++ b/configs/sites/tier2/blackpearl/config.yaml @@ -1,2 +1,2 @@ config: - build_jobs: 2 + build_jobs: 1 diff --git a/configs/sites/tier2/blackpearl/packages_gcc.yaml b/configs/sites/tier2/blackpearl/packages_gcc.yaml index a389fbcfc..730601342 100644 --- a/configs/sites/tier2/blackpearl/packages_gcc.yaml +++ b/configs/sites/tier2/blackpearl/packages_gcc.yaml @@ -2,4 +2,4 @@ packages: all: compiler:: [gcc@13.3.0] providers: - mpi:: [openmpi@5.0.3] + mpi:: [openmpi@5.0.5] diff --git a/configs/sites/tier2/bounty/packages_gcc.yaml b/configs/sites/tier2/bounty/packages_gcc.yaml index a91408366..0ddd36693 100644 --- a/configs/sites/tier2/bounty/packages_gcc.yaml +++ b/configs/sites/tier2/bounty/packages_gcc.yaml @@ -2,4 +2,4 @@ packages: all: compiler:: [gcc@12.3.0] providers: - mpi:: [openmpi@5.0.3] + mpi:: [openmpi@5.0.5] diff --git a/configs/templates/nco/spack.yaml b/configs/templates/nco/spack.yaml index 12787de13..e1263204e 100644 --- a/configs/templates/nco/spack.yaml +++ b/configs/templates/nco/spack.yaml @@ -35,7 +35,7 @@ spack: - g2 - g2c - g2tmpl - - gdal + - gdal@3.8.5 - geos - gfsio - gftl-shared diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index 370a03e7c..744789778 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -13,7 +13,7 @@ spack: - geos-gcm-env ^esmf@=8.6.1 - global-workflow-env ^esmf@=8.6.1 - gmao-swell-env - - gsi-env ^esmf@=8.6.1 + - gsi-env - jedi-fv3-env - jedi-geos-env ^esmf@=8.6.1 - jedi-mpas-env diff --git a/spack b/spack index 990f0b84b..b6ef265ec 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 990f0b84b7f6096c0542d04784c291666cdc2b7b +Subproject commit b6ef265ec4582ddca131534a9d314e233919ae69 From 877efc420ef91ff35a71e24e309310804f06b285 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 14 Nov 2024 20:28:36 -0700 Subject: [PATCH 30/33] Revert debug prints in GitHub actions workflows --- .github/workflows/ubuntu-ci-x86_64-gnu.yaml | 3 --- .github/workflows/ubuntu-ci-x86_64-intel.yaml | 3 --- .github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml | 3 --- .github/workflows/ubuntu-ci-x86_64-oneapi.yaml | 3 --- 4 files changed, 12 deletions(-) diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml index 63a75cc7d..1a73ad69b 100644 --- a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml @@ -217,6 +217,3 @@ jobs: module load ewok-env module load soca-env module list - - echo "Contents of MPI metamodule:" - cat ${ENVDIR}/install/modulefiles/gcc/11.4.0/stack-openmpi/5.0.5 diff --git a/.github/workflows/ubuntu-ci-x86_64-intel.yaml b/.github/workflows/ubuntu-ci-x86_64-intel.yaml index c53cc6912..f391ce901 100644 --- a/.github/workflows/ubuntu-ci-x86_64-intel.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-intel.yaml @@ -249,6 +249,3 @@ jobs: module load ewok-env module load soca-env module list - - echo "Contents of MPI metamodule:" - cat ${ENVDIR}/install/modulefiles/intel/2021.10.0/stack-intel-oneapi-mpi/2021.10.0 diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml index b73d3704f..0397b0e51 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml @@ -243,6 +243,3 @@ jobs: module load ewok-env module load soca-env module list - - echo "Contents of MPI metamodule:" - cat ${ENVDIR}/install/modulefiles/oneapi/2024.2.0/stack-intel-oneapi-mpi/2021.13 diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml index fdec50606..976813438 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml @@ -243,6 +243,3 @@ jobs: module load ewok-env module load soca-env module list - - echo "Contents of MPI metamodule:" - cat ${ENVDIR}/install/modulefiles/oneapi/2024.2.0/stack-intel-oneapi-mpi/2021.13 From 4bf5913ea0f93985bc22152f277f1e0d293eb90d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 15 Nov 2024 09:22:43 -0700 Subject: [PATCH 31/33] Follow-up 2 to #1363 (update from spack dev 20241031): fix JEDI CI container builds for gcc and clang, and work towards fixing intel (#1381) This PR is a follow up to the update from spack develop in #1363. It removes the external gmake packages so that the new requirement for gmake to be one of 4.2.1 or 4.4.1 can be honored. --- configs/containers/docker-ubuntu-clang-mpich.yaml | 5 ----- configs/containers/docker-ubuntu-gcc-openmpi.yaml | 5 ----- configs/containers/docker-ubuntu-intel-impi.yaml | 5 ----- 3 files changed, 15 deletions(-) diff --git a/configs/containers/docker-ubuntu-clang-mpich.yaml b/configs/containers/docker-ubuntu-clang-mpich.yaml index 6ac7f83e8..67a986d3d 100644 --- a/configs/containers/docker-ubuntu-clang-mpich.yaml +++ b/configs/containers/docker-ubuntu-clang-mpich.yaml @@ -59,11 +59,6 @@ spack: - spec: mpich@4.2.1 prefix: /opt/mpich-4.2.1 version: [4.2.1] - gmake: - buildable: false - externals: - - spec: gmake@4.3 - prefix: /usr diffutils: buildable: false externals: diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index 526d47589..43b5b2fb8 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -40,11 +40,6 @@ spack: externals: - spec: gcc-runtime@12.3.0 prefix: /usr - gmake: - buildable: false - externals: - - spec: gmake@4.3 - prefix: /usr diffutils: buildable: false externals: diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml index 5aca2e63d..6dd5ad15d 100644 --- a/configs/containers/docker-ubuntu-intel-impi.yaml +++ b/configs/containers/docker-ubuntu-intel-impi.yaml @@ -50,11 +50,6 @@ spack: # externals: # - spec: intel-oneapi-mkl@2022.1.0 # prefix: /opt/intel/oneapi - gmake: - buildable: false - externals: - - spec: gmake@4.3 - prefix: /usr diffutils: buildable: false externals: From 5043e49c2bcf3f4742d68b5562adffc6440df435 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Sat, 16 Nov 2024 11:38:06 -0700 Subject: [PATCH 32/33] Add experimental support for DoD HPCMP Blueback, DOD HPCMP ParallelWorks AWS, and clean up documentation (maintainers section) (#1374) 1. Add a preliminary site config for the upcoming DoD HPCMP Blueback system. This config was tested on an early-access test system for the full machine. Support is for oneAPI and GNU only until the actual machine comes online. 2. Add a preliminary site config for the upcoming DoD HPCMP ParallelWorks cloud system (AWS only). This config was tested on the current Rocky 8 image, which will not be used once the system is inside a secured environment. Support is for GNU only until the correct image is ready. 3. Clean up the maintainers section in the documentation: remove external ecflow and miniconda (both legacy) and other packages that are no longer built externally by the spack-stack team. Delete corresponding modulefile templates. --- configs/sites/tier1/blueback/compilers.yaml | 45 +++ configs/sites/tier1/blueback/config.yaml | 2 + configs/sites/tier1/blueback/mirrors.yaml | 18 + configs/sites/tier1/blueback/modules.yaml | 7 + configs/sites/tier1/blueback/packages.yaml | 129 +++++++ .../sites/tier1/blueback/packages_gcc.yaml | 13 + .../sites/tier1/blueback/packages_oneapi.yaml | 27 ++ configs/sites/tier1/navy-aws/README.md | 41 ++ configs/sites/tier1/navy-aws/compilers.yaml | 28 ++ configs/sites/tier1/navy-aws/config.yaml | 10 + configs/sites/tier1/navy-aws/mirrors.yaml | 18 + configs/sites/tier1/navy-aws/modules.yaml | 7 + configs/sites/tier1/navy-aws/packages.yaml | 124 ++++++ .../sites/tier1/navy-aws/packages_gcc.yaml | 15 + doc/modulefile_templates/cmake | 15 - doc/modulefile_templates/ecflow | 28 -- doc/modulefile_templates/miniconda | 22 -- doc/modulefile_templates/mvapich2 | 35 -- doc/source/MaintainersSection.rst | 361 +----------------- doc/source/PreConfiguredSites.rst | 80 +++- 20 files changed, 572 insertions(+), 453 deletions(-) create mode 100644 configs/sites/tier1/blueback/compilers.yaml create mode 100644 configs/sites/tier1/blueback/config.yaml create mode 100644 configs/sites/tier1/blueback/mirrors.yaml create mode 100644 configs/sites/tier1/blueback/modules.yaml create mode 100644 configs/sites/tier1/blueback/packages.yaml create mode 100644 configs/sites/tier1/blueback/packages_gcc.yaml create mode 100644 configs/sites/tier1/blueback/packages_oneapi.yaml create mode 100644 configs/sites/tier1/navy-aws/README.md create mode 100644 configs/sites/tier1/navy-aws/compilers.yaml create mode 100644 configs/sites/tier1/navy-aws/config.yaml create mode 100644 configs/sites/tier1/navy-aws/mirrors.yaml create mode 100644 configs/sites/tier1/navy-aws/modules.yaml create mode 100644 configs/sites/tier1/navy-aws/packages.yaml create mode 100644 configs/sites/tier1/navy-aws/packages_gcc.yaml delete mode 100644 doc/modulefile_templates/cmake delete mode 100644 doc/modulefile_templates/ecflow delete mode 100644 doc/modulefile_templates/miniconda delete mode 100644 doc/modulefile_templates/mvapich2 diff --git a/configs/sites/tier1/blueback/compilers.yaml b/configs/sites/tier1/blueback/compilers.yaml new file mode 100644 index 000000000..28f061c75 --- /dev/null +++ b/configs/sites/tier1/blueback/compilers.yaml @@ -0,0 +1,45 @@ +compilers:: + - compiler: + spec: oneapi@2024.2.0 + paths: + cc: cc + cxx: CC + f77: ftn + fc: ftn + flags: {} + operating_system: sles15 + modules: + - PrgEnv-intel/8.4.0 + - intel/2024.2 + - cray-libsci/23.05.1.4 + - libfabric/1.12.1.2.2.1 + environment: + prepend_path: + PATH: '/opt/cray/pe/gcc/10.3.0/snos/bin' + CPATH: '/opt/cray/pe/gcc/10.3.0/snos/include' + LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/23.05.1.4/INTEL/2022.2/x86_64/lib:/opt/cray/pe/gcc/10.3.0/snos/lib:/opt/cray/pe/gcc/10.3.0/snos/lib64' + append_path: + CPATH: '/opt/intel/oneapi_2024.2.0.634/compiler/2024.2/opt/compiler/include/intel64' + set: + CRAYPE_LINK_TYPE: 'dynamic' + extra_rpaths: [] + - compiler: + spec: gcc@12.1.0 + paths: + cc: cc + cxx: CC + f77: ftn + fc: ftn + flags: {} + operating_system: sles15 + modules: + - PrgEnv-gnu/8.4.0 + - gcc/12.1.0 + - cray-libsci/23.05.1.4 + - libfabric/1.12.1.2.2.1 + environment: + prepend_path: + LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/23.05.1.4/GNU/10.3/x86_64/lib' + set: + CRAYPE_LINK_TYPE: 'dynamic' + extra_rpaths: [] diff --git a/configs/sites/tier1/blueback/config.yaml b/configs/sites/tier1/blueback/config.yaml new file mode 100644 index 000000000..84fe05ab9 --- /dev/null +++ b/configs/sites/tier1/blueback/config.yaml @@ -0,0 +1,2 @@ +config: + build_jobs: 8 diff --git a/configs/sites/tier1/blueback/mirrors.yaml b/configs/sites/tier1/blueback/mirrors.yaml new file mode 100644 index 000000000..2b67bea03 --- /dev/null +++ b/configs/sites/tier1/blueback/mirrors.yaml @@ -0,0 +1,18 @@ +mirrors: + local-source: + fetch: + url: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache + access_pair: + - null + - null + access_token: null + profile: null + endpoint_url: null + push: + url: file:///p/cwfs/projects/NEPTUNE/spack-stack/source-cache + access_pair: + - null + - null + access_token: null + profile: null + endpoint_url: null diff --git a/configs/sites/tier1/blueback/modules.yaml b/configs/sites/tier1/blueback/modules.yaml new file mode 100644 index 000000000..b134d3669 --- /dev/null +++ b/configs/sites/tier1/blueback/modules.yaml @@ -0,0 +1,7 @@ +modules: + default: + enable:: + - tcl + tcl: + include: + - python diff --git a/configs/sites/tier1/blueback/packages.yaml b/configs/sites/tier1/blueback/packages.yaml new file mode 100644 index 000000000..bb4d39c4d --- /dev/null +++ b/configs/sites/tier1/blueback/packages.yaml @@ -0,0 +1,129 @@ +packages: + autoconf: + externals: + - spec: autoconf@2.69 + prefix: /usr + automake: + externals: + - spec: automake@1.15.1 + prefix: /usr + binutils: + externals: + - spec: binutils@2.43.1 + prefix: /usr + cmake: + externals: + - spec: cmake@3.20.4 + prefix: /usr + coreutils: + externals: + - spec: coreutils@8.32 + prefix: /usr + curl: + externals: + - spec: curl@8.0.1+gssapi+ldap+nghttp2 + prefix: /usr + cvs: + externals: + - spec: cvs@1.12.12 + prefix: /usr + diffutils: + externals: + - spec: diffutils@3.6 + prefix: /usr + findutils: + externals: + - spec: findutils@4.8.0 + prefix: /usr + flex: + externals: + - spec: flex@2.6.4+lex + prefix: /usr + gawk: + externals: + - spec: gawk@4.2.1 + prefix: /usr + gettext: + externals: + - spec: gettext@0.20.2 + prefix: /usr + git: + externals: + - spec: git@2.35.3+tcltk + prefix: /usr + git-lfs: + externals: + - spec: git-lfs@3.0.2 + prefix: /usr + gmake: + externals: + - spec: gmake@4.2.1 + prefix: /usr + groff: + externals: + - spec: groff@1.22.4 + prefix: /usr + libtool: + externals: + - spec: libtool@2.4.6 + prefix: /usr + m4: + externals: + - spec: m4@1.4.18 + prefix: /usr + mysql: + buildable: False + externals: + - spec: mysql@8.0.31 + prefix: /p/app/projects/NEPTUNE/spack-stack/mysql-8.0.31 + modules: + - mysql/8.0.31 + openssh: + externals: + - spec: openssh@8.4p1 + prefix: /usr + - spec: openssh@9.8p1a + prefix: /usr/local/krb5 + openssl: + externals: + - spec: openssl@1.1.1l-fips + prefix: /usr + perl: + externals: + - spec: perl@5.26.1~cpanm+opcode+open+shared+threads + prefix: /usr + pkg-config: + externals: + - spec: pkg-config@0.29.2 + prefix: /usr + qt: + buildable: False + externals: + - spec: qt@5.15.2 + prefix: /p/app/projects/NEPTUNE/spack-stack/qt-5.15.2/5.15.2/gcc_64 + modules: + - qt/5.15.2 + sed: + externals: + - spec: sed@4.4 + prefix: /usr + subversion: + externals: + - spec: subversion@1.14.1 + prefix: /usr + swig: + externals: + - spec: swig@3.0.12 + prefix: /usr + tar: + externals: + - spec: tar@1.34 + prefix: /usr + texinfo: + externals: + - spec: texinfo@6.5 + prefix: /usr + wget: + externals: + - spec: wget@1.20.3 + prefix: /usr diff --git a/configs/sites/tier1/blueback/packages_gcc.yaml b/configs/sites/tier1/blueback/packages_gcc.yaml new file mode 100644 index 000000000..13c9f8d48 --- /dev/null +++ b/configs/sites/tier1/blueback/packages_gcc.yaml @@ -0,0 +1,13 @@ +packages: + all: + compiler:: [gcc@12.1.0] + providers: + mpi:: [cray-mpich@8.1.21] + mpi: + buildable: False + cray-mpich: + externals: + - spec: cray-mpich@8.1.21%gcc@12.1.0 ~wrappers + modules: + - cray-mpich-ucx/8.1.21 + - craype-network-ucx diff --git a/configs/sites/tier1/blueback/packages_oneapi.yaml b/configs/sites/tier1/blueback/packages_oneapi.yaml new file mode 100644 index 000000000..3bcf99f38 --- /dev/null +++ b/configs/sites/tier1/blueback/packages_oneapi.yaml @@ -0,0 +1,27 @@ +packages: + all: + compiler:: [oneapi@2024.2.0,gcc@10.3.0] + providers: + mpi:: [cray-mpich@8.1.21] + mpi: + buildable: False + cray-mpich: + externals: + - spec: cray-mpich@8.1.21%oneapi@2024.2.0 ~wrappers + modules: + - cray-mpich-ucx/8.1.21 + - craype-network-ucx + intel-oneapi-mkl: + externals: + - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.0 + prefix: /opt/intel/oneapi_2024.2.0.634 + intel-oneapi-tbb: + externals: + - spec: intel-oneapi-tbb@2021.13%oneapi@2024.2.0 + prefix: /opt/intel/oneapi_2024.2.0.634 + modules: + - tbb/2021.13 + intel-oneapi-runtime: + externals: + - spec: intel-oneapi-runtime@2024.2.0%oneapi@2024.2.0 + prefix: /opt/intel/oneapi_2024.2.0.634 diff --git a/configs/sites/tier1/navy-aws/README.md b/configs/sites/tier1/navy-aws/README.md new file mode 100644 index 000000000..b2b0e58f0 --- /dev/null +++ b/configs/sites/tier1/navy-aws/README.md @@ -0,0 +1,41 @@ +# Provisiong NRL ParallelWorks AWS clusters + +## Steps to perform before installing spack-stack develop as of 2024/11/24 + +Note. Some of these packages may already be installed, but for the sake of completeness, they are listed here. +``` +sudo su - +chmod 777 /contrib + +yum install -y gcc-toolset-13 +yum install -y gcc-toolset-13-runtime +yum install -y gcc-toolset-13-binutils +yum install -y gcc-toolset-13-gcc +yum install -y gcc-toolset-13-gcc-c++ +yum install -y gcc-toolset-13-gcc-gfortran +yum install -y gcc-toolset-13-gdb + +yum install -y binutils-devel +yum install -y m4 +yum install -y wget +yum install -y git +yum install -y git-lfs +yum install -y bash-completion +yum install -y bzip2 bzip2-devel +yum install -y unzip +yum install -y patch +yum install -y automake +yum install -y xorg-x11-xauth +yum install -y xterm +yum install -y perl-IPC-Cmd +yum install -y gettext-devel +yum install -y texlive +yum install -y bison +yum install -y screen + +yum install -y qt5-qtbase +yum install -y qt5-qttools-devel +yum install -y qt5-qtsvg-devel +``` + +For instructions for building spack-stack, see the spack-stack documentation on readthedocs (https://spack-stack.readthedocs.io/en/latest). For instructions for using pre-built spack-stack environments, see the spack-stack wiki (https://github.com/JCSDA/spack-stack/wiki). diff --git a/configs/sites/tier1/navy-aws/compilers.yaml b/configs/sites/tier1/navy-aws/compilers.yaml new file mode 100644 index 000000000..4c161e8cd --- /dev/null +++ b/configs/sites/tier1/navy-aws/compilers.yaml @@ -0,0 +1,28 @@ +compilers: +- compiler: + spec: gcc@=13.3.1 + paths: + cc: /opt/rh/gcc-toolset-13/root/usr/bin/gcc + cxx: /opt/rh/gcc-toolset-13/root/usr/bin/g++ + f77: /opt/rh/gcc-toolset-13/root/usr/bin/gfortran + fc: /opt/rh/gcc-toolset-13/root/usr/bin/gfortran + flags: {} + operating_system: rocky8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] +- compiler: + spec: gcc@=8.5.0 + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + flags: {} + operating_system: rocky8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + diff --git a/configs/sites/tier1/navy-aws/config.yaml b/configs/sites/tier1/navy-aws/config.yaml new file mode 100644 index 000000000..d01ba35fe --- /dev/null +++ b/configs/sites/tier1/navy-aws/config.yaml @@ -0,0 +1,10 @@ +config: + build_jobs: 6 + + # Overrides for spack build and staging areas to speed up builds + # and avoid errors with hard links on the NFS filesystem /contrib + + build_stage: /tmp/spack-stack/cache/build_stage + test_stage: /tmp/spack-stack/cache/test_stage + source_cache: /tmp/spack-stack/cache/source_cache + misc_cache: /tmp/spack-stack/cache/misc_cache diff --git a/configs/sites/tier1/navy-aws/mirrors.yaml b/configs/sites/tier1/navy-aws/mirrors.yaml new file mode 100644 index 000000000..3d247981f --- /dev/null +++ b/configs/sites/tier1/navy-aws/mirrors.yaml @@ -0,0 +1,18 @@ +mirrors: + local-source: + fetch: + url: file:///contrib/spack-stack/source-cache + access_pair: + - null + - null + access_token: null + profile: null + endpoint_url: null + push: + url: file:///contrib/spack-stack/source-cache + access_pair: + - null + - null + access_token: null + profile: null + endpoint_url: null diff --git a/configs/sites/tier1/navy-aws/modules.yaml b/configs/sites/tier1/navy-aws/modules.yaml new file mode 100644 index 000000000..b134d3669 --- /dev/null +++ b/configs/sites/tier1/navy-aws/modules.yaml @@ -0,0 +1,7 @@ +modules: + default: + enable:: + - tcl + tcl: + include: + - python diff --git a/configs/sites/tier1/navy-aws/packages.yaml b/configs/sites/tier1/navy-aws/packages.yaml new file mode 100644 index 000000000..7ac471cdf --- /dev/null +++ b/configs/sites/tier1/navy-aws/packages.yaml @@ -0,0 +1,124 @@ +packages: + autoconf: + externals: + - spec: autoconf@2.69 + prefix: /usr + automake: + externals: + - spec: automake@1.16.1 + prefix: /usr + binutils: + externals: + - spec: binutils@2.40.21~gold~headers + prefix: /opt/rh/gcc-toolset-13/root/usr + - spec: binutils@2.30.123~gold+headers + prefix: /usr + bison: + externals: + - spec: bison@3.0.4 + prefix: /usr + cmake: + externals: + - spec: cmake@3.26.5 + prefix: /usr + coreutils: + externals: + - spec: coreutils@8.30 + prefix: /usr + cvs: + externals: + - spec: cvs@1.11.23 + prefix: /usr + diffutils: + externals: + - spec: diffutils@3.6 + prefix: /usr + doxygen: + externals: + - spec: doxygen@1.8.14+graphviz~mscgen + prefix: /usr + findutils: + externals: + - spec: findutils@4.6.0 + prefix: /usr + flex: + externals: + - spec: flex@2.6.1+lex + prefix: /usr + gawk: + externals: + - spec: gawk@4.2.1 + prefix: /usr + gettext: + externals: + - spec: gettext@0.19.8.1 + prefix: /usr + git: + externals: + - spec: git@2.43.5+tcltk + prefix: /usr + git-lfs: + externals: + - spec: git-lfs@3.4.1 + prefix: /usr + gmake: + externals: + - spec: gmake@4.2.1 + prefix: /usr + groff: + externals: + - spec: groff@1.22.3 + prefix: /usr + libtool: + externals: + - spec: libtool@2.4.6 + prefix: /usr + m4: + externals: + - spec: m4@1.4.18 + prefix: /usr + openssh: + externals: + - spec: openssh@8.0p1 + prefix: /usr + perl: + externals: + - spec: perl@5.26.3~cpanm+opcode+open+shared+threads + prefix: /usr + pkgconf: + externals: + - spec: pkgconf@1.4.2 + prefix: /usr + qt: + buildable: False + externals: + - spec: qt@5.15.3 + prefix: /usr/lib64/qt5 + sed: + externals: + - spec: sed@4.5 + prefix: /usr + subversion: + externals: + - spec: subversion@1.10.2 + prefix: /usr + tar: + externals: + - spec: tar@1.30 + prefix: /usr + texinfo: + externals: + - spec: texinfo@6.5 + prefix: /usr + texlive: + externals: + - spec: texlive@2018 + prefix: /usr + wget: + externals: + - spec: wget@1.19.5 + prefix: /usr + zlib: + externals: + - spec: zlib@1.2.11 + prefix: /usr diff --git a/configs/sites/tier1/navy-aws/packages_gcc.yaml b/configs/sites/tier1/navy-aws/packages_gcc.yaml new file mode 100644 index 000000000..b8f1b3c48 --- /dev/null +++ b/configs/sites/tier1/navy-aws/packages_gcc.yaml @@ -0,0 +1,15 @@ +packages: + all: + compiler:: [gcc@13.3.1] + providers: + mpi:: [openmpi@5.0.5] + mpi: + buildable: False + openmpi: + buildable: False + externals: + - spec: openmpi@5.0.5%gcc@13.3.1 fabrics=auto ~gpfs ~internal-hwloc ~internal-libevent + ~internal-pmix +lustre +openshmem schedulers=slurm +romio romio-filesystem=lustre + modules: + - libfabric-aws/1.22.0amzn2.0 + - openmpi5/5.0.5 diff --git a/doc/modulefile_templates/cmake b/doc/modulefile_templates/cmake deleted file mode 100644 index 2478d236b..000000000 --- a/doc/modulefile_templates/cmake +++ /dev/null @@ -1,15 +0,0 @@ -#%Module1.0 - -module-whatis "Provides a cmake-3.27.2 installation for use with spack." - -conflict cmake - -proc ModulesHelp { } { -puts stderr "Provides a cmake-3.27.2 installation for use with spack." -} - -# Set this value -set CMAKE_PATH "/contrib/spack-stack/cmake-3.27.2" - -prepend-path PATH "${CMAKE_PATH}/bin" -prepend-path MANPATH "${CMAKE_PATH}/man" diff --git a/doc/modulefile_templates/ecflow b/doc/modulefile_templates/ecflow deleted file mode 100644 index 037421120..000000000 --- a/doc/modulefile_templates/ecflow +++ /dev/null @@ -1,28 +0,0 @@ -#%Module1.0 - -module-whatis "Provides an ecflow-5.8.4 server+ui installation for use with spack." - -conflict ecflow - -proc ModulesHelp { } { -puts stderr "Provides an ecflow-5.8.4 server+ui installation for use with spack." -} - -# If it is necessary to load a module for Qt (e.g. when not installed -# in the OS system paths), use the following logic; otherwise remove. -if { [ module-info mode load ] && ![ is-loaded qt/5.15.2 ] } { - module load qt/5.15.2 -} - -# Set this value -set ECFLOW_PATH "/discover/swdev/jcsda/spack-stack/ecflow-5.8.4" - -setenv ecflow_ROOT "${ECFLOW_PATH}" -prepend-path PATH "${ECFLOW_PATH}/bin" -prepend-path LD_LIBRARY_PATH "${ECFLOW_PATH}/lib" -prepend-path LD_LIBRARY_PATH "${ECFLOW_PATH}/lib64" -prepend-path LIBRARY_PATH "${ECFLOW_PATH}/lib" -prepend-path LIBRARY_PATH "${ECFLOW_PATH}/lib64" -prepend-path CPATH "${ECFLOW_PATH}/include" -prepend-path CMAKE_PREFIX_PATH "${ECFLOW_PATH}" -prepend-path PYTHONPATH "${ECFLOW_PATH}/lib/python3.9/site-packages" \ No newline at end of file diff --git a/doc/modulefile_templates/miniconda b/doc/modulefile_templates/miniconda deleted file mode 100644 index 24e412aa4..000000000 --- a/doc/modulefile_templates/miniconda +++ /dev/null @@ -1,22 +0,0 @@ -#%Module1.0 - -module-whatis "Provides a basic python-3.9.12 installation based on miniconda for use with spack." - -conflict python,anaconda,conda,miniconda,intelpython -#family python - -proc ModulesHelp { } { -puts stderr "Provides a basic python-3.9.12 installation based on miniconda for use with spack." -} - -# Set this value -set MINICONDA_PATH "/glade/work/jedipara/cheyenne/spack-stack/miniconda-3.9.12" - -prepend-path PATH "${MINICONDA_PATH}/bin" -prepend-path MANPATH "${MINICONDA_PATH}/share/man" -prepend-path LD_LIBRARY_PATH "${MINICONDA_PATH}/lib" -prepend-path LIBRARY_PATH "${MINICONDA_PATH}/lib" -prepend-path CPATH "${MINICONDA_PATH}/include" -prepend-path CMAKE_PREFIX_PATH "${MINICONDA_PATH}" -prepend-path PYTHONPATH "${MINICONDA_PATH}/lib/python3.9/site-packages" - diff --git a/doc/modulefile_templates/mvapich2 b/doc/modulefile_templates/mvapich2 deleted file mode 100644 index 042c16358..000000000 --- a/doc/modulefile_templates/mvapich2 +++ /dev/null @@ -1,35 +0,0 @@ -#%Module1.0 - -module-whatis "Provides an mvapich2-2.3.7 installation for use with spack and gcc-13.3.1." - -conflict openmpi -conflict mvapich2 -conflict mpi -conflict intel-mpi -conflict intel-oneapi-mpi - -proc ModulesHelp { } { -puts stderr "Provides an mvapich2-2.3.7 installation for use with spack and gcc-13.3.1." -} - -if { [ module-info mode load ] && ![ is-loaded slurm/22.05.8 ] } { - module load slurm/22.05.8 -} -#if { [ module-info mode load ] && ![ is-loaded ucx/1.13.1 ] } { -# module load ucx/1.13.1 -#`} - -# Set this value -set MPICH_PATH "/work/noaa/epic/role-epic/spack-stack/hercules/mvapich2-2.3.7/gcc-11.3.1" - -prepend-path PATH "${MPICH_PATH}/bin" -prepend-path LD_LIBRARY_PATH "${MPICH_PATH}/lib" -prepend-path LIBRARY_PATH "${MPICH_PATH}/lib" -prepend-path CPATH "${MPICH_PATH}/include" -prepend-path CMAKE_PREFIX_PATH "${MPICH_PATH}" -prepend-path MANPATH "${MPICH_PATH}/share/man" - -# Settings specific for Hercules -setenv MPI_ROOT ${MPICH_PATH} -setenv SLURM_MPI_TYPE "pmi2" -setenv MV2_HOMOGENEOUS_CLUSTER "1" diff --git a/doc/source/MaintainersSection.rst b/doc/source/MaintainersSection.rst index 79cc8e336..985e0d6e3 100644 --- a/doc/source/MaintainersSection.rst +++ b/doc/source/MaintainersSection.rst @@ -29,28 +29,6 @@ Building ``git-lfs`` with spack isn't straightforward as it requires ``go-bootst Following this "installation", create modulefile from template ``doc/modulefile_templates/git-lfs``. -.. _MaintainersSection_Miniconda: - ------------------------------- -Miniconda (legacy) ------------------------------- - -miniconda can be used to provide a basic version of Python that spack-stack uses to support its Python packages. This is not recommended on configurable systems (user workstations and laptops using GNU compiler) where Python gets installed by spack. But any system using Intel compilers with spack-stack will need an external Python to build ecflow with Python bindings (because ecflow requires a boost serialization function that does **not** work with Intel, a known yet ignored bug), and then both Python and ecflow are presented to spack as external packages. Often, it is possible to use the default (OS) Python if new enough (3.9+), or a module provided by the system administrators. If none of this works, use the following instructions to install a basic Python interpreter using miniconda: - -The following is for the example of ``miniconda_ver="py39_4.12.0"`` (for which ``python_ver=3.9.12``) and ``platform="MacOSX-x86_64"`` or ``platform="Linux-x86_64"`` - -.. code-block:: console - - cd /path/to/top-level/spack-stack/ - mkdir -p miniconda-${python_ver}/src - cd miniconda-${python_ver}/src - wget https://repo.anaconda.com/miniconda/Miniconda3-${miniconda_ver}-${platform}.sh - sh Miniconda3-${miniconda_ver}-${platform}.sh -u -b -p /path/to/top-level/spack-stack/miniconda-${python_ver} - eval "$(/path/to/top-level/spack-stack/miniconda-${python_ver}/bin/conda shell.bash hook)" - conda install -y -c conda-forge libpython-static - -After the successful installation, create modulefile ``/path/to/top-level/spack-stack/modulefiles/miniconda/${python_ver}`` from template ``doc/modulefile_templates/miniconda`` and update ``MINICONDA_PATH`` and the Python version in this file. - .. _MaintainersSection_Qt5: ------------------------------ @@ -80,75 +58,6 @@ Sign into qt, select customized installation, choose qt@5.15.2 only (uncheck all .. note:: If ``./qt-unified-linux-x64-online.run`` fails to start with the error ``qt.qpa.xcb: could not connect to display`` and a role account is being used, follow the procedure described in https://www.thegeekdiary.com/how-to-set-x11-forwarding-export-remote-display-for-users-who-switch-accounts-using-sudo to export the display. A possible warning ``xauth: file /ncrc/home1/role.epic/.Xauthority does not exist`` can be ignored, since this file gets created by the ``xauth`` command. -.. _MaintainersSection_ecFlow: - ------------------------------- -ecFlow (with GUI and Python) ------------------------------- - -Building ``ecFlow`` with spack is pretty tricky, because it requires functions from the ``boost`` serialization library that do not build cleanly with the Intel classic compilers (see https://github.com/USCiLab/cereal/issues/606 for a description of the problem of Intel with json cereal). When using the Intel compilers on HPC systems, it is therefore necessary to build ``ecFlow`` with the GNU compilers, preferably the same version that is used as the C++ backend for Intel, outside of spack-stack and make it available as a module. The build of ``ecFlow`` described below links against this ``boost`` library statically, therefore it does not interfere with ``boost`` built by spack-stack for other applications. ``ecFlow`` also uses ``Python3`` and ``qt5``. - -.. note:: - Installing ``ecFlow`` with ``conda``, ``brew``, etc. is not recommended, since these install a number of packages as dependencies (e.g. ``numpy``, dynamically-linked ``boost``) that may interfere with the spack software stack. - -After loading the required modules for this system (typically the same ``gcc`` used as backend for Intel or for GNU spack-stack builds, ``cmake``, ``qt5``, ``Python3``), follow these instructions to install ecFlow with the graphical user interface (GUI) and Python3 API. See also https://confluence.ecmwf.int/display/ECFLOW/ecflow5. - -.. code-block:: console - - mkdir -p /lustre/f2/pdata/esrl/gsd/spack-stack/ecflow-5.8.4/src - cd /lustre/f2/pdata/esrl/gsd/spack-stack/ecflow-5.8.4/src - wget https://confluence.ecmwf.int/download/attachments/8650755/ecFlow-5.8.4-Source.tar.gz?api=v2 - wget https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz - mv ecFlow-5.8.4-Source.tar.gz\?api\=v2 ecFlow-5.8.4-Source.tar.gz - tar -xvzf boost_1_78_0.tar.gz - tar -xvzf ecFlow-5.8.4-Source.tar.gz - export WK=/lustre/f2/pdata/esrl/gsd/spack-stack/ecflow-5.8.4/src/ecFlow-5.8.4-Source - export BOOST_ROOT=/lustre/f2/pdata/esrl/gsd/spack-stack/ecflow-5.8.4/src/boost_1_78_0 - - # Build static boost (to not interfere with spack-stack boost) - cd $BOOST_ROOT - ./bootstrap.sh 2>&1 | tee bootstrap.log - $WK/build_scripts/boost_build.sh 2>&1 | tee boost_build.log - - # Build ecFlow - cd $WK - mkdir build - cd build - cmake .. -DCMAKE_INSTALL_PREFIX=/lustre/f2/pdata/esrl/gsd/spack-stack/ecflow-5.8.4 2>&1 | tee log.cmake - make -j4 2>&1 | tee log.make - make install 2>&1 | tee log.install - -Create modulefile ``/lustre/f2/pdata/esrl/gsd/spack-stack/modulefiles/ecflow/5.8.4`` from template ``doc/modulefile_templates/ecflow`` and update ``ECFLOW_PATH`` in this file. - -.. note:: - For Cray systems, for example NRL's Narwhal, NOAA's Gaea C5, or NCAR's Derecho, the following modifications are necessary: After extracting the ecflow tarball, edit ``ecFlow-5.8.4-Source/build_scripts/boost_build.sh`` and remove the following lines: - -.. code-block:: console - - if [ "$PE_ENV" = INTEL ] ; then - tool=intel - fi - if [ "$PE_ENV" = CRAY ] ; then - tool=cray - fi - -.. note:: - Further on Narwhal, the ``cmake`` command for ``ecbuild`` must be told to use the GNU compilers: - -.. code-block:: console - - CC=gcc CXX=g++ FC=gfortran cmake .. -DCMAKE_INSTALL_PREFIX=/path/to/ecflow/installation 2>&1 | tee log.cmake - -.. note:: - Further, on Gaea C5, one needs to pass the correct ``python3`` executable to the ``cmake`` command: - -.. code-block:: console - - cmake .. -DPython3_EXECUTABLE=`which python3` -DCMAKE_INSTALL_PREFIX=/path/to/ecflow/installation 2>&1 | tee log.cmake - -.. note:: - Finally, on Casper, Derecho, or any other system with ``gcc@12.2.0``, one needs to patch file ``ecflow-5.8.4/src/ecFlow-5.8.4-Source/ACore/src/Passwd.cpp`` by adding ``#include `` below line ``#include "Passwd.hpp"`` before running ``make``. - .. _MaintainersSection_MySQL: ----------------------------------- @@ -209,6 +118,7 @@ If a mirror exists, add new packages to the mirror. Here, ``/path/to/mirror`` is If this fails with ``git lfs`` errors, check the site config for which module to load for ``git lfs`` support. Load the module, then run the ``spack mirror add`` command, then unload the module and proceed with the installation. + ============================== Pre-configuring sites ============================== @@ -219,7 +129,7 @@ Pre-configuring sites Preface/general instructions ------------------------------ -Preconfigured sites are defined through spack configuration files in the spack-stack directory ``configs/sites``, for example ``configs/sites/orion``. All files in the site-specific subdirectory will be copied into the environment into ``envs/env-name/site``. Site-specific configurations consist of general definitions (``config.yaml``), packages (``packages.yaml``), compilers (``compilers.yaml``), modules (``modules.yaml``), mirrors (``mirrors.yaml``) etc. These configurations overwrite the common configurations that are copied from ``configs/common`` into ``envs/env-name/common``. +Preconfigured sites are defined through spack configuration files in the spack-stack directory ``configs/sites``, for example ``configs/sites/orion``. All files in the site-specific subdirectory will be copied into the environment into ``envs/env-name/site``. Site-specific configurations consist of general definitions (``config.yaml``), packages (``packages.yaml``, ``packages_*.yaml``), compilers (``compilers.yaml``), modules (``modules.yaml``), mirrors (``mirrors.yaml``) etc. These configurations overwrite the common configurations that are copied from ``configs/common`` into ``envs/env-name/common``. The instructions below are platform-specific tasks that only need to be done once and can be reused for new spack environments. To build new environments on preconfigured platforms, follow the instructions in :numref:`Section %s `. @@ -229,139 +139,27 @@ Note that, for official installations of new environments on any supported platf spack install --source --verbose -.. _MaintainersSection_Orion: - ------------------------------- -MSU Orion ------------------------------- - -On Orion, it is necessary to change the default ``umask`` from ``0027`` to ``0022`` so that users not in the group of the role account can still see and use the software stack. This can be done by running ``umask 022`` after logging into the role account. - -ecflow - ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library. After installing `miniconda`, and loading the following modules, follow the instructions in :numref:`Section %s `. Note that the default/system ``qt@5`` can be used on Orion. - -.. code-block:: console - - module purge - module load python/3.9.2 - module load cmake/3.22.1 - module load gcc/10.2.0 - -.. _MaintainersSection_Hercules: - ------------------------------- -MSU Hercules ------------------------------- - -ecflow - ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library, using an available ``Qt5`` installation. After loading the following modules, follow the instructions in :numref:`Section %s ` to install ``ecflow`` in ``/work/noaa/epic/role-epic/spack-stack/hercules/ecflow-5.8.4``. NOTE: do NOT include the ``Qt5`` module dependency in the ``ecflow`` modulefile, as it is only needed at build time (and causes issues with zlib/tar if the depedency is kept in the modulefile). - -.. code-block:: console - - module purge - module load qt/5.15.8 - -openmpi - Because of difficulties with the default ``openmpi`` on Hercules, we build ``openmpi`` outside of spack and provide it as an external package. It is necessary to load the ``gcc`` compiler module and the ``zlib`` module for consistency. The configuration options are mostly adopted from the default OpenMPI installations that were done by the system administrators using spack (many of them are default values), except that we use internal ``hwloc`` and ``pmix``. Create modulefile ``openmpi`` from template ``doc/modulefile_templates/openmpi``. - -.. code-block:: console - - ./configure \ - --enable-shared \ - --disable-silent-rules \ - --disable-builtin-atomics \ - --with-pmi=/opt/slurm \ - --enable-static \ - --enable-mpi1-compatibility \ - --without-hcoll \ - --without-psm2 \ - --without-knem \ - --without-verbs \ - --without-psm \ - --without-cma \ - --without-ucx \ - --without-mxm \ - --without-fca \ - --without-xpmem \ - --without-ofi \ - --without-cray-xpmem \ - --without-sge \ - --without-lsf \ - --without-loadleveler \ - --without-alps \ - --without-tm \ - --with-slurm \ - --disable-memchecker \ - --with-pmix=internal \ - --with-zlib=/apps/spack-managed/gcc-12.2.0/zlib-1.2.13-p3sxbyfgvvjy7jx4kizib2jwvhm4s6l4 \ - --with-hwloc=internal \ - --disable-java \ - --disable-mpi-java \ - --with-gpfs=no \ - --without-cuda \ - --enable-wrapper-rpath \ - --disable-wrapper-runpath \ - --disable-mpi-cxx \ - --disable-cxx-exceptions \ - --with-wrapper-ldflags="-Wl,-rpath,/apps/spack-managed/gcc-11.3.1/gcc-12.2.0-7cu3qahzhsxpauy4jlnsbcqmlbkxbbbo/lib/gcc/x86_64-pc-linux-gnu/12.2.0 -Wl,-rpath,/apps/spack-managed/gcc-11.3.1/gcc-12.2.0-7cu3qahzhsxpauy4jlnsbcqmlbkxbbbo/lib64" \ - --prefix=/work/noaa/epic/role-epic/spack-stack/hercules/openmpi-4.1.6/gcc-12.2.0-spack 2>&1 | tee log.config - make VERBOSE=1 -j4 - make check - make install - .. _MaintainersSection_Discover_SCU16: ------------------------------ NASA Discover SCU16 ------------------------------ -On Discover SCU16, ``miniconda``, ``qt``, and ``ecflow`` need to be installed as a one-off before spack can be used. When using the GNU compiler, it is also necessary to build your own ``openmpi`` or other MPI library, which requires adapting the installation to the network hardware and ``slurm`` scheduler. - -miniconda - Follow the instructions in :numref:`Section %s ` to create a basic ``miniconda`` installation and associated modulefile for working with spack. Don't forget to log off and back on to forget about the conda environment. +On Discover SCU16, ``qt`` needs to be installed as a one-off before spack can be used. When using the GNU compiler, it is also necessary to build your own ``openmpi`` or other MPI library, which requires adapting the installation to the network hardware and ``slurm`` scheduler. qt (qt@5) The default ``qt@5`` in ``/usr`` is incomplete and thus insufficient for building ``ecflow``. After loading/unloading the modules as shown below, refer to :numref:`Section %s ` to install ``qt@5.15.2`` in ``/discover/swdev/jcsda/spack-stack/scu16/qt-5.15.2`` (note: it is currently installed in ``/discover/swdev/jcsda/spack-stack/qt-5.15.2``; an upcoming large system update will require is to rebuild anyway). -ecflow - ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library. After installing `miniconda`, `qt5`, and loading the following modules, follow the instructions in :numref:`Section %s `. - -.. code-block:: console - - module purge - module load cmake/3.28.2 - module load comp/gcc/12.1.0 - module use /discover/swdev/jcsda/spack-stack/modulefiles - module load miniconda/3.10.13 - module load qt/5.15.2 - .. _MaintainersSection_Discover_SCU17: ------------------------------ NASA Discover SCU17 ------------------------------ -On Discover SCU17 ``ecflow`` needs to be installed as a one-off before spack can be used. +On Discover SCU17, ``qt`` needs to be installed as a one-off before spack can be used. -ecflow - ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library. After loading the following modules, follow the instructions in :numref:`Section %s ` (cont'd below). - -.. code-block:: console - - module purge - module load cmake/3.28.2 - -The following workaround is required after installing ``ecflow`` and creating the modulefile: edit ``path/to/ecflow/bin/ecflow_ui`` and change the last few lines to (i.e. prepend the ``LD_PRELOAD`` command): - -.. code-block:: console - - if [ $ECFLOWUI_BT != "no" ] - then - LD_PRELOAD=/usr/lib64/libstdc++.so.6 catchsegv ${ECFLOWUI_USER_START_CMD} "$exe" - else - LD_PRELOAD=/usr/lib64/libstdc++.so.6 ${ECFLOWUI_USER_START_CMD} "$exe" - fi +**These instructions are missing. The current `qt` used in the SCU17 site config does not work (/usr/local/other/xpdf/4.04/Deps).** .. _MaintainersSection_Narwhal: @@ -369,7 +167,7 @@ The following workaround is required after installing ``ecflow`` and creating th NAVY HPCMP Narwhal ------------------------------ -On Narwhal, ``git-lfs``, ``qt``, and ``ecflow`` need to be installed as a one-off before spack can be used. Also, temporarily it is necessary to install ``node.js`` as an external package to work around build errors for ``py-jupyter-server`` (see https://github.com/JCSDA/spack-stack/issues/928 and https://github.com/spack/spack/issues/41899). +On Narwhal, ``git-lfs`` and ``qt`` need to be installed as a one-off before spack can be used. Also, temporarily it is necessary to install ``node.js`` as an external package to work around build errors for ``py-jupyter-server`` (see https://github.com/JCSDA/spack-stack/issues/928 and https://github.com/spack/spack/issues/41899). git-lfs The following instructions install ``git-lfs`` in ``/p/app/projects/NEPTUNE/spack-stack/git-lfs-2.10.0``. Version 2.10.0 is the default version for Narwhal. First, download the ``git-lfs`` RPM on a system with full internet access (e.g., Derecho) using ``wget https://download.opensuse.org/repositories/openSUSE:/Leap:/15.2/standard/x86_64/git-lfs-2.10.0-lp152.1.2.x86_64.rpm`` and copy this file to ``/p/app/projects/NEPTUNE/spack-stack/git-lfs-2.10.0/src``. Then switch to Narwhal and run the following commands. @@ -399,24 +197,6 @@ qt (qt@5) module load gcc/10.3.0 -ecflow - ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library. After installing `qt5`, and loading the following modules, follow the instructions in :numref:`Section %s ` to install ``ecflow`` in ``/p/app/projects/NEPTUNE/spack-stack/ecflow-5.8.4``. Ensure to follow the extra instructions in that section for Narwhal. - -.. code-block:: console - - module unload PrgEnv-cray - module load PrgEnv-intel/8.1.0 - module unload intel - - module unload cray-python - module load cray-python/3.9.7.1 - module unload cray-libsci - module load cray-libsci/22.08.1.1 - - module load gcc/10.3.0 - module use /p/app/projects/NEPTUNE/spack-stack/modulefiles - module load qt/5.15.2 - node.js ``node.js`` is difficult to install via ``spack``, but is needed to install certain Python packages. The complication is that when using a newer ``gcc`` compiler (either directly or as backend for ``icc`` etc.), the OS ``node.js`` errors out with unresolved symbols in the ``libstdc++`` library. Therefore, we need to install ``node.js`` with ``gcc@10.3.0`` loaded, and create modulefile ``node.js/20.10.0`` from template ``modulefiles/node.js``. @@ -437,42 +217,6 @@ node.js make 2>&1 | tee log.make make install 2>&1 | tee log.install -.. _MaintainersSection_Nautilus: - ------------------------------- -NAVY HPCMP Nautilus ------------------------------- - -On Nautilus, ``ecflow`` must be installed as a one-off before spack can be used. - -ecflow - ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library. After loading the following modules, follow the instructions in :numref:`Section %s ` to install ``ecflow`` in ``/p/app/projects/NEPTUNE/spack-stack/ecflow-5.8.4``. - -.. code-block:: console - - module purge - - module load slurm - module load amd/aocc/4.0.0 - module load amd/aocl/aocc/4.0 - -.. _MaintainersSection_Casper: - ------------------------------- -NCAR-Wyoming Casper ------------------------------- - -On Casper, there are problems with newer versions of the Intel compiler/MPI library when trying to run MPI jobs with just one task (``mpiexec -np 1``) - for JEDI, job hangs forever in a particular MPI communication call in oops. This is why an older version Intel 19 is used here. - -ecflow - ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library. After loading the following modules, follow the instructions in :numref:`Section %s `. - -.. code-block:: console - - module purge - export LMOD_TMOD_FIND_FIRST=yes - module load gnu/12.2.0 - .. _MaintainersSection_Derecho: ------------------------------ @@ -485,31 +229,13 @@ libfabric (temporary) cray-pals (temporary) Until CISL fixes its unusual way of setting up Cray module environments, it is necessary to create a cray-pals (parallel application launcher) module to be able to find ``mpirun`` etc. Create directory ``/glade/work/epicufsrt/contrib/spack-stack/derecho/cray-pals`` and copy file ``/opt/cray/pe/lmod/modulefiles/core/cray-pals/1.2.11.lua`` into this directory. -ecflow - ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library. After loading the following modules, follow the instructions in :numref:`Section %s ` to install ``ecflow``. Be sure to follow the extra instructions for Derecho in that section. - -.. code-block:: console - - module purge - export LMOD_TMOD_FIND_FIRST=yes - module load gcc/12.2.0 - module load cmake/3.26.3 - -.. _MaintainersSection_WCOSS2: - ------------------------------- -NOAA NCO WCOSS2 ------------------------------- - -**WORK IN PROGRESS** - .. _MaintainersSection_Parallel_Works: ---------------------------------------- NOAA Parallel Works (AWS, Azure, Gcloud) ---------------------------------------- -See ``configs/sites/noaa-aws/README.md``. These instructions are identical for all three vendors. +See ``configs/sites/noaa-{aws,azure,gcloud}/README.md``. .. _MaintainersSection_GaeaC5: @@ -517,7 +243,7 @@ See ``configs/sites/noaa-aws/README.md``. These instructions are identical for a NOAA RDHPCS Gaea C5 ------------------------------ -On Gaea C5, ``miniconda``, ``qt``, and ``ecflow`` need to be installed as a one-off before spack can be used. +On Gaea C5, ``qt`` needs to be installed as a one-off before spack can be used. qt (qt@5) The default ``qt@5`` in ``/usr`` is incomplete and thus insufficient for building ``ecflow``. After loading/unloading the modules as shown below, refer to :numref:`Section %s ` to install ``qt@5.15.2`` in ``/ncrc/proj/epic/spack-stack/qt-5.15.2``. :numref:`Section %s ` describes how to export the X windows environment in order to install ``qt@5`` using the role account. @@ -528,45 +254,12 @@ qt (qt@5) module load gcc/10.3.0 module load PrgEnv-gnu/8.3.3 -ecflow - ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library. After installing `qt5` and loading the following modules, follow the instructions in :numref:`Section %s `. Because of the dependency on ``miniconda``, that module must be loaded automatically in the ``ecflow`` module (similar to ``qt@5.15.2-c5``). Ensure to follow the extra instructions in that section for Gaea C5 in ``/ncrc/proj/epic/spack-stack/ecflow-5.8.4``. - - Ensure to follow the extra instructions in that section for Gaea. - -.. code-block:: console - - module load PrgEnv-gnu/8.3.3 - module use /ncrc/proj/epic/spack-stack/modulefiles/ - module load qt/5.15.2 - module load python/3.9.12 - module load cmake/3.23.1 - .. _MaintainersSection_Hera: ------------------------------ NOAA RDHPCS Hera ------------------------------ -On Hera, ``miniconda`` must be installed as a one-off before spack can be used. When using the GNU compiler, it is also necessary to build your own ``openmpi`` or other MPI library. - -miniconda - Follow the instructions in :numref:`Section %s ` to create a basic ``miniconda`` installation and associated modulefile for working with spack. Don't forget to log off and back on to forget about the conda environment. - -openmpi - It is easier to build and test ``openmpi`` manually and use it as an external package, instead of building it as part of spack-stack. These instructions were used to build the ``openmpi@4.1.5`` MPI library with ``gcc@9.2.0`` as referenced in the Hera site config. After the installation, create modulefile `openmpi/4.1.5` using the template ``doc/modulefile_templates/openmpi``. Note the site-specific module settings at the end of the template, this will likely be different for other HPCs. - -.. code-block:: console - - module purge - module load gnu/9.2.0 - ./configure \ - --prefix=/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/openmpi-4.1.5 \ - --with-pmi=/apps/slurm/default \ - --with-lustre - make VERBOSE=1 -j4 - make check - make install - Hera sits behind the NOAA firewall and doesn't have access to all packages on the web. It is therefore necessary to create a spack mirror on another platform. This can be done as described in section :numref:`Section %s ` for air-gapped systems. .. _MaintainersSection_Jet: @@ -575,42 +268,9 @@ Hera sits behind the NOAA firewall and doesn't have access to all packages on th NOAA RDHPCS Jet ------------------------------ -Note that the ``target`` architecture for Jet must be set to ``core2`` to satisfy differences between the various Jet partitions and ensure that installations run on the front-end nodes (xjet-like) will function on the other partitions. - -miniconda - Follow the instructions in :numref:`Section %s ` to create a basic ``miniconda`` installation and associated modulefile for working with spack. Don't forget to log off and back on to forget about the conda environment. - -.. code-block:: console - - module use /lfs4/HFIP/hfv3gfs/spack-stack/modulefiles - module load miniconda/3.9.12 - # Need a newer gcc compiler than the default OS compiler gcc-4.8.5 - module load gnu/9.2.0 - -.. _MaintainersSection_S4: - ------------------------------- -UW (Univ. of Wisconsin) S4 ------------------------------- - -gnu (module only) - The ``gnu/9.3.0`` module provided by the system administrators is broken. To create a usable version, turn ``/data/prod/hpc-stack/modulefiles/core/gnu/9.3.0.lua`` into a simple environment module (``tcl``) in ``/data/prod/jedi/spack-stack/modulefiles/gnu``. - -mpich (module only) - The ``mpich/4.0.1`` module provided by the system administrators is broken. To create a usable version, turn ``/data/prod/hpc-stack/modulefiles/compiler/gnu/9.3.0/mpich/4.0.1.lua`` into a simple environment module (``tcl``) in ``/data/prod/jedi/spack-stack/modulefiles/mpich``. +On Jet, the ``target`` architecture must be set to ``core2`` to satisfy differences between the various Jet partitions and ensure that installations run on the front-end nodes (xjet-like) will function on the other partitions. -miniconda - Follow the instructions in :numref:`Section %s ` to create a basic ``miniconda`` installation and associated modulefile for working with spack. Don't forget to log off and back on to forget about the conda environment. - -ecflow - ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library. After installing `miniconda`, and loading the following modules, follow the instructions in :numref:`Section %s `. - -.. code-block:: console - - module purge - module use /data/prod/jedi/spack-stack/modulefiles - module load miniconda/3.9.12 - module load gcc/9.3.0 +Like Hera, Jet sits behind the NOAA firewall and doesn't have access to all packages on the web. It is therefore necessary to create a spack mirror on another platform. This can be done as described in section :numref:`Section %s ` for air-gapped systems. .. _MaintainersSection_AWS_Pcluster_Ubuntu: @@ -620,6 +280,7 @@ Amazon Web Services Parallelcluster Ubuntu 20.04 See ``configs/sites/aws-pcluster/README.md``. + .. _MaintainersSection_Testing_New_Packages: .. _MaintainersSection_spack_mirrors: diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 1fcf8ab7b..c4962ecbc 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -64,10 +64,12 @@ Pre-configured sites (tier 1) | | Jet | GCC, Intel | ``/contrib/spack-stack`` | EPIC / NOAA-EMC | +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ | | Narwhal | GCC, Intel, oneAPI | ``/p/app/projects/NEPTUNE/spack-stack/`` | NRL | -| U.S. Navy (HPCMP) +-----------------------+--------------------+--------------------------------------------------------+-----------------+ -| | Nautilus | GCC, Intel, oneAPI | ``/p/app/projects/NEPTUNE/spack-stack/`` | NRL | +| +-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| U.S. Navy (HPCMP) | Nautilus | GCC, Intel, oneAPI | ``/p/app/projects/NEPTUNE/spack-stack/`` | NRL | +| +-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| | Blueback (earlyaccess)| GCC, oneAPI | (experimental only, no project directories yet | NRL | +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ -| Univ. of Wisconsin | S4 | Intel | ``/data/prod/jedi/spack-stack/`` | JCSDA | +| Univ. of Wisconsin | S4 | Intel | ``/data/prod/jedi/spack-stack/`` | SSEC | +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ | **Cloud platforms** | +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ @@ -77,6 +79,9 @@ Pre-configured sites (tier 1) +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ | NOAA (RDHPCS) | RDHPCS Parallel Works | Intel | ``/contrib/spack-stack-rocky8/`` | EPIC / JCSDA | +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| U.S. Navy (HPCMP) | HPCMP Parallel Works | GCC | ``/contrib/spack-stack/`` | NRL | ++---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ + .. _Preconfigured_Sites_Orion: @@ -230,6 +235,59 @@ The following is required for building new spack environments with any supported module purge +.. _Preconfigured_Sites_Blueback: + +------------------------------ +NAVY HPCMP Blueback +------------------------------ + +The following is required for building new spack environments with Intel oneAPI on this platform.. Don't use ``module purge`` on Blueback! + +.. code-block:: console + + umask 0022 + module unload PrgEnv-cray + module load PrgEnv-intel/8.4.0 + module unload intel + module load intel/2024.2 + module unload cray-mpich + module unload craype-network-ofi + # Warning. Do not load craype-network-ucx + # or cray-mpich-ucx/8.1.21! + # There is a bug in the modulefile that prevents + # spack from setting the environment for its + # build steps when the module is already + # loaded. Instead, let spack load it when the + # package requires it. + #module load craype-network-ucx + #module load cray-mpich-ucx/8.1.21 + module load libfabric/1.12.1.2.2.1 + module unload cray-libsci + module load cray-libsci/23.05.1.4 + +The following is required for building new spack environments with GNU on this platform.. Don't use ``module purge`` on Blueback! + + umask 0022 + module unload PrgEnv-cray + module load PrgEnv-gnu/8.4.0 + module unload gcc + module load gcc/12.1.0 + module unload cray-mpich + module unload craype-network-ofi + # Warning. Do not load craype-network-ucx + # or cray-mpich-ucx/8.1.21! + # There is a bug in the modulefile that prevents + # spack from setting the environment for its + # build steps when the module is already + # loaded. Instead, let spack load it when the + # package requires it. + #module load craype-network-ucx + #module load cray-mpich-ucx/8.1.21 + module load libfabric/1.12.1.2.2.1 + module unload cray-libsci + module load cray-libsci/23.05.1.4 + + .. _Preconfigured_Sites_Derecho: -------------------- @@ -262,6 +320,7 @@ Note that for the installation using Intel 19, the system GCC, 7.5.0, is used on .. note:: System-wide ``spack`` software installations are maintained by NCO on this platform, which are not associated with spack-stack. + .. _Preconfigured_Sites_Parallel_Works: ---------------------------------------- @@ -275,6 +334,21 @@ The following is required for building new spack environments with any supported module purge +.. _Preconfigured_Sites_Parallel_Works_Navy: + +---------------------------------------- +U.S. Navy Parallel Works (AWS) +---------------------------------------- + +The following is required for building new spack environments with GNU on this platform. + +.. code-block:: console + + umask 0022 + module purge + scl enable gcc-toolset-13 bash + + .. _Preconfigured_Sites_Gaea_C5: ------------------------------ From 8b326424204bda54fac154312b5fe53d77279a04 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 19 Nov 2024 21:18:12 -0500 Subject: [PATCH 33/33] Add configuration for Intel oneAPI 2025.0.0 on Nautilus (#1382) Add configuration for Intel oneAPI 2025.0.0 on Nautilus. This is with ifx, since there is no more ifort in oneapi@2025.0.0. This configuration is currently commented out, since the default is and should be 2024.2.1 with ifort for now. This requires a bug fix in py-scipy in JCSDA/spack#483 (see there for more information and the upstream spack develop PR). --- configs/sites/tier1/nautilus/compilers.yaml | 25 +++++++++++++++++++ .../sites/tier1/nautilus/packages_oneapi.yaml | 19 ++++++++++++++ spack | 2 +- 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/configs/sites/tier1/nautilus/compilers.yaml b/configs/sites/tier1/nautilus/compilers.yaml index aff3f05dd..147614a13 100644 --- a/configs/sites/tier1/nautilus/compilers.yaml +++ b/configs/sites/tier1/nautilus/compilers.yaml @@ -59,6 +59,31 @@ compilers: append_path: CPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/opt/compiler/include/intel64' extra_rpaths: [] +#- compiler: +# spec: oneapi@2025.0.0 +# paths: +# cc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.0.0/compiler/2025.0/bin/icx +# cxx: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.0.0/compiler/2025.0/bin/icpx +# f77: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.0.0/compiler/2025.0/bin/ifx +# fc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.0.0/compiler/2025.0/bin/ifx +# flags: {} +# operating_system: rhel8 +# target: x86_64 +# modules: +# - slurm +# - tbb/2022.0 +# - compiler-rt/2025.0.0 +# - umf/0.9.0 +# - compiler-intel-llvm/2025.0.0 +# environment: +# prepend_path: +# PATH: '/opt/rh/gcc-toolset-11/root/usr/bin' +# CPATH: '/opt/rh/gcc-toolset-11/root/usr/include' +# LD_LIBRARY_PATH: '/opt/scyld/slurm/lib64:/opt/scyld/slurm/lib64/slurm:/opt/rh/gcc-toolset-11/root/usr/lib64:/opt/rh/gcc-toolset-11/root/usr/lib' +# MODULEPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2025.0.0/modulefiles' +# append_path: +# CPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2025.0.0/compiler/2025.0/opt/compiler/include/intel64' +# extra_rpaths: [] - compiler: spec: gcc@12.2.1 paths: diff --git a/configs/sites/tier1/nautilus/packages_oneapi.yaml b/configs/sites/tier1/nautilus/packages_oneapi.yaml index 53254442f..263c72bf9 100644 --- a/configs/sites/tier1/nautilus/packages_oneapi.yaml +++ b/configs/sites/tier1/nautilus/packages_oneapi.yaml @@ -1,8 +1,10 @@ packages: all: compiler:: [oneapi@2024.2.1,gcc@11.2.1] + #compiler:: [oneapi@2025.0.0, gcc@11.2.1] providers: mpi:: [intel-oneapi-mpi@2021.13] + #mpi:: [intel-oneapi-mpi@2021.14] mpi: buildable: False intel-oneapi-mpi: @@ -11,18 +13,30 @@ packages: prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 modules: - mpi/2021.13 + #- spec: intel-oneapi-mpi@2021.14%oneapi@2025.0.0 + # prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.0.0 + # modules: + # - mpi/2021.14 intel-oneapi-mkl: externals: - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.1 prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 modules: - mkl/2024.2 + #- spec: intel-oneapi-mkl@2025.0%oneapi@2025.0.0 + # prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.0.0 + # modules: + # - mkl/2025.0 intel-oneapi-tbb: externals: - spec: intel-oneapi-tbb@2021.13%oneapi@2024.2.1 prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 modules: - tbb/2021.13 + #- spec: intel-oneapi-tbb@2022.0%oneapi@2025.0.0 + # prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.0.0 + # modules: + # - tbb/2022.0 intel-oneapi-runtime: externals: - spec: intel-oneapi-runtime@2024.2.1%oneapi@2024.2.1 @@ -30,3 +44,8 @@ packages: modules: - tbb/2021.13 - compiler-rt/2024.2.1 + #- spec: intel-oneapi-runtime@2025.0.0%oneapi@2025.0.0 + # prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2025.0.0 + # modules: + # - tbb/2022.0 + # - compiler-rt/2025.0.0 diff --git a/spack b/spack index b6ef265ec..b274d591f 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit b6ef265ec4582ddca131534a9d314e233919ae69 +Subproject commit b274d591fc82d38609124e3fe6f8ef1753357947