Merge pull request #790 from climbfuji/feature/merge_rel150_into_develop

Merge release/1.5.0 into develop

.github/workflows/macos-ci-aarch64.yaml

@@ -111,6 +111,12 @@ jobs:
           echo "Packages in combined spack build caches:"
           spack buildcache list
 
+          # Workaround for limited disk space on macOS arm instance
+          spack config add "config:build_stage:/Users/ec2-user/spack-stack/spack-cache/build_stage"
+          spack config add "config:test_stage:/Users/ec2-user/spack-stack/spack-cache/test_stage"
+          spack config add "config:source_cache:/Users/ec2-user/spack-stack/spack-cache/source_cache"
+          spack config add "config:misc_cache:/Users/ec2-user/spack-stack/spack-cache/misc_cache"
+
           # Break installation up in pieces and create build caches in between
           # This allows us to "spin up" builds that altogether take longer than
           # six hours, and/or fail later in the build process.

configs/common/packages.yaml

@@ -11,13 +11,13 @@
         lapack: [openblas]
         yacc: [bison]
     #
+    # This version of awscli goes with [email protected]
     awscli:
-      version: ['1.29.41']
-    bacio:  
+      version: ['1.27.84']
+    bacio:
       version: ['2.4.1']
     bison:
       version: ['3.8.2']
-    # Attention - when updating also check orion site config
     boost:
       version: ['1.78.0']
       variants: ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden
@@ -103,7 +103,7 @@
       # the container builds.
       #version: ['2.11.0']
     grib-util:
-      version: ['1.2.3']
+      version: ['1.3.0']
     gsibec:
       version: ['1.1.3']
     gsi-ncdiag:
@@ -117,7 +117,7 @@
       version: ['1.14.0']
       variants: +hl +fortran +mpi ~threadsafe +szip
     ip:
-      version: ['3.3.3']
+      version: ['4.3.0']
     ip2:
       version: ['1.1.2']
     jasper:
@@ -137,8 +137,12 @@
       version: ['0.2.5']
     mapl:
       # 2.35.2 goes with [email protected], 2.40.3 goes with [email protected]
+      # turn off ~pflogger and extdata2g to avoid compilation
+      # errors with [email protected]+, see
+      # https://github.com/JCSDA/spack-stack/issues/769
+      # also: ... extdata2g segfault UFS?
       version: ['2.35.2']
-      variants: ~shared
+      variants: ~shared ~extdata2g ~pflogger
     # If making changes here, also check the Discover site config and the CI workflows
     met:
       version: ['11.1.0']
@@ -260,8 +264,9 @@
       # Versions earlier than 0.11.0 don't compile on macOS with llvm-clang/13.0.0 and Python/3.9,
       # and 0.11.0 leads to downstream errors in py-scipy with the Intel compilers
       version: ['0.12.2']
+    # This version of py-pyyaml goes with [email protected]
     py-pyyaml:
-      version: ['6.0']
+      version: ['5.4.1']
     py-scipy:
       version: ['1.9.3']
     # Pin the py-setuptools version to avoid duplicate Python packages
@@ -298,7 +303,7 @@
     upp:
       version: ['10.0.10']
     w3emc:
-      version: ['2.9.2']
+      version: ['2.10.0']
     w3nco:
       version: ['2.4.1']
     wget:

configs/containers/README.md

@@ -8,38 +8,21 @@ To avoid hardcoding specs in the generic container recipes, we keep the specs li
     [email protected], [email protected], [email protected], [email protected], [email protected], [email protected], ecflow@5,
     [email protected], [email protected] +trans ~fftw, [email protected], [email protected] ~fftw, [email protected],
     [email protected], fms@release-jcsda, [email protected], [email protected], [email protected],
-    [email protected], [email protected], [email protected].1-2, ip@3.3.3, [email protected], [email protected],
+    [email protected], [email protected], [email protected].0, ip@4.3.0, [email protected], [email protected],
     [email protected], [email protected], [email protected], [email protected],
     [email protected], [email protected], [email protected],
     [email protected], [email protected], [email protected], [email protected],
     [email protected], [email protected], [email protected],
-    [email protected], py-pip, [email protected], [email protected], [email protected], [email protected],
-    [email protected], [email protected], [email protected], [email protected], [email protected], [email protected],
-    [email protected], [email protected], [email protected], shumlib@macos_clang_linux_intel_port]
-    # Don't build CRTM by default so that it gets built in the JEDI bundles:
-    # [email protected]
-    # Comment out for now until build problems are solved
-    # https://github.com/jcsda/spack-stack/issues/522
-    # [email protected]
-```
-
-### spack-stack-1.5.0 / ufs-weather-model-x.y.z containers for ufs-weather-model as of July 5, 2023
-
-**Note. This is not yet working correctly, some libraries are missing. Please do not use yet! Also, if using the clang-mpich container, need to disable openmp for fms, not clear how to do this cleanly.**
-
-```
-  specs: [[email protected],
-    [email protected], [email protected], [email protected], [email protected], [email protected], ecflow@5,
-    [email protected], [email protected] +trans ~fftw, [email protected], [email protected] ~fftw, [email protected],
-    [email protected], [email protected], [email protected], [email protected], [email protected],
-    [email protected], [email protected], [email protected], [email protected], [email protected], [email protected],
-    [email protected], [email protected], [email protected], [email protected],
-    [email protected], [email protected], [email protected],
-    [email protected], [email protected], [email protected], [email protected],
-    [email protected], [email protected], [email protected], [email protected],
-    py-pip, [email protected], [email protected], [email protected], [email protected],
-    [email protected], [email protected], [email protected], [email protected], [email protected], [email protected], [email protected],
-    [email protected], [email protected], [email protected], [email protected], [email protected]_emc]
+    [email protected], py-pip, [email protected], [email protected], [email protected], [email protected],
+    [email protected], [email protected], [email protected], [email protected], [email protected], [email protected], [email protected],
+    [email protected], [email protected], [email protected], [email protected], shumlib@macos_clang_linux_intel_port,
+    [email protected], [email protected]]
+    # Notes:
+    # 1. Remove [email protected] from clang/mpich container, because mapl doesn't work with mpich@4
+    # 2. Don't build CRTM by default so that it gets built in the JEDI bundles
+    # 3. Comment out for now until build problems are solved
+    #    https://github.com/jcsda/spack-stack/issues/522
+    #    [email protected]
 ```
 
 ### Create an AMI on AWS EC2 to build docker containers

configs/containers/docker-ubuntu-clang-mpich.yaml

@@ -38,6 +38,11 @@ spack:
       externals:
       - spec: [email protected]
         prefix: /usr
+    gmake:
+      buildable: false
+      externals:
+      - spec: [email protected]
+        prefix: /usr
     diffutils:
       buildable: false
       externals:
@@ -105,7 +110,7 @@ spack:
       os: ubuntu:20.04
       spack:
         url: https://github.com/jcsda/spack
-        ref: jcsda_emc_spack_stack
+        ref: spack-stack-1.5.0
         resolve_sha: false
 
     # Whether or not to strip binaries
@@ -235,8 +240,6 @@ spack:
         echo "export CXX=clang++" >> /root/.bashrc && \
         echo "export FC=gfortran" >> /root/.bashrc && \
         printf "[credential]\n    helper = cache --timeout=7200\n" >> /root/.gitconfig
-        # Source spack environment script for root
-        RUN echo "source /etc/profile.d/z10_spack_environment.sh\n" >> /root/.bashrc
 
     # Labels for the image
     labels:

configs/containers/docker-ubuntu-gcc-openmpi.yaml

@@ -36,6 +36,12 @@ spack:
       externals:
       - spec: [email protected]
         prefix: /usr
+    # Try this next time, already used for clang/intel
+    #gmake:
+    #  buildable: false
+    #  externals:
+    #  - spec: [email protected]
+    #    prefix: /usr
     diffutils:
       buildable: false
       externals:
@@ -92,7 +98,7 @@ spack:
       os: ubuntu:20.04
       spack:
         url: https://github.com/jcsda/spack
-        ref: jcsda_emc_spack_stack
+        ref: spack-stack-1.5.0
         resolve_sha: false
 
     # Whether or not to strip binaries
@@ -184,8 +190,6 @@ spack:
         RUN echo "export OMPI_ALLOW_RUN_AS_ROOT=1" >> /root/.bashrc && \
         echo "export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1" >> /root/.bashrc && \
         echo "export OMPI_MCA_rmaps_base_oversubscribe=1" >> /root/.bashrc
-        # Source spack environment script for root
-        RUN echo "source /etc/profile.d/z10_spack_environment.sh\n" >> /root/.bashrc
 
     # Labels for the image
     labels:

configs/containers/docker-ubuntu-intel-impi.yaml

@@ -50,6 +50,11 @@ spack:
       externals:
       - spec: [email protected]
         prefix: /opt/intel/oneapi
+    gmake:
+      buildable: false
+      externals:
+      - spec: [email protected]
+        prefix: /usr
     diffutils:
       buildable: false
       externals:
@@ -110,7 +115,7 @@ spack:
       os: ubuntu:20.04
       spack:
         url: https://github.com/jcsda/spack
-        ref: jcsda_emc_spack_stack
+        ref: spack-stack-1.5.0
         resolve_sha: false
 
     # Whether or not to strip binaries
@@ -211,8 +216,6 @@ spack:
         echo "source /opt/intel/oneapi/compiler/latest/env/vars.sh" >> /root/.bashrc && \
         echo "source /opt/intel/oneapi/mpi/latest/env/vars.sh" >> /root/.bashrc && \
         printf "[credential]\n    helper = cache --timeout=7200\n" >> /root/.gitconfig
-        # Source spack environment script for root
-        RUN echo "source /etc/profile.d/z10_spack_environment.sh\n" >> /root/.bashrc
 
     # Labels for the image
     labels:

configs/sites/acorn/compilers.yaml

@@ -20,6 +20,27 @@ compilers:
         # which confuses some packages.
         CONFIG_SITE: ''
     extra_rpaths: []
+- compiler:
+    spec: [email protected]
+    paths:
+      cc: cc
+      cxx: CC
+      f77: ftn
+      fc: ftn
+    flags: {}
+    operating_system: sles15
+    modules:
+    - PrgEnv-intel/8.3.3
+    - craype/2.7.13
+    - intel-classic/2022.2.0.262
+    - libfabric
+    environment:
+      set:
+        # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so
+        # Automake-based builds are installed in lib64
+        # which confuses some packages.
+        CONFIG_SITE: ''
+    extra_rpaths: []
 - compiler:
     spec: [email protected]
     paths:

configs/sites/acorn/modules.yaml

@@ -2,3 +2,6 @@ modules:
   default:
     enable::
     - lmod
+    lmod:
+      include:
+      - python

configs/sites/acorn/packages.yaml

@@ -1,6 +1,6 @@
   packages:
     all:
-      compiler:: [[email protected],[email protected]]
+      compiler:: [[email protected],[email protected],[email protected]]
       providers:
         mpi:: [[email protected]]
     cray-mpich:
@@ -22,13 +22,6 @@
       externals:
       - spec: [email protected]
         modules: [git-lfs/2.11.0]
-    python:
-      buildable: false
-      externals:
-      - spec: [email protected]%intel
-        prefix: /apps/spack/python/3.8.6/intel/19.1.3.304/pjn2nzkjvqgmjw4hmyz43v5x4jbxjzpk
-      - spec: [email protected]%gcc
-        prefix: /apps/spack/python/3.8.6/gcc/11.2.0/gr4zqejk5oxwsrmxvkenmye4nni4zt6e
     perl:
       buildable: false
       externals:
@@ -61,3 +54,7 @@
       externals:
       - spec: [email protected]+lex
         prefix: /usr
+    eckit:
+      require:
+      - any_of: ["@1.23.1"]
+        when: "%[email protected]"

configs/sites/aws-pcluster/modules.yaml

@@ -4,7 +4,6 @@ modules:
     - lmod
     lmod:
       include:
-      # List of packages for which we need modules that are blacklisted by default
       - python
       exclude:
       - ecflow

configs/sites/aws-pcluster/packages.yaml

@@ -43,21 +43,11 @@ packages:
     externals:
     - spec: [email protected]
       prefix: /usr
-  bison:
-    externals:
-    - spec: [email protected]
-      prefix: /usr
   boost:
     buildable: False
     externals:
     - spec: [email protected] +atomic +chrono +date_time +exception +filesystem +graph +iostreams +locale +log +math +mpi +numpy +pic +program_options +python +random +regex +serialization +signals +system +test +thread +timer +wave cxxstd=17 visibility=hidden
       prefix: /usr
-  # Don't use external cmake, this confuses the spack concretizer
-  # leading to duplicate packages being installed.
-  #cmake:
-  #  externals:
-  #  - spec: [email protected]
-  #    prefix: /usr
   coreutils:
     externals:
     - spec: [email protected]
@@ -120,13 +110,6 @@ packages:
     externals:
     - spec: [email protected]
       prefix: /usr
-  # Don't use external openssl, this confuses the spack concretizer
-  # leading to duplicate packages (libarchive with different compression
-  # settings, somehow triggered by different cmake versions) being installed.
-  #openssl:
-  #  externals:
-  #  - spec: [email protected]
-  #    prefix: /usr
   perl:
     externals:
     - spec: [email protected]~cpanm+shared+threads

configs/sites/casper/compilers.yaml

@@ -1,23 +1,24 @@
 compilers::
   - compiler:
-      spec: intel@19.1.1.217
+      spec: intel@2022.0.2
       paths:
-        cc: /glade/u/apps/opt/intel/2020u1/compilers_and_libraries/linux/bin/intel64/icc
-        cxx: /glade/u/apps/opt/intel/2020u1/compilers_and_libraries/linux/bin/intel64/icpc
-        f77: /glade/u/apps/opt/intel/2020u1/compilers_and_libraries/linux/bin/intel64/ifort
-        fc: /glade/u/apps/opt/intel/2020u1/compilers_and_libraries/linux/bin/intel64/ifort
+        cc: /glade/u/apps/opt/intel/2022.1/compiler/2022.0.2/linux/bin/intel64/icc
+        cxx: /glade/u/apps/opt/intel/2022.1/compiler/2022.0.2/linux/bin/intel64/icpc
+        f77: /glade/u/apps/opt/intel/2022.1/compiler/2022.0.2/linux/bin/intel64/ifort
+        fc: /glade/u/apps/opt/intel/2022.1/compiler/2022.0.2/linux/bin/intel64/ifort
       flags: {}
       operating_system: centos7
       target: x86_64
       modules:
-      - intel/19.1.1
+      - intel/2022.1
       environment:
         prepend_path:
-          PATH: '/glade/u/apps/dav/opt/gnu/9.1.0/bin'
-          CPATH: '/glade/u/apps/dav/opt/gnu/9.1.0/include'
-          LD_LIBRARY_PATH: '/glade/u/apps/opt/intel/2020u1/compilers_and_libraries/linux/lib/intel64_lin:/glade/u/apps/opt/intel/2020u1/compilers_and_libraries_2020.1.217/linux/mpi/intel64/libfabric/lib:/glade/u/apps/dav/opt/gnu/9.1.0/lib64'
+          PATH: '/glade/work/epicufsrt/contrib/spack-stack/casper/gcc-10.1.0/bin'
+          CPATH: '/glade/work/epicufsrt/contrib/spack-stack/casper/gcc-10.1.0/include'
+          LD_LIBRARY_PATH: '/glade/u/apps/opt/intel/2022.1/compiler/latest/linux/compiler/lib/intel64_lin:/glade/u/apps/opt/intel/2022.1/mpi/2021.5.1/libfabric/lib:/glade/work/epicufsrt/contrib/spack-stack/casper/gcc-10.1.0/lib64'
         set:
           INTEL_LICENSE_FILE: '[email protected]'
           LM_LICENSE_FILE: '[email protected]'
-          I_MPI_ROOT: '/glade/u/apps/opt/intel/2020u1/impi/2019.7.217/intel64'
+          I_MPI_ROOT: '/glade/u/apps/opt/intel/2022.1/mpi/2021.5.1'
+
       extra_rpaths: []

configs/sites/casper/modules.yaml

@@ -5,3 +5,5 @@ modules:
     lmod:
       exclude:
       - ecflow
+      include:
+      - python