diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml index 3087b7f1f..e905915f2 100644 --- a/.github/workflows/macos-ci-aarch64.yaml +++ b/.github/workflows/macos-ci-aarch64.yaml @@ -20,7 +20,9 @@ jobs: run: | pwd ls -lart + set +e find ./* -type d -exec chmod u+xw {} \; + set -e rm -fr * - name: checkout diff --git a/.github/workflows/ubuntu-ci-containers-x86_64.yaml b/.github/workflows/ubuntu-ci-containers-x86_64.yaml index b9265c094..b69d0d324 100644 --- a/.github/workflows/ubuntu-ci-containers-x86_64.yaml +++ b/.github/workflows/ubuntu-ci-containers-x86_64.yaml @@ -8,7 +8,7 @@ on: # - '**.md' # - '.github/ISSUE_TEMPLATE/*' # - '.gitignore' - # + # schedule: - cron: '0 8 * * *' workflow_dispatch: diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml index 13414615c..3b12d1c11 100644 --- a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml @@ -20,7 +20,9 @@ jobs: run: | pwd ls -lart + set +e find ./* -type d -exec chmod u+xw {} \; + set -e rm -fr * - name: checkout diff --git a/.github/workflows/ubuntu-ci-x86_64-intel.yaml b/.github/workflows/ubuntu-ci-x86_64-intel.yaml index d1ed33d65..eed14ce5b 100644 --- a/.github/workflows/ubuntu-ci-x86_64-intel.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-intel.yaml @@ -20,7 +20,9 @@ jobs: run: | pwd ls -lart + set +e find ./* -type d -exec chmod u+xw {} \; + set -e rm -fr * - name: checkout diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml index e2bf9e604..0b4f7d337 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml @@ -20,7 +20,9 @@ jobs: run: | pwd ls -lart + set +e find ./* -type d -exec chmod u+xw {} \; + set -e rm -fr * - name: checkout diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 4e78711ea..fc7089741 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -54,7 +54,7 @@ packages: # Also, check the acorn and derecho site configs which have esmf modifications. esmf: require: - - '~xerces ~pnetcdf +shared +external-parallelio' + - '~xerces ~pnetcdf +shared +external-parallelio +python' - any_of: ['@=8.6.1 snapshot=none', '@=8.7.0b11 snapshot=b11'] - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"'] when: "%intel" @@ -75,8 +75,9 @@ packages: # Pin version to avoid duplicates require: '@2.6.4' fms: - require: '@2024.02' - variants: 'precision=32,64 +quad_precision +gfs_phys +openmp +pic constants=GFS build_type=Release +deprecated_io' + require: + - '@2024.02' + - 'precision=32,64 +quad_precision +gfs_phys +openmp +pic constants=GFS build_type=Release +deprecated_io' fontconfig: require: '+pic' freetype: @@ -214,6 +215,9 @@ packages: ## To avoid duplicate packages #py-jinja2: # require: '@3.1.2' + # https://github.com/JCSDA/spack-stack/issues/1276 + py-matplotlib: + require: '@3.7.4' py-meson-python: require: '@0.15.0' # Comment out for now until build problems are solved @@ -226,13 +230,17 @@ packages: require: '@1.5.8 ~mpi' # py-numpy@1.26 causes many build problems with older Python packages # also check Nautilus site config when making changes here + # https://github.com/JCSDA/spack-stack/issues/1276 py-numpy: require: - - '@:1.25' + - '@:1.23.5' py-pandas: require: '+excel' py-pybind11: require: '@2.11.0' + # To avoid duplicate packages + py-ruamel-yaml: + require: '@0.17.16' # Pin the py-setuptools version to avoid duplicate Python packages py-setuptools: require: '@63.4.3' @@ -270,7 +278,8 @@ packages: w3nco: require: '@2.4.1' wgrib2: - require: '@3.1.1' + require: + - '@3.1.1' wrf-io: require: '@1.2.0' zstd: diff --git a/configs/common/packages_aocc.yaml b/configs/common/packages_aocc.yaml new file mode 100644 index 000000000..70012d1a0 --- /dev/null +++ b/configs/common/packages_aocc.yaml @@ -0,0 +1,15 @@ +packages: + all: + providers: + blas: [openblas] + fftw-api: [fftw] + lapack: [openblas] + ectrans: + require: + - '~mkl +fftw' + gsibec: + require: + - '~mkl' + py-numpy: + require: + - '^openblas' diff --git a/configs/containers/docker-ubuntu-clang-mpich.yaml b/configs/containers/docker-ubuntu-clang-mpich.yaml index f55603577..6ac7f83e8 100644 --- a/configs/containers/docker-ubuntu-clang-mpich.yaml +++ b/configs/containers/docker-ubuntu-clang-mpich.yaml @@ -42,12 +42,6 @@ spack: externals: - spec: gcc-runtime@12.3.0 prefix: /usr - g2: - # https://github.com/NOAA-EMC/NCEPLIBS-g2/issues/745 - require: - - any_of: ['fflags="-fno-range-check"'] - when: "@3.5.1 %clang" - message: "Extra g2 compile options for version 3.5.1 with clang" g2tmpl: # https://github.com/JCSDA/spack-stack/issues/525 require: diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index d7bfe3f1b..f0e495073 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -45,12 +45,6 @@ spack: externals: - spec: gmake@4.3 prefix: /usr - g2: - # https://github.com/NOAA-EMC/NCEPLIBS-g2/issues/745 - require: - - any_of: ['fflags="-fno-range-check"'] - when: "@3.5.1 %gcc" - message: "Extra g2 compile options for version 3.5.1 with gcc/gfortran" diffutils: buildable: false externals: diff --git a/configs/containers/specs/jedi-ci.yaml b/configs/containers/specs/jedi-ci.yaml index 6b110ea7d..dea11f800 100644 --- a/configs/containers/specs/jedi-ci.yaml +++ b/configs/containers/specs/jedi-ci.yaml @@ -38,7 +38,7 @@ py-f90nml@1.4.3, py-gitpython@3.1.40, py-h5py@3.11.0, - py-numpy@1.25.2, + py-numpy@1.23.5, py-pandas@1.5.3, py-pip, py-pyyaml@6.0, diff --git a/configs/sites/tier1/acorn/compilers.yaml b/configs/sites/tier1/acorn/compilers.yaml index e118f32c3..179fcbdbf 100644 --- a/configs/sites/tier1/acorn/compilers.yaml +++ b/configs/sites/tier1/acorn/compilers.yaml @@ -6,18 +6,17 @@ compilers: cxx: CC f77: ftn fc: ftn - flags: {} + flags: + cflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread + cxxflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread + fflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread operating_system: sles15 modules: - - PrgEnv-intel/8.3.3 - - craype/2.7.13 + - PrgEnv-intel/8.5.0 + - craype/2.7.17 - intel/19.1.3.304 - libfabric environment: - prepend_path: - PATH: /opt/cray/pe/gcc/10.2.0/bin - LD_LIBRARY_PATH: /opt/cray/pe/gcc/10.2.0/snos/lib64 - CPATH: /opt/cray/pe/gcc/10.2.0/snos/include set: # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so # Automake-based builds are installed in lib64 @@ -31,11 +30,14 @@ compilers: cxx: CC f77: ftn fc: ftn - flags: {} + flags: + cflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread + cxxflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread + fflags: -static-libgcc -static-libstdc++ -Bstatic -lstdc++ -Bdynamic -lm -lpthread operating_system: sles15 modules: - - PrgEnv-intel/8.3.3 - - craype/2.7.13 + - PrgEnv-intel/8.5.0 + - craype/2.7.17 - intel-classic/2022.2.0.262 - libfabric environment: diff --git a/configs/sites/tier1/acorn/packages.yaml b/configs/sites/tier1/acorn/packages.yaml index fcd79f254..6bd55d67c 100644 --- a/configs/sites/tier1/acorn/packages.yaml +++ b/configs/sites/tier1/acorn/packages.yaml @@ -9,11 +9,6 @@ externals: - spec: git-lfs@2.11.0 modules: [git-lfs/2.11.0] - perl: - buildable: false - externals: - - spec: perl@5.26.1~cpanm+shared+threads - prefix: /usr mysql: buildable: false externals: @@ -38,6 +33,7 @@ gdal: variants: ~curl flex: + buildable: false externals: - spec: flex@2.6.4+lex prefix: /usr diff --git a/configs/sites/tier1/acorn/packages_intel.yaml b/configs/sites/tier1/acorn/packages_intel.yaml index 34065c870..b9cfc096c 100644 --- a/configs/sites/tier1/acorn/packages_intel.yaml +++ b/configs/sites/tier1/acorn/packages_intel.yaml @@ -31,7 +31,10 @@ - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.25 ^openblas' + - '^[virtuals=lapack,blas] openblas' + - '@:1.25' + - any_of: ['@:1.24'] + when: '%intel@19.1.3.304' # *DH eckit: require: @@ -39,13 +42,18 @@ when: "%intel@19.1.3.304" eccodes: require: - - any_of: ["@2.27.0"] + - any_of: ["@2.25.0"] when: "%intel@19.1.3.304" + message: "2.25.0 is the last version to use C++11 (as opposed to C++17)" py-scipy: require: - any_of: ["@1.10.1"] when: "%intel@19.1.3.304" cdo: require: + - any_of: ["@2.3.0"] + when: "%intel@2022.0.2.262" + message: "2.3.0 is the last version to use C++17" - any_of: ["@2.0.5"] when: "%intel@19.1.3.304" + message: "2.0.5 is the last version to use C++14" diff --git a/configs/sites/tier1/atlantis/compilers.yaml b/configs/sites/tier1/atlantis/compilers.yaml index 15e8de396..d6ffdec29 100644 --- a/configs/sites/tier1/atlantis/compilers.yaml +++ b/configs/sites/tier1/atlantis/compilers.yaml @@ -54,4 +54,3 @@ compilers: append_path: CPATH: /cm/shared/apps/intel/oneapi-2024.2.1/compiler/2024.2/opt/compiler/include/intel64/ extra_rpaths: [] -# Todo: Define gcc compiler (default for Atlantis) for Intel (ecflow) diff --git a/configs/sites/tier1/atlantis/modules.yaml b/configs/sites/tier1/atlantis/modules.yaml index e5e02cbd2..11c5514e2 100644 --- a/configs/sites/tier1/atlantis/modules.yaml +++ b/configs/sites/tier1/atlantis/modules.yaml @@ -6,5 +6,3 @@ modules: include: # List of packages for which we need modules that are blacklisted by default - python - exclude: - - ecflow diff --git a/configs/sites/tier1/atlantis/packages_oneapi.yaml b/configs/sites/tier1/atlantis/packages_oneapi.yaml index 191223298..95ed8242f 100644 --- a/configs/sites/tier1/atlantis/packages_oneapi.yaml +++ b/configs/sites/tier1/atlantis/packages_oneapi.yaml @@ -2,12 +2,12 @@ packages: all: compiler:: [oneapi@2024.2.1,gcc@11.2.0] providers: - mpi:: [intel-oneapi-mpi@2021.13.0] + mpi:: [intel-oneapi-mpi@2021.13] mpi: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.13%2024.2.1 + - spec: intel-oneapi-mpi@2021.13%oneapi@2024.2.1 prefix: /cm/shared/apps/intel/oneapi-2024.2.1 modules: - mpi/2021.13 diff --git a/configs/sites/tier1/aws-pcluster/README.md b/configs/sites/tier1/aws-pcluster/README.md index 5c3c0d86e..b1ee5e1c0 100644 --- a/configs/sites/tier1/aws-pcluster/README.md +++ b/configs/sites/tier1/aws-pcluster/README.md @@ -116,7 +116,7 @@ apt install -y \ liblcms2-dev \ liblcms2-2 -# Install QT5 for ecflow +# Install QT5, needed for install of ecflow apt install -y \ qtcreator \ qtbase5-dev \ @@ -217,51 +217,7 @@ sudo usermod -aG docker $USER exit ``` -4. Build ecflow outside of spack to be able to link against OS boost -``` -mkdir -p /home/ubuntu/jedi/ecflow-5.8.4/src -cd /home/ubuntu/jedi/ecflow-5.8.4/src -wget https://github.com/ecmwf/ecflow/archive/refs/tags/5.8.4.tar.gz -tar -xvzf 5.8.4.tar.gz -export WK=/home/ubuntu/jedi/ecflow-5.8.4/src/ecflow-5.8.4 -export BOOST_ROOT=/usr - -# Build ecFlow -cd $WK -mkdir build -cd build -cmake .. -DPython3_EXECUTABLE=/usr/bin/python3 -DENABLE_STATIC_BOOST_LIBS=OFF -DCMAKE_INSTALL_PREFIX=/home/ubuntu/jedi/ecflow-5.8.4 2>&1 | tee log.cmake -make -j10 2>&1 | tee log.make -make install 2>&1 | tee log.install - -# Create a modulefiles directory and the "ecflow/5.8.4" module file. -mkdir -p /home/ubuntu/jedi/modulefiles/ecflow - -cat << 'EOF' > /home/ubuntu/jedi/modulefiles/ecflow/5.8.4 -#%Module1.0 - -module-whatis "Provides an ecflow-5.8.4 server+ui installation for use with spack." - -conflict ecflow - -proc ModulesHelp { } { -puts stderr "Provides an ecflow-5.8.4 server+ui installation for use with spack." -} - -# Set this value -set ECFLOW_PATH "/home/ubuntu/jedi/ecflow-5.8.4" - -prepend-path PATH "${ECFLOW_PATH}/bin" -prepend-path LD_LIBRARY_PATH "${ECFLOW_PATH}/lib" -prepend-path LIBRARY_PATH "${ECFLOW_PATH}/lib" -prepend-path CPATH "${ECFLOW_PATH}/include" -prepend-path CMAKE_PREFIX_PATH "${ECFLOW_PATH}" -prepend-path PYTHONPATH "${ECFLOW_PATH}/lib/python3.10/site-packages" -EOF - -``` - -5. Install lmod. This step must be done as `root`. +4. Install lmod. This step must be done as `root`. ``` # Install lua/lmod manually because apt only has older versions # that are not compatible with the modern lua modules spack produces @@ -343,7 +299,7 @@ module purge module list ``` -6. Install msql community server +5. Install msql community server ``` cd /home/ubuntu/jedi mkdir -p mysql-8.0.31/src @@ -365,7 +321,7 @@ exit rm *.deb ``` -7. Option 1: Testing existing site config in spack-stack (skip steps +6. Option 1: Testing existing site config in spack-stack (skip steps 8-9 afterwards) this install is done directly on the NFS drive. If you are testing an update to the configuration, do this on the faster EBS volume (use a directory in /home/ubuntu) in order to ensure a faster build. Once you have @@ -398,7 +354,7 @@ spack module lmod refresh spack stack setup-meta-modules ``` -8. Option 2: Test configuring site from scratch +7. Option 2: Test configuring site from scratch ``` mkdir -p /home/ubuntu/jedi && cd /home/ubuntu/jedi git clone -b develop --recursive https://github.com/jcsda/spack-stack spack-stack @@ -450,15 +406,6 @@ cat << 'EOF' >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml prefix: /usr EOF -# Add external ecflow -cat << 'EOF' >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - ecflow: - buildable: False - externals: - - spec: ecflow@5.8.4 +ui - prefix: /home/ubuntu/jedi/ecflow-5.8.4 -EOF - spack compiler find --scope system export -n SPACK_SYSTEM_CONFIG_PATH @@ -479,12 +426,12 @@ spack config add "packages:all:compiler:[intel@2021.10.0, gcc@11.4.0]" # I_MPI_PMI_LIBRARY: '/opt/slurm/lib/libpmi.so' ``` -9. Option 2: To avoid duplicate library versions edit ``envs/unified-dev/site/packages.yaml`` +8. Option 2: To avoid duplicate library versions edit ``envs/unified-dev/site/packages.yaml`` and remove entries for meson, ninja, hdf5, cmake and remove the external `cmake` and `openssl` entries. -10. Concretize and install +9. Concretize and install ``` spack concretize 2>&1 | tee log.concretize.unified-env.001 ./util/show_duplicate_packages.py -d log.concretize.unified-env.001 @@ -493,7 +440,7 @@ spack module lmod refresh spack stack setup-meta-modules ``` -11. Test spack-stack installation using your favorite application. +10. Test spack-stack installation using your favorite application. ``` # Example given for building jedi-bundle @@ -515,12 +462,12 @@ make -j10 ctest ``` -12. (Optional) Remove test installs of spack-stack environments, if desired. +11. (Optional) Remove test installs of spack-stack environments, if desired. -13. Create the AMI for use in the AWS parallelcluster config. You can follow +12. Create the AMI for use in the AWS parallelcluster config. You can follow the official instructions for [Modifying an AWS ParallelCluster AMI](https://docs.aws.amazon.com/parallelcluster/latest/ug/building-custom-ami-v3.html#modify-an-aws-parallelcluster-ami-v3) -14. Use the install to build +13. Use the install to build ``` # Load the intel toolchain into your environment. @@ -551,7 +498,7 @@ make -j10 ctest ``` -15. Once the parallel cluster head node image is fully configured and tested you +14. Once the parallel cluster head node image is fully configured and tested you can create an AMI snapshot based on the configured instance using the [instructions](https://docs.aws.amazon.com/parallelcluster/latest/ug/building-custom-ami-v3.html) published by AWS. Included here is a short summary of those instructions. diff --git a/configs/sites/tier1/aws-pcluster/modules.yaml b/configs/sites/tier1/aws-pcluster/modules.yaml index d7e195ab6..65801706f 100644 --- a/configs/sites/tier1/aws-pcluster/modules.yaml +++ b/configs/sites/tier1/aws-pcluster/modules.yaml @@ -5,5 +5,3 @@ modules: lmod: include: - python - exclude: - - ecflow \ No newline at end of file diff --git a/configs/sites/tier1/aws-pcluster/packages.yaml b/configs/sites/tier1/aws-pcluster/packages.yaml index 17d5b64a6..6158d0119 100644 --- a/configs/sites/tier1/aws-pcluster/packages.yaml +++ b/configs/sites/tier1/aws-pcluster/packages.yaml @@ -19,11 +19,6 @@ packages: externals: - spec: diffutils@3.8 prefix: /usr - ecflow: - buildable: False - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /home/ubuntu/jedi/ecflow-5.8.4 findutils: externals: - spec: findutils@4.8.0 diff --git a/configs/sites/tier1/aws-pcluster/packages_intel.yaml b/configs/sites/tier1/aws-pcluster/packages_intel.yaml index 1363ff58f..7fb3d8477 100644 --- a/configs/sites/tier1/aws-pcluster/packages_intel.yaml +++ b/configs/sites/tier1/aws-pcluster/packages_intel.yaml @@ -31,5 +31,5 @@ packages: - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.25 ^openblas' + - '@:1.23 ^openblas' # *DH diff --git a/configs/sites/tier1/derecho/compilers.yaml b/configs/sites/tier1/derecho/compilers.yaml index 20cf2574c..db05f7f08 100644 --- a/configs/sites/tier1/derecho/compilers.yaml +++ b/configs/sites/tier1/derecho/compilers.yaml @@ -14,9 +14,9 @@ compilers:: - intel-classic/2023.2.1 environment: prepend_path: - PATH: '/opt/cray/pe/gcc/12.2.0/bin' - CPATH: '/opt/cray/pe/gcc/12.2.0/snos/include' - LD_LIBRARY_PATH: '/glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2023.2.1/compiler/2023.2.1/linux/compiler/lib/intel64_lin:/opt/cray/pe/gcc/12.2.0/snos/lib:/opt/cray/pe/gcc/12.2.0/lib64' + PATH: '/opt/cray/pe/gcc/11.2.0/snos/bin' + CPATH: '/opt/cray/pe/gcc/11.2.0/snos/include' + LD_LIBRARY_PATH: '/glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2023.2.1/compiler/2023.2.1/linux/compiler/lib/intel64_lin:/opt/cray/pe/gcc/11.2.0/snos/lib:/opt/cray/pe/gcc/11.2.0/snos/lib64' set: # https://github.com/JCSDA/spack-stack/issues/957 FI_CXI_RX_MATCH_MODE: 'hybrid' @@ -41,4 +41,23 @@ compilers:: # https://github.com/JCSDA/spack-stack/issues/957 FI_CXI_RX_MATCH_MODE: 'hybrid' extra_rpaths: [] +# Needed for intel +- compiler: + spec: gcc@11.2.0 + paths: + cc: /opt/cray/pe/gcc/11.2.0/bin/gcc + cxx: /opt/cray/pe/gcc/11.2.0/bin/g++ + f77: /opt/cray/pe/gcc/11.2.0/bin/gfortran + fc: /opt/cray/pe/gcc/11.2.0/bin/gfortran + flags: {} + operating_system: sles15 + target: x86_64 + modules: + - ncarenv/23.09 + - gcc/11.2.0 + environment: + set: + # https://github.com/JCSDA/spack-stack/issues/957 + FI_CXI_RX_MATCH_MODE: 'hybrid' + extra_rpaths: [] diff --git a/configs/sites/tier1/derecho/config.yaml b/configs/sites/tier1/derecho/config.yaml index 84fe05ab9..fd1f8aa47 100644 --- a/configs/sites/tier1/derecho/config.yaml +++ b/configs/sites/tier1/derecho/config.yaml @@ -1,2 +1,2 @@ config: - build_jobs: 8 + build_jobs: 3 diff --git a/configs/sites/tier1/derecho/packages.yaml b/configs/sites/tier1/derecho/packages.yaml index 8ea313601..7105a030a 100644 --- a/configs/sites/tier1/derecho/packages.yaml +++ b/configs/sites/tier1/derecho/packages.yaml @@ -7,6 +7,11 @@ packages: # Need to force spack to build python with ssl variant when using older openssl from OS python: require: "+ssl" + # py-scipy 1.12 does not build with intel on Derecho, pin to 1.11.4 + # use same version with gcc for consistency on the machine + py-scipy: + require: + - '@1.11.4' ### All other external packages listed alphabetically autoconf: externals: @@ -26,8 +31,8 @@ packages: prefix: /usr curl: externals: - - spec: curl@7.79.1+gssapi+ldap+nghttp2 - prefix: /usr + - spec: curl@8.1.2+gssapi+ldap+nghttp2 + prefix: /glade/u/apps/derecho/23.09/spack/opt/spack/curl/8.1.2/gcc/7.5.0/uq6y cvs: externals: - spec: cvs@1.12.13 @@ -36,13 +41,6 @@ packages: externals: - spec: diffutils@3.6 prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /glade/work/epicufsrt/contrib/spack-stack/derecho/ecflow-5.8.4 - modules: - - ecflow/5.8.4 findutils: externals: - spec: findutils@4.8.0 @@ -96,6 +94,11 @@ packages: externals: - spec: pkg-config@0.29.2 prefix: /usr + qt: + buildable: False + externals: + - spec: qt@5.14.2 + prefix: /glade/u/apps/derecho/23.09/spack/opt/spack/qt/5.14.2/gcc/7.5.0/eqcj sed: externals: - spec: sed@4.4 diff --git a/configs/sites/tier1/derecho/packages_intel.yaml b/configs/sites/tier1/derecho/packages_intel.yaml index f37ddcacf..06f45d62d 100644 --- a/configs/sites/tier1/derecho/packages_intel.yaml +++ b/configs/sites/tier1/derecho/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.10.0,gcc@12.2.0] + compiler:: [intel@2021.10.0,gcc@11.2.0] providers: mpi:: [cray-mpich@8.1.25] # Remove the next three lines to switch to intel-oneapi-mkl @@ -37,6 +37,6 @@ packages: - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.25 ^openblas' + - '@:1.23 ^openblas' # *DH diff --git a/configs/sites/tier1/discover-scu16/modules.yaml b/configs/sites/tier1/discover-scu16/modules.yaml index 4c530d42d..65801706f 100644 --- a/configs/sites/tier1/discover-scu16/modules.yaml +++ b/configs/sites/tier1/discover-scu16/modules.yaml @@ -3,7 +3,5 @@ modules: enable:: - lmod lmod: - exclude: - - ecflow include: - python diff --git a/configs/sites/tier1/discover-scu16/packages.yaml b/configs/sites/tier1/discover-scu16/packages.yaml index 4cfc9f64e..d47b8cbaa 100644 --- a/configs/sites/tier1/discover-scu16/packages.yaml +++ b/configs/sites/tier1/discover-scu16/packages.yaml @@ -38,13 +38,6 @@ packages: externals: - spec: diffutils@3.3 prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /discover/swdev/jcsda/spack-stack/ecflow-5.8.4 - modules: - - ecflow/5.8.4 file: externals: - spec: file@5.22 @@ -129,6 +122,7 @@ packages: - spec: pkg-config@0.28 prefix: /usr qt: + buildable: False externals: - spec: qt@5.15.2 prefix: /discover/swdev/jcsda/spack-stack/qt-5.15.2/5.15.2/gcc_64 diff --git a/configs/sites/tier1/discover-scu17/modules.yaml b/configs/sites/tier1/discover-scu17/modules.yaml index 4c530d42d..65801706f 100644 --- a/configs/sites/tier1/discover-scu17/modules.yaml +++ b/configs/sites/tier1/discover-scu17/modules.yaml @@ -3,7 +3,5 @@ modules: enable:: - lmod lmod: - exclude: - - ecflow include: - python diff --git a/configs/sites/tier1/discover-scu17/packages.yaml b/configs/sites/tier1/discover-scu17/packages.yaml index 086f342a5..906a2af31 100644 --- a/configs/sites/tier1/discover-scu17/packages.yaml +++ b/configs/sites/tier1/discover-scu17/packages.yaml @@ -46,13 +46,6 @@ packages: externals: - spec: diffutils@3.6 prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.11.4+ui+static_boost - prefix: /discover/swdev/jcsda/spack-stack/scu17/ecflow-5.11.4 - modules: - - ecflow/5.11.4 findutils: externals: - spec: findutils@4.8.0 @@ -119,6 +112,7 @@ packages: - spec: pkg-config@0.29.2 prefix: /usr qt: + buildable: False externals: - spec: qt@5.15.10 prefix: /usr/local/other/xpdf/4.04/Deps diff --git a/configs/sites/tier1/discover-scu17/packages_intel.yaml b/configs/sites/tier1/discover-scu17/packages_intel.yaml index 3512790ab..d971e2387 100644 --- a/configs/sites/tier1/discover-scu17/packages_intel.yaml +++ b/configs/sites/tier1/discover-scu17/packages_intel.yaml @@ -13,5 +13,5 @@ packages: - mpi/impi/2021.10.0 intel-oneapi-mkl: externals: - - spec: intel-oneapi-mkl@2032.2.0%intel@2021.10.0 + - spec: intel-oneapi-mkl@2023.2.0%intel@2021.10.0 prefix: /usr/local/intel/oneapi/2021 diff --git a/configs/sites/tier1/gaea-c5/compilers.yaml b/configs/sites/tier1/gaea-c5/compilers.yaml index e60d7fece..ad1fd416d 100644 --- a/configs/sites/tier1/gaea-c5/compilers.yaml +++ b/configs/sites/tier1/gaea-c5/compilers.yaml @@ -23,4 +23,16 @@ compilers: # which confuses some packages. CONFIG_SITE: '' extra_rpaths: [] -# Todo: Define gcc compiler 10.3.0 for Intel backend (ecflow) +- compiler: + spec: gcc@12.2.0 + paths: + cc: /opt/cray/pe/gcc/12.2.0/bin/gcc + cxx: /opt/cray/pe/gcc/12.2.0/bin/g++ + f77: /opt/cray/pe/gcc/12.2.0/bin/gfortran + fc: /opt/cray/pe/gcc/12.2.0/bin/gfortran + flags: {} + operating_system: sles15 + modules: + - gcc/12.2.0 + environment: {} + extra_rpaths: [] diff --git a/configs/sites/tier1/gaea-c5/packages.yaml b/configs/sites/tier1/gaea-c5/packages.yaml index da5173fad..6e13f710e 100644 --- a/configs/sites/tier1/gaea-c5/packages.yaml +++ b/configs/sites/tier1/gaea-c5/packages.yaml @@ -35,7 +35,7 @@ packages: - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.25 ^openblas' + - '@:1.23 ^openblas' # *DH ### All other external packages listed alphabetically diff --git a/configs/sites/tier1/gaea-c6/compilers.yaml b/configs/sites/tier1/gaea-c6/compilers.yaml index 1bca116f0..aac06161c 100644 --- a/configs/sites/tier1/gaea-c6/compilers.yaml +++ b/configs/sites/tier1/gaea-c6/compilers.yaml @@ -23,4 +23,16 @@ compilers: # which confuses some packages. CONFIG_SITE: '' extra_rpaths: [] -# Todo: Define gcc compiler 12.x.y for Intel backend (ecflow) +- compiler: + spec: gcc@12.3.0 + paths: + cc: /usr/bin/gcc-12 + cxx: /usr/bin/g++-12 + f77: /usr/bin/gfortran-12 + fc: /usr/bin/gfortran-12 + flags: {} + operating_system: sles15 + modules: + - gcc-native-mixed/12.3 + environment: {} + extra_rpaths: [] diff --git a/configs/sites/tier1/gaea-c6/packages.yaml b/configs/sites/tier1/gaea-c6/packages.yaml index 92f7d0a7c..75473846c 100644 --- a/configs/sites/tier1/gaea-c6/packages.yaml +++ b/configs/sites/tier1/gaea-c6/packages.yaml @@ -34,7 +34,7 @@ packages: - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.25 ^openblas' + - '@:1.23 ^openblas' # *DH ### All other external packages listed alphabetically diff --git a/configs/sites/tier1/hera/packages_intel.yaml b/configs/sites/tier1/hera/packages_intel.yaml index 4f8025c5f..5ac52dd8b 100644 --- a/configs/sites/tier1/hera/packages_intel.yaml +++ b/configs/sites/tier1/hera/packages_intel.yaml @@ -30,7 +30,7 @@ packages: - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.25 ^openblas' + - '@:1.23 ^openblas' # *DH zlib-ng: require: diff --git a/configs/sites/tier1/hercules/packages_intel.yaml b/configs/sites/tier1/hercules/packages_intel.yaml index 78339c1c5..eb4a22dc7 100644 --- a/configs/sites/tier1/hercules/packages_intel.yaml +++ b/configs/sites/tier1/hercules/packages_intel.yaml @@ -30,4 +30,4 @@ packages: - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.25 ^openblas' + - '@:1.23 ^openblas' diff --git a/configs/sites/tier1/jet/packages.yaml b/configs/sites/tier1/jet/packages.yaml index 665e659ac..04633817d 100644 --- a/configs/sites/tier1/jet/packages.yaml +++ b/configs/sites/tier1/jet/packages.yaml @@ -57,10 +57,6 @@ packages: externals: - spec: gawk@4.0.2 prefix: /usr - gettext: - externals: - - spec: gettext@0.19.8.1 - prefix: /usr ghostscript: externals: - spec: ghostscript@9.25 diff --git a/configs/sites/tier1/jet/packages_intel.yaml b/configs/sites/tier1/jet/packages_intel.yaml index d9cdd7209..cc0cac045 100644 --- a/configs/sites/tier1/jet/packages_intel.yaml +++ b/configs/sites/tier1/jet/packages_intel.yaml @@ -8,8 +8,7 @@ packages: intel-oneapi-mpi: externals: - spec: intel-oneapi-mpi@2021.5.1%intel@2021.5.0 - modules: - - impi/2022.1.2 + prefix: /apps/oneapi intel-oneapi-mkl: externals: - spec: intel-oneapi-mkl@2022.0.2%intel@2021.5.0 diff --git a/configs/sites/tier1/narwhal/modules.yaml b/configs/sites/tier1/narwhal/modules.yaml index f83e5dc2c..b134d3669 100644 --- a/configs/sites/tier1/narwhal/modules.yaml +++ b/configs/sites/tier1/narwhal/modules.yaml @@ -3,7 +3,5 @@ modules: enable:: - tcl tcl: - exclude: - - ecflow include: - python diff --git a/configs/sites/tier1/narwhal/packages.yaml b/configs/sites/tier1/narwhal/packages.yaml index 9351731bb..49fad5ebe 100644 --- a/configs/sites/tier1/narwhal/packages.yaml +++ b/configs/sites/tier1/narwhal/packages.yaml @@ -19,13 +19,6 @@ packages: externals: - spec: diffutils@3.6 prefix: /usr - ecflow:: - buildable: false - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /p/app/projects/NEPTUNE/spack-stack/ecflow-5.8.4-cray-python-3.9.7.1 - modules: - - ecflow/5.8.4-cray-python-3.9.7.1 findutils: externals: - spec: findutils@4.6.0 @@ -80,6 +73,7 @@ packages: - spec: pkg-config@0.29.2 prefix: /usr qt: + buildable: False externals: - spec: qt@5.15.2 prefix: /p/app/projects/NEPTUNE/spack-stack/qt-5.15.2/5.15.2/gcc_64 diff --git a/configs/sites/tier1/nautilus/compilers.yaml b/configs/sites/tier1/nautilus/compilers.yaml index c4460ac05..aff3f05dd 100644 --- a/configs/sites/tier1/nautilus/compilers.yaml +++ b/configs/sites/tier1/nautilus/compilers.yaml @@ -6,13 +6,11 @@ compilers: cxx: /p/app/compilers/amd/aocc/4.0.0/bin/clang++ f77: /p/app/compilers/amd/aocc/4.0.0/bin/flang fc: /p/app/compilers/amd/aocc/4.0.0/bin/flang - flags: - cflags: null - cxxflags: null - fflags: null + flags: {} operating_system: rhel8 target: x86_64 modules: + - slurm - amd/aocc/4.0.0 - amd/aocl/aocc/4.0 environment: {} @@ -28,6 +26,7 @@ compilers: operating_system: rhel8 target: x86_64 modules: + - slurm - intel/compiler/2022.0.2 environment: prepend_path: @@ -36,23 +35,29 @@ compilers: LD_LIBRARY_PATH: '/opt/scyld/slurm/lib64:/opt/scyld/slurm/lib64/slurm:/p/app/compilers/intel/oneapi/compiler/2022.0.2/linux/compiler/lib/intel64_lin:/opt/rh/gcc-toolset-11/root/usr/lib64:/opt/rh/gcc-toolset-11/root/usr/lib' extra_rpaths: [] - compiler: - spec: oneapi@2024.1.2 + spec: oneapi@2024.2.1 paths: - cc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/icx - cxx: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/icpx - f77: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/ifort - fc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/ifort + cc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/icx + cxx: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/icpx + f77: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/ifort + fc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/bin/ifort flags: {} operating_system: rhel8 target: x86_64 - modules: [] + modules: + - slurm + - tbb/2021.13 + - compiler-rt/2024.2.1 + - compiler/2024.2.1 + - ifort/2024.2.1 environment: prepend_path: PATH: '/opt/rh/gcc-toolset-11/root/usr/bin' CPATH: '/opt/rh/gcc-toolset-11/root/usr/include' LD_LIBRARY_PATH: '/opt/scyld/slurm/lib64:/opt/scyld/slurm/lib64/slurm:/opt/rh/gcc-toolset-11/root/usr/lib64:/opt/rh/gcc-toolset-11/root/usr/lib' + MODULEPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/modulefiles' append_path: - CPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/opt/compiler/include/intel64' + CPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1/compiler/2024.2/opt/compiler/include/intel64' extra_rpaths: [] - compiler: spec: gcc@12.2.1 @@ -65,6 +70,7 @@ compilers: operating_system: rhel8 target: x86_64 modules: + - slurm - scl/gcc-toolset-12 environment: {} extra_rpaths: [] @@ -80,6 +86,7 @@ compilers: operating_system: rhel8 target: x86_64 modules: + - slurm - scl/gcc-toolset-11 environment: {} extra_rpaths: [] diff --git a/configs/sites/tier1/nautilus/modules.yaml b/configs/sites/tier1/nautilus/modules.yaml index fb279947b..aeb254c2e 100644 --- a/configs/sites/tier1/nautilus/modules.yaml +++ b/configs/sites/tier1/nautilus/modules.yaml @@ -6,5 +6,3 @@ modules: include: # List of packages for which we need modules that are blacklisted by default - python - exclude: - - ecflow diff --git a/configs/sites/tier1/nautilus/packages.yaml b/configs/sites/tier1/nautilus/packages.yaml index f8dbc3501..2a07d88c3 100644 --- a/configs/sites/tier1/nautilus/packages.yaml +++ b/configs/sites/tier1/nautilus/packages.yaml @@ -26,13 +26,6 @@ packages: externals: - spec: diffutils@3.6 prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /p/app/projects/NEPTUNE/spack-stack/ecflow-5.8.4 - modules: - - ecflow/5.8.4 findutils: externals: - spec: findutils@4.6.0 @@ -84,6 +77,11 @@ packages: externals: - spec: pkgconf@1.4.2 prefix: /usr + qt: + buildable: False + externals: + - spec: qt@5.15.3 + prefix: /usr sed: externals: - spec: sed@4.5 diff --git a/configs/sites/tier1/nautilus/packages_aocc.yaml b/configs/sites/tier1/nautilus/packages_aocc.yaml new file mode 100644 index 000000000..380d51b6e --- /dev/null +++ b/configs/sites/tier1/nautilus/packages_aocc.yaml @@ -0,0 +1,35 @@ +packages: +# On Nautilus, use intel-oneapi-mkl as provider +# for blas, lapack, fftw-api with AOCC because of +# problems using openblas in downstream applications +# (e.g. py-pandas): +# ELF load command address/offset not properly aligned" when loading libopenblas.so +# https://github.com/OpenMathLib/OpenBLAS/wiki/Faq#ELFoffset + all: + compiler:: [aocc@4.0.0] + providers: + mpi:: [openmpi@4.1.6] + blas:: [intel-oneapi-mkl] + fftw-api:: [intel-oneapi-mkl] + lapack:: [intel-oneapi-mkl] + mpi: + buildable: False + openmpi: + externals: + - spec: openmpi@4.1.6%aocc@4.0.0~cuda~java~memchecker~static~wrapper-rpath fabrics=ucx schedulers=slurm + prefix: /p/app/penguin/openmpi/4.1.6/aocc-4.0.0 + modules: + - penguin/openmpi/4.1.6/aocc-4.0.0 + - penguin/mpi-vars/aocc + - slurm + openblas: + buildable: False + ectrans: + require:: + - '@1.2.0 +mkl ~fftw' + gsibec: + require:: + - '@1.2.1 +mkl' + py-numpy: + require:: + - '@:1.25 ^intel-oneapi-mkl' diff --git a/configs/sites/tier1/nautilus/packages_gcc.yaml b/configs/sites/tier1/nautilus/packages_gcc.yaml index 841ff05b1..723bfd315 100644 --- a/configs/sites/tier1/nautilus/packages_gcc.yaml +++ b/configs/sites/tier1/nautilus/packages_gcc.yaml @@ -31,4 +31,4 @@ packages: - '@1.2.1 +mkl' py-numpy: require:: - - '@:1.25 ^intel-oneapi-mkl' + - '@:1.23 ^intel-oneapi-mkl' diff --git a/configs/sites/tier1/nautilus/packages_oneapi.yaml b/configs/sites/tier1/nautilus/packages_oneapi.yaml index ce40a89e7..53254442f 100644 --- a/configs/sites/tier1/nautilus/packages_oneapi.yaml +++ b/configs/sites/tier1/nautilus/packages_oneapi.yaml @@ -1,19 +1,32 @@ packages: all: - compiler:: [oneapi@2024.1.2,gcc@11.2.1] + compiler:: [oneapi@2024.2.1,gcc@11.2.1] providers: - mpi:: [intel-oneapi-mpi@2021.12] + mpi:: [intel-oneapi-mpi@2021.13] mpi: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.12%oneapi@2024.1.2 - prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2 + - spec: intel-oneapi-mpi@2021.13%oneapi@2024.2.1 + prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 + modules: + - mpi/2021.13 intel-oneapi-mkl: externals: - - spec: intel-oneapi-mkl@2024.1%oneapi@2024.1.2 - prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2 + - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.1 + prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 + modules: + - mkl/2024.2 + intel-oneapi-tbb: + externals: + - spec: intel-oneapi-tbb@2021.13%oneapi@2024.2.1 + prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 + modules: + - tbb/2021.13 intel-oneapi-runtime: externals: - - spec: intel-oneapi-runtime@2024.1.2%oneapi@2024.1.2 - prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2 + - spec: intel-oneapi-runtime@2024.2.1%oneapi@2024.2.1 + prefix: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.2.1 + modules: + - tbb/2021.13 + - compiler-rt/2024.2.1 diff --git a/configs/sites/tier1/noaa-aws/README.md b/configs/sites/tier1/noaa-aws/README.md index 3fd415810..1910fb757 100644 --- a/configs/sites/tier1/noaa-aws/README.md +++ b/configs/sites/tier1/noaa-aws/README.md @@ -1,164 +1,51 @@ # Provisiong ParallelWorks AWS clusters -## Steps to perform when provisioning a cluster for the very first time - -This section provides step by step instructions for installing the basic packages (OS packages, external packages) for spack-stack. Following these steps ensures that the site configuration files in `configs/sites/noaa-aws` work out of the box. - -### Basic system packages -``` -sudo su -chmod 777 /contrib - -# The following three commands were necessary to fix failures accessing these repos -yum-config-manager --disable intel-clck-2019-repo -yum-config-manager --disable intel-hpc-platform -yum-config-manager --disable intelpython - -yum install -y qt5-qtbase-devel -yum install -y qt5-qtsvg-devel -yum install -y xorg-x11-xauth -yum install -y xorg-x11-apps -yum install -y perl-IPC-Cmd -yum install -y gettext-devel -yum install -y m4 -exit -``` -Create a script that can be added to the cluster resource config so that these packages get installed automatically when provisioning new clusters later: -``` -mkdir -p /contrib/admin -cat < /contrib/admin/basic_setup.sh -#!/bin/bash +## Steps to perform before installing spack-stack version 1.8.0 +sudo su - chmod 777 /contrib yum install -y qt5-qtbase-devel yum install -y qt5-qtsvg-devel -yum install -y xorg-x11-xauth -yum install -y xorg-x11-apps -yum install -y perl-IPC-Cmd -yum install -y gettext-devel -yum install -y m4 -EOF - -chmod a+x /contrib/admin/basic_setup.sh -``` - -### Create a mysql config for local R2D2 use (if applicable) -``` -sudo su -cat < /contrib/admin/my.cnf -[mysqld] -datadir=/mysql_local/data -socket=/mysql_local/data/mysql.sock -symbolic-links=0 -default-authentication-plugin=mysql_native_password - -[mysqld_safe] -log-error=/mysql_local/log/mariadb.log -pid-file=/mysql_local/run/mariadb.pid -EOF -chmod 644 /contrib/admin/my.cnf -exit -``` - -### Build external packages for spack-stack -``` -mkdir -p /contrib/spack-stack -mkdir /contrib/spack-stack/modulefiles -cd /contrib/spack-stack/ - -mkdir -p git-lfs-2.4.1/src -cd git-lfs-2.4.1/src -wget http://mirror.centos.org/centos/7/sclo/x86_64/rh/Packages/r/rh-git218-git-lfs-2.4.1-3.el7.x86_64.rpm -rpm2cpio rh-git218-git-lfs-2.4.1-3.el7.x86_64.rpm | cpio -idmv -mv opt/rh/rh-git218/root/usr/* .. -rm -fr opt -cd /contrib/spack-stack/modulefiles -mkdir git-lfs -# Create the modulefile from the template in doc/modulefile_templates -cd /contrib/spack-stack/ -mkdir -p mysql-8.0.31/src -cd mysql-8.0.31/src -ldd --version -wget https://dev.mysql.com/get/Downloads/MySQL-8.0/mysql-8.0.31-linux-glibc2.17-x86_64-minimal.tar.xz -tar -xvf mysql-8.0.31-linux-glibc2.17-x86_64-minimal.tar.xz -mv mysql-8.0.31-linux-glibc2.17-x86_64-minimal/* .. -rmdir mysql-8.0.31-linux-glibc2.17-x86_64-minimal -cd /contrib/spack-stack/modulefiles -mkdir mysql -# Create the modulefile from the template in doc/modulefile_templates -cd /contrib/spack-stack/ -mkdir -p cmake-3.27.2/src -cd cmake-3.27.2/src -wget https://github.com/Kitware/CMake/releases/download/v3.27.2/cmake-3.27.2-linux-x86_64.tar.gz -tar -xvzf cmake-3.27.2-linux-x86_64.tar.gz -mv cmake-3.27.2-linux-x86_64/* .. -rmdir cmake-3.27.2-linux-x86_64 -cd /contrib/spack-stack/modulefiles -mkdir cmake -# Create the modulefile from the template in doc/modulefile_templates +## Steps to install spack-stack version 1.8.0 -# Set up basic modules for building the external ecflow package -module unuse /opt/cray/craype/default/modulefiles -module unuse /opt/cray/modulefiles +sudo su - +chmod 777 /contrib module purge -module load gnu/9.2.0 -module use /contrib/spack-stack/modulefiles -module load cmake/3.27.2 - -mkdir -p /contrib/spack-stack/ecflow-5.8.4/src -cd /contrib/spack-stack/ecflow-5.8.4/src -wget https://confluence.ecmwf.int/download/attachments/8650755/ecFlow-5.8.4-Source.tar.gz?api=v2 -wget https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz -mv ecFlow-5.8.4-Source.tar.gz\?api\=v2 ecFlow-5.8.4-Source.tar.gz -tar -xvzf boost_1_78_0.tar.gz -tar -xvzf ecFlow-5.8.4-Source.tar.gz -export WK=/contrib/spack-stack/ecflow-5.8.4/src/ecFlow-5.8.4-Source -export BOOST_ROOT=/contrib/spack-stack/ecflow-5.8.4/src/boost_1_78_0 - -# Build static boost (to not interfere with spack-stack boost) -cd $BOOST_ROOT -./bootstrap.sh 2>&1 | tee bootstrap.log -$WK/build_scripts/boost_build.sh 2>&1 | tee boost_build.log - -# Build ecFlow -cd $WK -mkdir build -cd build -cmake .. -DCMAKE_INSTALL_PREFIX=/contrib/spack-stack/ecflow-5.8.4 2>&1 | tee log.cmake -make -j4 2>&1 | tee log.make -make install 2>&1 | tee log.install - -cd /contrib/spack-stack/modulefiles -mkdir ecflow -# Create the modulefile from the template in doc/modulefile_templates -``` - -## Steps to perform when starting a new cluster - -This should be done automatically, but doesn't hurt to run again just in case: -``` -sudo /contrib/admin/basic_setup.sh -``` -Configure `git` and `aws` command line utilities -``` module unuse /opt/cray/craype/default/modulefiles module unuse /opt/cray/modulefiles -module use /contrib/spack-stack/modulefiles -module load cmake/3.27.2 -module load ecflow/5.8.4 -module load mysql/8.0.31 -module load git-lfs/2.4.1 - -git lfs install -git config --global credential.helper cache # or store -git config --global user.email "your.name@domain.com" -git config --global user.name "Your Name" +### For noaa-aws, run the line below as well: +module unuse /opt/intel/impi/2019.5.281/intel64/modulefiles +module load gnu +module load intel/2023.2.0 +module load impi/2023.2.0 +module unload gnu + +cd /contrib/spack-stack-rocky8/ +git clone --recursive https://github.com/JCSDA/spack-stack -b release/1.8.0 spack-stack-1.8.0 +cd spack-stack-1.8.0 +. setup.sh +spack stack create env --name ue-intel-2021.10.0 --template unified-dev --site noaa-aws --compiler intel +cd envs/ue-intel-2021.10.0 +spack env activate . +spack concretize 2>&1 | tee log.concretize +spack install --verbose 2>&1 | tee log.install +spack module lmod refresh -y +spack stack setup-meta-modules + +## Steps to install GSI addon + +sudo su - +chmod 777 /contrib -mkdir ~/.aws -# Create ~/.aws/config -# Create ~/.aws/credentials -chmod 400 ~/.aws/credentials -``` +cd /contrib/spack-stack-rocky8/spack-stack-1.8.0 +. setup.sh +spack stack create env --name gsi-intel-2021.10.0 --template gsi-addon-dev --site noaa-aws --upstream /contrib/spack-stack-rocky8/spack-stack-1.8.0/envs/ue-intel-2021.10.0/install --compiler intel +cd envs/gsi-intel-2021.10.0 +spack env activate . +spack concretize 2>&1 | tee log.concretize +spack install --verbose 2>&1 | tee log.install +spack module lmod refresh --upstream-modules +spack stack setup-meta-modules diff --git a/configs/sites/tier1/noaa-aws/compilers.yaml b/configs/sites/tier1/noaa-aws/compilers.yaml index aec6336cf..343520e4c 100644 --- a/configs/sites/tier1/noaa-aws/compilers.yaml +++ b/configs/sites/tier1/noaa-aws/compilers.yaml @@ -1,20 +1,19 @@ compilers: - compiler: - spec: intel@2021.3.0 + spec: intel@2021.10.0 paths: - cc: /apps/oneapi/compiler/2021.3.0/linux/bin/intel64/icc - cxx: /apps/oneapi/compiler/2021.3.0/linux/bin/intel64/icpc - f77: /apps/oneapi/compiler/2021.3.0/linux/bin/intel64/ifort - fc: /apps/oneapi/compiler/2021.3.0/linux/bin/intel64/ifort - flags: {} - operating_system: centos7 + cc: /apps/oneapi/compiler/2023.2.0/linux/bin/intel64/icc + cxx: /apps/oneapi/compiler/2023.2.0/linux/bin/intel64/icpc + f77: /apps/oneapi/compiler/2023.2.0/linux/bin/intel64/ifort + fc: /apps/oneapi/compiler/2023.2.0/linux/bin/intel64/ifort + operating_system: rocky8 target: x86_64 modules: - - intel/2021.3.0 + - intel/2023.2.0 environment: prepend_path: - PATH: '/apps/gnu/gcc-9.2.0/bin' - LD_LIBRARY_PATH: '/apps/oneapi/compiler/2021.3.0/linux/compiler/lib/intel64_lin:/apps/gnu/gcc-9.2.0/lib64' + LD_LIBRARY_PATH: '/apps/oneapi/compiler/2023.2.0/linux/compiler/lib/intel64_lin:/apps/gnu/gcc-13.2.0/lib64' + flags: {} extra_rpaths: [] - compiler: spec: gcc@9.2.0 @@ -24,8 +23,7 @@ compilers: f77: /apps/gnu/gcc-9.2.0/bin/gfortran fc: /apps/gnu/gcc-9.2.0/bin/gfortran flags: {} - operating_system: centos7 - target: x86_64 + operating_system: rocky8 modules: - gnu/9.2.0 environment: {} diff --git a/configs/sites/tier1/noaa-aws/packages.yaml b/configs/sites/tier1/noaa-aws/packages.yaml index a16a87b90..ee153e8a5 100644 --- a/configs/sites/tier1/noaa-aws/packages.yaml +++ b/configs/sites/tier1/noaa-aws/packages.yaml @@ -1,115 +1,114 @@ packages: -### Modifications of common packages - # Pin flex to avoid duplicate packages - flex: - version: ['2.6.4'] -### All other external packages listed alphabetically bash: externals: - - spec: bash@4.2.46 + - spec: bash@4.4.20 prefix: /usr berkeley-db: externals: - - spec: berkeley-db@5.3.21 + - spec: berkeley-db@5.3.28 prefix: /usr cpio: externals: - - spec: cpio@2.11 + - spec: cpio@2.12 prefix: /usr diffutils: externals: - - spec: diffutils@3.3 + - spec: diffutils@3.6 prefix: /usr ecflow:: buildable: False externals: - spec: ecflow@5.8.4+ui+static_boost - prefix: /contrib/spack-stack/ecflow-5.8.4 + prefix: /contrib/spack-stack-rocky8/ecflow-5.8.4 + modules: + - ecflow/5.8.4 file: externals: - - spec: file@5.11 + - spec: file@5.33 prefix: /usr findutils: externals: - - spec: findutils@4.5.11 + - spec: findutils@4.6.0 prefix: /usr - gawk: + flex: externals: - - spec: gawk@4.0.2 + - spec: flex@2.6.1 prefix: /usr - gettext: + gawk: externals: - - spec: gettext@0.19.8.1 + - spec: gawk@4.2.1 prefix: /usr git: externals: - - spec: git@1.8.3.1~tcltk + - spec: git@2.39.3~tcltk prefix: /usr git-lfs: externals: - - spec: git-lfs@2.4.1 - prefix: /contrib/spack-stack/git-lfs-2.4.1 + - spec: git-lfs@3.2.0 + prefix: /usr gmake: externals: - - spec: gmake@3.82 + - spec: gmake@4.2.1 prefix: /usr groff: externals: - - spec: groff@1.22.2 + - spec: groff@1.21 prefix: /usr hwloc: externals: - - spec: hwloc@1.11.8 + - spec: hwloc@2.2.0 prefix: /usr krb5: externals: - - spec: krb5@1.15.1 - prefix: /usr - lustre: - externals: - - spec: lustre@2.12.7 + - spec: krb5@1.18.2 prefix: /usr mysql: externals: - spec: mysql@8.0.31 - prefix: /contrib/spack-stack/mysql-8.0.31 + prefix: /contrib/spack-stack-rocky8/mysql-8.0.31 + ncurses: + externals: + - spec: ncurses@6.1-10.20180224+termlib abi=5 + prefix: /usr openjdk: externals: - - spec: openjdk@1.8.0_322-b06 + - spec: openjdk@1.8.0_402-b06 prefix: /usr perl: externals: - - spec: perl@5.16.3~cpanm+shared+threads + - spec: perl@5.26.3~cpanm+shared+threads prefix: /usr pkg-config: externals: - - spec: pkg-config@0.27.1 + - spec: pkg-config@1.4.2 prefix: /usr rsync: externals: - - spec: rsync@3.1.2 + - spec: rsync@3.1.3 prefix: /usr ruby: externals: - - spec: ruby@2.0.0 + - spec: ruby@2.5.9p229 prefix: /usr sed: externals: - - spec: sed@4.2.2 + - spec: sed@4.5 prefix: /usr tar: externals: - - spec: tar@1.26 + - spec: tar@1.30 prefix: /usr texinfo: externals: - - spec: texinfo@5.1 + - spec: texinfo@6.5-7 prefix: /usr wget: externals: - - spec: wget@1.14 + - spec: wget@1.19.5 prefix: /usr zip: externals: - spec: zip@3.0 prefix: /usr + zlib: + require: [~shared] diff --git a/configs/sites/tier1/noaa-aws/packages_intel.yaml b/configs/sites/tier1/noaa-aws/packages_intel.yaml index 995404d2d..1ffe7cf2f 100644 --- a/configs/sites/tier1/noaa-aws/packages_intel.yaml +++ b/configs/sites/tier1/noaa-aws/packages_intel.yaml @@ -1,8 +1,8 @@ packages: all: - compiler:: [intel@2021.3.0,gcc@9.2.0] + compiler:: [intel@2021.10.0] providers: - mpi:: [intel-oneapi-mpi@2021.3.0] + mpi:: [intel-oneapi-mpi@2021.10.0] # Remove the next three lines to switch to intel-oneapi-mkl blas:: [openblas] fftw-api:: [fftw] @@ -11,10 +11,10 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.3.0%intel@2021.3.0 + - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 prefix: /apps/oneapi modules: - - impi/2021.3.0 + - impi/2023.2.0 intel-oneapi-mkl: # Remove buildable: False and configure+uncomment externals section below to use intel-oneapi-mkl buildable: False @@ -30,5 +30,5 @@ packages: - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.25 ^openblas' + - '@:1.23 ^openblas' # *DH diff --git a/configs/sites/tier1/noaa-azure/README.md b/configs/sites/tier1/noaa-azure/README.md index 2253cda9f..eff315412 100644 --- a/configs/sites/tier1/noaa-azure/README.md +++ b/configs/sites/tier1/noaa-azure/README.md @@ -1,168 +1,49 @@ # Provisiong ParallelWorks Azure clusters -## Steps to perform when provisioning a cluster for the very first time - -This section provides step by step instructions for installing the basic packages (OS packages, external packages) for spack-stack. Following these steps ensures that the site configuration files in `configs/sites/noaa-aws` work out of the box. - -### Basic system packages -``` -sudo su -chmod 777 /contrib - -# The following three commands were necessary to fix failures accessing these repos -yum-config-manager --disable intel-clck-2019-repo -yum-config-manager --disable intel-hpc-platform -yum-config-manager --disable intelpython - -yum install -y qt5-qtbase-devel -yum install -y qt5-qtsvg-devel -yum install -y xorg-x11-xauth -yum install -y xorg-x11-apps -yum install -y perl-IPC-Cmd -yum install -y gettext-devel -yum install -y ncurses-devel -yum install -y ncurses-static -yum install -y m4 -exit -``` -Create a script that can be added to the cluster resource config so that these packages get installed automatically when provisioning new clusters later: -``` -mkdir -p /contrib/admin -cat < /contrib/admin/basic_setup.sh -#!/bin/bash +## Steps to perform before installing spack-stack version 1.8.0 +sudo su - chmod 777 /contrib yum install -y qt5-qtbase-devel yum install -y qt5-qtsvg-devel -yum install -y xorg-x11-xauth -yum install -y xorg-x11-apps -yum install -y perl-IPC-Cmd -yum install -y gettext-devel -yum install -y ncurses-devel -yum install -y ncurses-static -yum install -y m4 -EOF - -chmod a+x /contrib/admin/basic_setup.sh -``` - -### Create a mysql config for local R2D2 use (if applicable) -``` -sudo su -cat < /contrib/admin/my.cnf -[mysqld] -datadir=/mysql_local/data -socket=/mysql_local/data/mysql.sock -symbolic-links=0 -default-authentication-plugin=mysql_native_password - -[mysqld_safe] -log-error=/mysql_local/log/mariadb.log -pid-file=/mysql_local/run/mariadb.pid -EOF -chmod 644 /contrib/admin/my.cnf -exit -``` - -### Build external packages for spack-stack -``` -mkdir -p /contrib/spack-stack -mkdir /contrib/spack-stack/modulefiles -cd /contrib/spack-stack/ - -mkdir -p git-lfs-2.4.1/src -cd git-lfs-2.4.1/src -wget http://mirror.centos.org/centos/7/sclo/x86_64/rh/Packages/r/rh-git218-git-lfs-2.4.1-3.el7.x86_64.rpm -rpm2cpio rh-git218-git-lfs-2.4.1-3.el7.x86_64.rpm | cpio -idmv -mv opt/rh/rh-git218/root/usr/* .. -rm -fr opt -cd /contrib/spack-stack/modulefiles -mkdir git-lfs -# Create the modulefile from the template in doc/modulefile_templates -cd /contrib/spack-stack/ -mkdir -p mysql-8.0.31/src -cd mysql-8.0.31/src -ldd --version -wget https://dev.mysql.com/get/Downloads/MySQL-8.0/mysql-8.0.31-linux-glibc2.17-x86_64-minimal.tar.xz -tar -xvf mysql-8.0.31-linux-glibc2.17-x86_64-minimal.tar.xz -mv mysql-8.0.31-linux-glibc2.17-x86_64-minimal/* .. -rmdir mysql-8.0.31-linux-glibc2.17-x86_64-minimal -cd /contrib/spack-stack/modulefiles -mkdir mysql -# Create the modulefile from the template in doc/modulefile_templates -cd /contrib/spack-stack/ -mkdir -p cmake-3.27.2/src -cd cmake-3.27.2/src -wget https://github.com/Kitware/CMake/releases/download/v3.27.2/cmake-3.27.2-linux-x86_64.tar.gz -tar -xvzf cmake-3.27.2-linux-x86_64.tar.gz -mv cmake-3.27.2-linux-x86_64/* .. -rmdir cmake-3.27.2-linux-x86_64 -cd /contrib/spack-stack/modulefiles -mkdir cmake -# Create the modulefile from the template in doc/modulefile_templates +## Steps to install spack-stack version 1.8.0 -# Set up basic modules for building the external ecflow package -module unuse /opt/cray/craype/default/modulefiles -module unuse /opt/cray/modulefiles +sudo su - +chmod 777 /contrib module purge -module load gnu/9.2.0 -module use /contrib/spack-stack/modulefiles -module load cmake/3.27.2 - -mkdir -p /contrib/spack-stack/ecflow-5.8.4/src -cd /contrib/spack-stack/ecflow-5.8.4/src -wget https://confluence.ecmwf.int/download/attachments/8650755/ecFlow-5.8.4-Source.tar.gz?api=v2 -wget https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz -mv ecFlow-5.8.4-Source.tar.gz\?api\=v2 ecFlow-5.8.4-Source.tar.gz -tar -xvzf boost_1_78_0.tar.gz -tar -xvzf ecFlow-5.8.4-Source.tar.gz -export WK=/contrib/spack-stack/ecflow-5.8.4/src/ecFlow-5.8.4-Source -export BOOST_ROOT=/contrib/spack-stack/ecflow-5.8.4/src/boost_1_78_0 - -# Build static boost (to not interfere with spack-stack boost) -cd $BOOST_ROOT -./bootstrap.sh 2>&1 | tee bootstrap.log -$WK/build_scripts/boost_build.sh 2>&1 | tee boost_build.log - -# Build ecFlow -cd $WK -mkdir build -cd build -cmake .. -DCMAKE_INSTALL_PREFIX=/contrib/spack-stack/ecflow-5.8.4 2>&1 | tee log.cmake -make -j4 2>&1 | tee log.make -make install 2>&1 | tee log.install - -cd /contrib/spack-stack/modulefiles -mkdir ecflow -# Create the modulefile from the template in doc/modulefile_templates -``` - -## Steps to perform when starting a new cluster - -This should be done automatically, but doesn't hurt to run again just in case: -``` -sudo /contrib/admin/basic_setup.sh -``` -Configure `git` and `aws` command line utilities -``` module unuse /opt/cray/craype/default/modulefiles module unuse /opt/cray/modulefiles -module use /contrib/spack-stack/modulefiles -module load cmake/3.27.2 -module load ecflow/5.8.4 -module load mysql/8.0.31 -module load git-lfs/2.4.1 - -git lfs install -git config --global credential.helper cache # or store -git config --global user.email "your.name@domain.com" -git config --global user.name "Your Name" +module load gnu +module load intel/2023.2.0 +module load impi/2023.2.0 +module unload gnu + +cd /contrib/spack-stack-rocky8/ +git clone --recursive https://github.com/JCSDA/spack-stack -b release/1.8.0 spack-stack-1.8.0 +cd spack-stack-1.8.0 +. setup.sh +spack stack create env --name ue-intel-2021.10.0 --template unified-dev --site noaa-azure --compiler intel +cd envs/ue-intel-2021.10.0 +spack env activate . +spack concretize 2>&1 | tee log.concretize +spack install --verbose 2>&1 | tee log.install +spack module lmod refresh -y +spack stack setup-meta-modules + +## Steps to install GSI addon + +sudo su - +chmod 777 /contrib -mkdir ~/.aws -# Create ~/.aws/config -# Create ~/.aws/credentials -chmod 400 ~/.aws/credentials -``` +cd /contrib/spack-stack-rocky8/spack-stack-1.8.0 +. setup.sh +spack stack create env --name gsi-intel-2021.10.0 --template gsi-addon-dev --site noaa-azure --upstream /contrib/spack-stack-rocky8/spack-stack-1.8.0/envs/ue-intel-2021.10.0/install --compiler intel +cd envs/gsi-intel-2021.10.0 +spack env activate . +spack concretize 2>&1 | tee log.concretize +spack install --verbose 2>&1 | tee log.install +spack module lmod refresh --upstream-modules +spack stack setup-meta-modules diff --git a/configs/sites/tier1/noaa-azure/compilers.yaml b/configs/sites/tier1/noaa-azure/compilers.yaml index aec6336cf..343520e4c 100644 --- a/configs/sites/tier1/noaa-azure/compilers.yaml +++ b/configs/sites/tier1/noaa-azure/compilers.yaml @@ -1,20 +1,19 @@ compilers: - compiler: - spec: intel@2021.3.0 + spec: intel@2021.10.0 paths: - cc: /apps/oneapi/compiler/2021.3.0/linux/bin/intel64/icc - cxx: /apps/oneapi/compiler/2021.3.0/linux/bin/intel64/icpc - f77: /apps/oneapi/compiler/2021.3.0/linux/bin/intel64/ifort - fc: /apps/oneapi/compiler/2021.3.0/linux/bin/intel64/ifort - flags: {} - operating_system: centos7 + cc: /apps/oneapi/compiler/2023.2.0/linux/bin/intel64/icc + cxx: /apps/oneapi/compiler/2023.2.0/linux/bin/intel64/icpc + f77: /apps/oneapi/compiler/2023.2.0/linux/bin/intel64/ifort + fc: /apps/oneapi/compiler/2023.2.0/linux/bin/intel64/ifort + operating_system: rocky8 target: x86_64 modules: - - intel/2021.3.0 + - intel/2023.2.0 environment: prepend_path: - PATH: '/apps/gnu/gcc-9.2.0/bin' - LD_LIBRARY_PATH: '/apps/oneapi/compiler/2021.3.0/linux/compiler/lib/intel64_lin:/apps/gnu/gcc-9.2.0/lib64' + LD_LIBRARY_PATH: '/apps/oneapi/compiler/2023.2.0/linux/compiler/lib/intel64_lin:/apps/gnu/gcc-13.2.0/lib64' + flags: {} extra_rpaths: [] - compiler: spec: gcc@9.2.0 @@ -24,8 +23,7 @@ compilers: f77: /apps/gnu/gcc-9.2.0/bin/gfortran fc: /apps/gnu/gcc-9.2.0/bin/gfortran flags: {} - operating_system: centos7 - target: x86_64 + operating_system: rocky8 modules: - gnu/9.2.0 environment: {} diff --git a/configs/sites/tier1/noaa-azure/packages.yaml b/configs/sites/tier1/noaa-azure/packages.yaml index c3522a095..004aea213 100644 --- a/configs/sites/tier1/noaa-azure/packages.yaml +++ b/configs/sites/tier1/noaa-azure/packages.yaml @@ -1,146 +1,110 @@ packages: -### Modifications of common packages - # Pin flex to avoid duplicate packages - flex: - version: ['2.6.4'] -### All other external packages listed alphabetically - autoconf: - externals: - - spec: autoconf@2.69 - prefix: /usr - automake: - externals: - - spec: automake@1.13.4 - prefix: /usr bash: externals: - - spec: bash@4.2.46 + - spec: bash@4.4.20 prefix: /usr berkeley-db: externals: - - spec: berkeley-db@5.3.21 + - spec: berkeley-db@5.3.28 prefix: /usr cpio: externals: - - spec: cpio@2.11 + - spec: cpio@2.12 prefix: /usr diffutils: externals: - - spec: diffutils@3.3 - prefix: /usr - doxygen: - externals: - - spec: doxygen@1.8.5~graphviz~mscgen + - spec: diffutils@3.6 prefix: /usr ecflow:: buildable: False externals: - spec: ecflow@5.8.4+ui+static_boost - prefix: /contrib/spack-stack/ecflow-5.8.4 + prefix: /contrib/spack-stack-rocky8/ecflow-5.8.4 + modules: + - ecflow/5.8.4 file: externals: - - spec: file@5.11 + - spec: file@5.33 prefix: /usr findutils: externals: - - spec: findutils@4.5.11 + - spec: findutils@4.6.0 prefix: /usr - gawk: + flex: externals: - - spec: gawk@4.0.2 + - spec: flex@2.6.1 prefix: /usr - gettext: + gawk: externals: - - spec: gettext@0.19.8.1 + - spec: gawk@4.2.1 prefix: /usr git: externals: - - spec: git@1.8.3.1~tcltk + - spec: git@2.39.3~tcltk prefix: /usr git-lfs: externals: - - spec: git-lfs@2.4.1 - prefix: /contrib/spack-stack/git-lfs-2.4.1 - gmake: - externals: - - spec: gmake@3.82 + - spec: git-lfs@3.2.0 prefix: /usr - groff: + gmake: externals: - - spec: groff@1.22.2 + - spec: gmake@4.2.1 prefix: /usr hwloc: externals: - - spec: hwloc@1.11.8 + - spec: hwloc@2.2.0 prefix: /usr krb5: externals: - - spec: krb5@1.15.1 - prefix: /usr - libtool: - externals: - - spec: libtool@2.4.2 - prefix: /usr - # Problem untarring libxml2 on the Azure filesystem - # so mark it as external - libxml2: - buildable: False - externals: - - spec: libxml2@2.9.1 - prefix: /usr - lustre: - externals: - - spec: lustre@2.12.7 - prefix: /usr - m4: - externals: - - spec: m4@1.4.16 + - spec: krb5@1.18.2 prefix: /usr mysql: externals: - spec: mysql@8.0.31 - prefix: /contrib/spack-stack/mysql-8.0.31 + prefix: /contrib/spack-stack-rocky8/mysql-8.0.31 ncurses: externals: - - spec: ncurses@5.9-14.20130511+termlib abi=5 + - spec: ncurses@6.1-10.20180224+termlib abi=5 prefix: /usr openjdk: externals: - - spec: openjdk@1.8.0_322-b06 + - spec: openjdk@1.8.0_402-b06 prefix: /usr perl: externals: - - spec: perl@5.16.3~cpanm+shared+threads + - spec: perl@5.26.3~cpanm+shared+threads prefix: /usr pkg-config: externals: - - spec: pkg-config@0.27.1 + - spec: pkg-config@1.4.2 prefix: /usr rsync: externals: - - spec: rsync@3.1.2 + - spec: rsync@3.1.3 prefix: /usr ruby: externals: - - spec: ruby@2.0.0 + - spec: ruby@2.5.9p229 prefix: /usr sed: externals: - - spec: sed@4.2.2 + - spec: sed@4.5 prefix: /usr tar: externals: - - spec: tar@1.26 + - spec: tar@1.30 prefix: /usr texinfo: externals: - - spec: texinfo@5.1 + - spec: texinfo@6.5-7 prefix: /usr wget: externals: - - spec: wget@1.14 + - spec: wget@1.19.5 prefix: /usr zip: externals: - spec: zip@3.0 prefix: /usr + zlib: + require: [~shared] diff --git a/configs/sites/tier1/noaa-azure/packages_intel.yaml b/configs/sites/tier1/noaa-azure/packages_intel.yaml index 995404d2d..1ffe7cf2f 100644 --- a/configs/sites/tier1/noaa-azure/packages_intel.yaml +++ b/configs/sites/tier1/noaa-azure/packages_intel.yaml @@ -1,8 +1,8 @@ packages: all: - compiler:: [intel@2021.3.0,gcc@9.2.0] + compiler:: [intel@2021.10.0] providers: - mpi:: [intel-oneapi-mpi@2021.3.0] + mpi:: [intel-oneapi-mpi@2021.10.0] # Remove the next three lines to switch to intel-oneapi-mkl blas:: [openblas] fftw-api:: [fftw] @@ -11,10 +11,10 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.3.0%intel@2021.3.0 + - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 prefix: /apps/oneapi modules: - - impi/2021.3.0 + - impi/2023.2.0 intel-oneapi-mkl: # Remove buildable: False and configure+uncomment externals section below to use intel-oneapi-mkl buildable: False @@ -30,5 +30,5 @@ packages: - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.25 ^openblas' + - '@:1.23 ^openblas' # *DH diff --git a/configs/sites/tier1/noaa-gcloud/README.md b/configs/sites/tier1/noaa-gcloud/README.md index f1fe42a35..f0fab7b91 100644 --- a/configs/sites/tier1/noaa-gcloud/README.md +++ b/configs/sites/tier1/noaa-gcloud/README.md @@ -1,169 +1,48 @@ -# Provisiong ParallelWorks Gcloud clusters +# Provisiong ParallelWorks GCP clusters -## Steps to perform when provisioning a cluster for the very first time +## Steps to perform before installing spack-stack version 1.8.0 -This section provides step by step instructions for installing the basic packages (OS packages, external packages) for spack-stack. Following these steps ensures that the site configuration files in `configs/sites/noaa-aws` work out of the box. - -### Basic system packages -``` -sudo su -chmod 777 /contrib - -# The following three commands were necessary to fix failures accessing these repos -yum-config-manager --disable intel-clck-2019-repo -yum-config-manager --disable intel-hpc-platform -yum-config-manager --disable intelpython - -yum install -y qt5-qtbase-devel -yum install -y qt5-qtsvg-devel -yum install -y xorg-x11-xauth -yum install -y xorg-x11-apps -yum install -y perl-IPC-Cmd -yum install -y gettext-devel -yum install -y m4 -yum install -y finger -exit -``` -Create a script that can be added to the cluster resource config so that these packages get installed automatically when provisioning new clusters later: -``` -mkdir -p /contrib/admin -cat < /contrib/admin/basic_setup.sh -#!/bin/bash - -yum-config-manager --disable intel-clck-2019-repo -yum-config-manager --disable intel-hpc-platform -yum-config-manager --disable intelpython +sudo su - chmod 777 /contrib yum install -y qt5-qtbase-devel yum install -y qt5-qtsvg-devel -yum install -y xorg-x11-xauth -yum install -y xorg-x11-apps -yum install -y perl-IPC-Cmd -yum install -y gettext-devel -yum install -y m4 -yum install -y finger -EOF - -chmod a+x /contrib/admin/basic_setup.sh -``` - -### Create a mysql config for local R2D2 use (if applicable) -``` -sudo su -cat < /contrib/admin/my.cnf -[mysqld] -datadir=/mysql_local/data -socket=/mysql_local/data/mysql.sock -symbolic-links=0 -default-authentication-plugin=mysql_native_password - -[mysqld_safe] -log-error=/mysql_local/log/mariadb.log -pid-file=/mysql_local/run/mariadb.pid -EOF -chmod 644 /contrib/admin/my.cnf -exit -``` - -### Build external packages for spack-stack -``` -mkdir -p /contrib/spack-stack -mkdir /contrib/spack-stack/modulefiles -cd /contrib/spack-stack/ - -mkdir -p git-lfs-2.4.1/src -cd git-lfs-2.4.1/src -wget http://mirror.centos.org/centos/7/sclo/x86_64/rh/Packages/r/rh-git218-git-lfs-2.4.1-3.el7.x86_64.rpm -rpm2cpio rh-git218-git-lfs-2.4.1-3.el7.x86_64.rpm | cpio -idmv -mv opt/rh/rh-git218/root/usr/* .. -rm -fr opt -cd /contrib/spack-stack/modulefiles -mkdir git-lfs -# Create the modulefile from the template in doc/modulefile_templates -cd /contrib/spack-stack/ -mkdir -p mysql-8.0.31/src -cd mysql-8.0.31/src -ldd --version -wget https://dev.mysql.com/get/Downloads/MySQL-8.0/mysql-8.0.31-linux-glibc2.17-x86_64-minimal.tar.xz -tar -xvf mysql-8.0.31-linux-glibc2.17-x86_64-minimal.tar.xz -mv mysql-8.0.31-linux-glibc2.17-x86_64-minimal/* .. -rmdir mysql-8.0.31-linux-glibc2.17-x86_64-minimal -cd /contrib/spack-stack/modulefiles -mkdir mysql -# Create the modulefile from the template in doc/modulefile_templates -cd /contrib/spack-stack/ -mkdir -p cmake-3.27.2/src -cd cmake-3.27.2/src -wget https://github.com/Kitware/CMake/releases/download/v3.27.2/cmake-3.27.2-linux-x86_64.tar.gz -tar -xvzf cmake-3.27.2-linux-x86_64.tar.gz -mv cmake-3.27.2-linux-x86_64/* .. -rmdir cmake-3.27.2-linux-x86_64 -cd /contrib/spack-stack/modulefiles -mkdir cmake -# Create the modulefile from the template in doc/modulefile_templates +## Steps to install spack-stack version 1.8.0 -# Set up basic modules for building the external ecflow package -module unuse /opt/cray/craype/default/modulefiles -module unuse /opt/cray/modulefiles +sudo su - +chmod 777 /contrib module purge -module load gnu/9.2.0 -module use /contrib/spack-stack/modulefiles -module load cmake/3.27.2 - -mkdir -p /contrib/spack-stack/ecflow-5.8.4/src -cd /contrib/spack-stack/ecflow-5.8.4/src -wget https://confluence.ecmwf.int/download/attachments/8650755/ecFlow-5.8.4-Source.tar.gz?api=v2 -wget https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.gz -mv ecFlow-5.8.4-Source.tar.gz\?api\=v2 ecFlow-5.8.4-Source.tar.gz -tar -xvzf boost_1_78_0.tar.gz -tar -xvzf ecFlow-5.8.4-Source.tar.gz -export WK=/contrib/spack-stack/ecflow-5.8.4/src/ecFlow-5.8.4-Source -export BOOST_ROOT=/contrib/spack-stack/ecflow-5.8.4/src/boost_1_78_0 - -# Build static boost (to not interfere with spack-stack boost) -cd $BOOST_ROOT -./bootstrap.sh 2>&1 | tee bootstrap.log -$WK/build_scripts/boost_build.sh 2>&1 | tee boost_build.log - -# Build ecFlow -cd $WK -mkdir build -cd build -cmake .. -DCMAKE_INSTALL_PREFIX=/contrib/spack-stack/ecflow-5.8.4 2>&1 | tee log.cmake -make -j4 2>&1 | tee log.make -make install 2>&1 | tee log.install - -cd /contrib/spack-stack/modulefiles -mkdir ecflow -# Create the modulefile from the template in doc/modulefile_templates -``` - -## Steps to perform when starting a new cluster - -This should be done automatically, but doesn't hurt to run again just in case: -``` -sudo /contrib/admin/basic_setup.sh -``` -Configure `git` and `aws` command line utilities -``` module unuse /opt/cray/craype/default/modulefiles module unuse /opt/cray/modulefiles -module use /contrib/spack-stack/modulefiles -module load cmake/3.27.2 -module load ecflow/5.8.4 -module load mysql/8.0.31 -module load git-lfs/2.4.1 - -git lfs install -git config --global credential.helper cache # or store -git config --global user.email "your.name@domain.com" -git config --global user.name "Your Name" +module load gnu +module load intel/2023.2.0 +module load impi/2023.2.0 +module unload gnu + +cd /contrib/spack-stack-rocky8/ +git clone --recursive https://github.com/JCSDA/spack-stack -b release/1.8.0 spack-stack-1.8.0 +cd spack-stack-1.8.0 +. setup.sh +spack stack create env --name ue-intel-2021.10.0 --template unified-dev --site noaa-gcloud --compiler intel +cd envs/ue-intel-2021.10.0 +spack env activate . +spack concretize 2>&1 | tee log.concretize +spack install --verbose 2>&1 | tee log.install +spack module lmod refresh -y +spack stack setup-meta-modules + +## Steps to install GSI addon + +sudo su - +chmod 777 /contrib -mkdir ~/.aws -# Create ~/.aws/config -# Create ~/.aws/credentials -chmod 400 ~/.aws/credentials -``` +cd /contrib/spack-stack-rocky8/spack-stack-1.8.0 +. setup.sh +spack stack create env --name gsi-intel-2021.10.0 --template gsi-addon-dev --site noaa-gcloud --upstream /contrib/spack-stack-rocky8/spack-stack-1.8.0/envs/ue-intel-2021.10.0/install --compiler intel +cd envs/gsi-intel-2021.10.0 +spack concretize 2>&1 | tee log.concretize +spack install --verbose 2>&1 | tee log.install +spack module lmod refresh --upstream-modules +spack stack setup-meta-modules diff --git a/configs/sites/tier1/noaa-gcloud/compilers.yaml b/configs/sites/tier1/noaa-gcloud/compilers.yaml index aec6336cf..343520e4c 100644 --- a/configs/sites/tier1/noaa-gcloud/compilers.yaml +++ b/configs/sites/tier1/noaa-gcloud/compilers.yaml @@ -1,20 +1,19 @@ compilers: - compiler: - spec: intel@2021.3.0 + spec: intel@2021.10.0 paths: - cc: /apps/oneapi/compiler/2021.3.0/linux/bin/intel64/icc - cxx: /apps/oneapi/compiler/2021.3.0/linux/bin/intel64/icpc - f77: /apps/oneapi/compiler/2021.3.0/linux/bin/intel64/ifort - fc: /apps/oneapi/compiler/2021.3.0/linux/bin/intel64/ifort - flags: {} - operating_system: centos7 + cc: /apps/oneapi/compiler/2023.2.0/linux/bin/intel64/icc + cxx: /apps/oneapi/compiler/2023.2.0/linux/bin/intel64/icpc + f77: /apps/oneapi/compiler/2023.2.0/linux/bin/intel64/ifort + fc: /apps/oneapi/compiler/2023.2.0/linux/bin/intel64/ifort + operating_system: rocky8 target: x86_64 modules: - - intel/2021.3.0 + - intel/2023.2.0 environment: prepend_path: - PATH: '/apps/gnu/gcc-9.2.0/bin' - LD_LIBRARY_PATH: '/apps/oneapi/compiler/2021.3.0/linux/compiler/lib/intel64_lin:/apps/gnu/gcc-9.2.0/lib64' + LD_LIBRARY_PATH: '/apps/oneapi/compiler/2023.2.0/linux/compiler/lib/intel64_lin:/apps/gnu/gcc-13.2.0/lib64' + flags: {} extra_rpaths: [] - compiler: spec: gcc@9.2.0 @@ -24,8 +23,7 @@ compilers: f77: /apps/gnu/gcc-9.2.0/bin/gfortran fc: /apps/gnu/gcc-9.2.0/bin/gfortran flags: {} - operating_system: centos7 - target: x86_64 + operating_system: rocky8 modules: - gnu/9.2.0 environment: {} diff --git a/configs/sites/tier1/noaa-gcloud/packages.yaml b/configs/sites/tier1/noaa-gcloud/packages.yaml index b86026f11..004aea213 100644 --- a/configs/sites/tier1/noaa-gcloud/packages.yaml +++ b/configs/sites/tier1/noaa-gcloud/packages.yaml @@ -1,119 +1,110 @@ packages: -### Modifications of common packages - # Pin flex to avoid duplicate packages - flex: - version: ['2.6.4'] -### All other external packages listed alphabetically bash: externals: - - spec: bash@4.2.46 + - spec: bash@4.4.20 prefix: /usr berkeley-db: externals: - - spec: berkeley-db@5.3.21 + - spec: berkeley-db@5.3.28 prefix: /usr cpio: externals: - - spec: cpio@2.11 + - spec: cpio@2.12 prefix: /usr diffutils: externals: - - spec: diffutils@3.3 + - spec: diffutils@3.6 prefix: /usr ecflow:: buildable: False externals: - spec: ecflow@5.8.4+ui+static_boost - prefix: /contrib/spack-stack/ecflow-5.8.4 + prefix: /contrib/spack-stack-rocky8/ecflow-5.8.4 + modules: + - ecflow/5.8.4 file: externals: - - spec: file@5.11 + - spec: file@5.33 prefix: /usr findutils: externals: - - spec: findutils@4.5.11 + - spec: findutils@4.6.0 prefix: /usr - gawk: + flex: externals: - - spec: gawk@4.0.2 + - spec: flex@2.6.1 prefix: /usr - gettext: + gawk: externals: - - spec: gettext@0.19.8.1 + - spec: gawk@4.2.1 prefix: /usr git: externals: - - spec: git@1.8.3.1~tcltk + - spec: git@2.39.3~tcltk prefix: /usr git-lfs: externals: - - spec: git-lfs@2.4.1 - prefix: /contrib/spack-stack/git-lfs-2.4.1 - gmake: - externals: - - spec: gmake@3.82 + - spec: git-lfs@3.2.0 prefix: /usr - groff: + gmake: externals: - - spec: groff@1.22.2 + - spec: gmake@4.2.1 prefix: /usr hwloc: externals: - - spec: hwloc@1.11.8 + - spec: hwloc@2.2.0 prefix: /usr krb5: externals: - - spec: krb5@1.15.1 - prefix: /usr - lustre: - externals: - - spec: lustre@2.12.7 - prefix: /usr - m4: - externals: - - spec: m4@1.4.16 + - spec: krb5@1.18.2 prefix: /usr mysql: externals: - spec: mysql@8.0.31 - prefix: /contrib/spack-stack/mysql-8.0.31 + prefix: /contrib/spack-stack-rocky8/mysql-8.0.31 + ncurses: + externals: + - spec: ncurses@6.1-10.20180224+termlib abi=5 + prefix: /usr openjdk: externals: - - spec: openjdk@1.8.0_322-b06 + - spec: openjdk@1.8.0_402-b06 prefix: /usr perl: externals: - - spec: perl@5.16.3~cpanm+shared+threads + - spec: perl@5.26.3~cpanm+shared+threads prefix: /usr pkg-config: externals: - - spec: pkg-config@0.27.1 + - spec: pkg-config@1.4.2 prefix: /usr rsync: externals: - - spec: rsync@3.1.2 + - spec: rsync@3.1.3 prefix: /usr ruby: externals: - - spec: ruby@2.0.0 + - spec: ruby@2.5.9p229 prefix: /usr sed: externals: - - spec: sed@4.2.2 + - spec: sed@4.5 prefix: /usr tar: externals: - - spec: tar@1.26 + - spec: tar@1.30 prefix: /usr texinfo: externals: - - spec: texinfo@5.1 + - spec: texinfo@6.5-7 prefix: /usr wget: externals: - - spec: wget@1.14 + - spec: wget@1.19.5 prefix: /usr zip: externals: - spec: zip@3.0 prefix: /usr + zlib: + require: [~shared] diff --git a/configs/sites/tier1/noaa-gcloud/packages_intel.yaml b/configs/sites/tier1/noaa-gcloud/packages_intel.yaml index 995404d2d..1ffe7cf2f 100644 --- a/configs/sites/tier1/noaa-gcloud/packages_intel.yaml +++ b/configs/sites/tier1/noaa-gcloud/packages_intel.yaml @@ -1,8 +1,8 @@ packages: all: - compiler:: [intel@2021.3.0,gcc@9.2.0] + compiler:: [intel@2021.10.0] providers: - mpi:: [intel-oneapi-mpi@2021.3.0] + mpi:: [intel-oneapi-mpi@2021.10.0] # Remove the next three lines to switch to intel-oneapi-mkl blas:: [openblas] fftw-api:: [fftw] @@ -11,10 +11,10 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.3.0%intel@2021.3.0 + - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 prefix: /apps/oneapi modules: - - impi/2021.3.0 + - impi/2023.2.0 intel-oneapi-mkl: # Remove buildable: False and configure+uncomment externals section below to use intel-oneapi-mkl buildable: False @@ -30,5 +30,5 @@ packages: - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.25 ^openblas' + - '@:1.23 ^openblas' # *DH diff --git a/configs/sites/tier1/orion/packages_intel.yaml b/configs/sites/tier1/orion/packages_intel.yaml index 88131d85c..8c1e33385 100644 --- a/configs/sites/tier1/orion/packages_intel.yaml +++ b/configs/sites/tier1/orion/packages_intel.yaml @@ -32,4 +32,4 @@ packages: - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.25 ^openblas' + - '@:1.23 ^openblas' diff --git a/configs/sites/tier1/s4/packages.yaml b/configs/sites/tier1/s4/packages.yaml index 3650e0678..e0312b799 100644 --- a/configs/sites/tier1/s4/packages.yaml +++ b/configs/sites/tier1/s4/packages.yaml @@ -33,7 +33,7 @@ packages: - '@1.2.1 ~mkl' py-numpy: require:: - - '@:1.25 ^openblas' + - '@:1.23 ^openblas' ### All other external packages listed alphabetically autoconf: diff --git a/configs/sites/tier1/sandy/modules.yaml b/configs/sites/tier1/sandy/modules.yaml index fb279947b..aeb254c2e 100644 --- a/configs/sites/tier1/sandy/modules.yaml +++ b/configs/sites/tier1/sandy/modules.yaml @@ -6,5 +6,3 @@ modules: include: # List of packages for which we need modules that are blacklisted by default - python - exclude: - - ecflow diff --git a/configs/sites/tier2/blackpearl/README.md b/configs/sites/tier2/blackpearl/README.md index f79c9c72b..a9a8c43a6 100644 --- a/configs/sites/tier2/blackpearl/README.md +++ b/configs/sites/tier2/blackpearl/README.md @@ -1,3 +1 @@ -Blackpearl is @climbfuji's development system (Dell Laptop running Oracle Linux 9.1 under Windows WSL2). - -The site config currently requires a manual copy of the correct packages_COMPILER.yaml to the environment site directory. This will be improved later. \ No newline at end of file +Blackpearl is one of @climbfuji's development systems (Dell Laptop running Oracle Linux 9.1 under Windows WSL2). diff --git a/configs/sites/tier2/blackpearl/compilers.yaml b/configs/sites/tier2/blackpearl/compilers.yaml index f04aef5bb..420ff1307 100644 --- a/configs/sites/tier2/blackpearl/compilers.yaml +++ b/configs/sites/tier2/blackpearl/compilers.yaml @@ -1,17 +1,4 @@ compilers: -- compiler: - spec: clang@=14.0.6 - paths: - cc: /usr/bin/clang - cxx: /usr/bin/clang++ - f77: null - fc: null - flags: {} - operating_system: oracle9 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] - compiler: spec: gcc@=11.4.1 paths: @@ -39,14 +26,15 @@ compilers: environment: {} extra_rpaths: [] - compiler: - spec: oneapi@=2024.1.2 + spec: oneapi@=2024.2.1 paths: - cc: /opt/intel/oneapi/compiler/2024.1/bin/icx - cxx: /opt/intel/oneapi/compiler/2024.1/bin/icpx - f77: /opt/intel/oneapi/compiler/2024.1/bin/ifort - fc: /opt/intel/oneapi/compiler/2024.1/bin/ifort - #f77: /opt/intel/oneapi/compiler/2024.1/bin/ifx - #fc: /opt/intel/oneapi/compiler/2024.1/bin/ifx + cc: /opt/intel/oneapi/2024.2/bin/icx + cxx: /opt/intel/oneapi/2024.2/bin/icpx + # https://github.com/JCSDA/spack-stack/issues/1279 + f77: /opt/intel/oneapi/2024.2/bin/ifort + fc: /opt/intel/oneapi/2024.2/bin/ifort + #f77: /opt/intel/oneapi/2024.2/bin/ifx + #fc: /opt/intel/oneapi/2024.2/bin/ifx flags: {} operating_system: oracle9 target: x86_64 @@ -54,7 +42,7 @@ compilers: environment: # change to append_path once relevant PR is merged append_path: - CPATH: /opt/intel/oneapi/compiler/2024.1/opt/compiler/include/intel64 + CPATH: /opt/intel/oneapi/compiler/2024.2/opt/compiler/include/intel64 extra_rpaths: [] - compiler: spec: aocc@=4.2.0 diff --git a/configs/sites/tier2/blackpearl/packages_oneapi.yaml b/configs/sites/tier2/blackpearl/packages_oneapi.yaml index f4c243c1b..34b40c943 100644 --- a/configs/sites/tier2/blackpearl/packages_oneapi.yaml +++ b/configs/sites/tier2/blackpearl/packages_oneapi.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [oneapi@2024.1.2,gcc@11.4.1] + compiler:: [oneapi@2024.2.1,gcc@11.4.1] providers: mpi:: [intel-oneapi-mpi] @@ -8,13 +8,13 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.12%oneapi@2024.1.2 + - spec: intel-oneapi-mpi@2021.13%oneapi@2024.2.1 prefix: /opt/intel/oneapi intel-oneapi-mkl: externals: - - spec: intel-oneapi-mkl@2024.1%oneapi@2024.1.2 + - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.1 prefix: /opt/intel/oneapi intel-oneapi-runtime: externals: - - spec: intel-oneapi-runtime@2024.1.2%oneapi@2024.1.2 + - spec: intel-oneapi-runtime@2024.2.1%oneapi@2024.2.1 prefix: /opt/intel/oneapi diff --git a/configs/sites/tier2/bounty/README.md b/configs/sites/tier2/bounty/README.md new file mode 100644 index 000000000..00aaebc69 --- /dev/null +++ b/configs/sites/tier2/bounty/README.md @@ -0,0 +1 @@ +Bounty is one of @climbfuji's development systems (Dell Laptop running Ubuntu Linux 22.04 under Windows WSL2). diff --git a/configs/sites/tier2/bounty/compilers.yaml b/configs/sites/tier2/bounty/compilers.yaml new file mode 100644 index 000000000..76bb97735 --- /dev/null +++ b/configs/sites/tier2/bounty/compilers.yaml @@ -0,0 +1,27 @@ +compilers: +- compiler: + spec: gcc@=11.4.0 + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + flags: {} + operating_system: ubuntu22.04 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] +- compiler: + spec: gcc@=12.3.0 + paths: + cc: /usr/bin/gcc-12 + cxx: /usr/bin/g++-12 + f77: /usr/bin/gfortran-12 + fc: /usr/bin/gfortran-12 + flags: {} + operating_system: ubuntu22.04 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] diff --git a/configs/sites/tier1/casper/config.yaml b/configs/sites/tier2/bounty/config.yaml similarity index 100% rename from configs/sites/tier1/casper/config.yaml rename to configs/sites/tier2/bounty/config.yaml diff --git a/configs/sites/tier2/bounty/mirrors.yaml b/configs/sites/tier2/bounty/mirrors.yaml new file mode 100644 index 000000000..95ab7092e --- /dev/null +++ b/configs/sites/tier2/bounty/mirrors.yaml @@ -0,0 +1,2 @@ +mirrors: + local-source: file:///home/dom/prod/spack-source-cache diff --git a/configs/sites/tier2/bounty/modules.yaml b/configs/sites/tier2/bounty/modules.yaml new file mode 100644 index 000000000..57e6be9fe --- /dev/null +++ b/configs/sites/tier2/bounty/modules.yaml @@ -0,0 +1,10 @@ +modules: + default: + enable:: + - tcl + tcl: + include: + # List of packages for which we need modules that are blacklisted by default + - openmpi + - mpich + - python diff --git a/configs/sites/tier2/bounty/packages.yaml b/configs/sites/tier2/bounty/packages.yaml new file mode 100644 index 000000000..ce53d2fb1 --- /dev/null +++ b/configs/sites/tier2/bounty/packages.yaml @@ -0,0 +1,73 @@ +packages: + autoconf: + externals: + - spec: autoconf@2.71 + prefix: /usr + automake: + externals: + - spec: automake@1.16.5 + prefix: /usr + binutils: + externals: + - spec: binutils@2.38 + prefix: /usr + coreutils: + externals: + - spec: coreutils@8.32 + prefix: /usr + diffutils: + externals: + - spec: diffutils@3.8 + prefix: /usr + findutils: + externals: + - spec: findutils@4.8.0 + prefix: /usr + gawk: + externals: + - spec: gawk@5.1.0 + prefix: /usr + gettext: + externals: + - spec: gettext@0.21 + prefix: /usr + git: + externals: + - spec: git@2.34.1~tcltk + prefix: /usr + git-lfs: + externals: + - spec: git-lfs@3.0.2 + prefix: /usr + gmake: + externals: + - spec: gmake@4.3 + prefix: /usr + groff: + externals: + - spec: groff@1.22.4 + prefix: /usr + m4: + externals: + - spec: m4@1.4.18 + prefix: /usr + perl: + externals: + - spec: perl@5.34.0~cpanm+opcode+open+shared+threads + prefix: /usr + sed: + externals: + - spec: sed@4.8 + prefix: /usr + tar: + externals: + - spec: tar@1.34 + prefix: /usr + texlive: + externals: + - spec: texlive@20220321 + prefix: /usr + wget: + externals: + - spec: wget@1.21.2 + prefix: /usr diff --git a/configs/sites/tier2/bounty/packages_gcc.yaml b/configs/sites/tier2/bounty/packages_gcc.yaml new file mode 100644 index 000000000..a91408366 --- /dev/null +++ b/configs/sites/tier2/bounty/packages_gcc.yaml @@ -0,0 +1,5 @@ +packages: + all: + compiler:: [gcc@12.3.0] + providers: + mpi:: [openmpi@5.0.3] diff --git a/configs/sites/tier1/casper/compilers.yaml b/configs/sites/tier2/casper/compilers.yaml similarity index 100% rename from configs/sites/tier1/casper/compilers.yaml rename to configs/sites/tier2/casper/compilers.yaml diff --git a/configs/sites/tier2/casper/config.yaml b/configs/sites/tier2/casper/config.yaml new file mode 100644 index 000000000..8e40cc0c9 --- /dev/null +++ b/configs/sites/tier2/casper/config.yaml @@ -0,0 +1,2 @@ +config: + build_jobs: 4 diff --git a/configs/sites/tier1/casper/mirrors.yaml b/configs/sites/tier2/casper/mirrors.yaml similarity index 100% rename from configs/sites/tier1/casper/mirrors.yaml rename to configs/sites/tier2/casper/mirrors.yaml diff --git a/configs/sites/tier1/casper/modules.yaml b/configs/sites/tier2/casper/modules.yaml similarity index 100% rename from configs/sites/tier1/casper/modules.yaml rename to configs/sites/tier2/casper/modules.yaml diff --git a/configs/sites/tier1/casper/packages.yaml b/configs/sites/tier2/casper/packages.yaml similarity index 100% rename from configs/sites/tier1/casper/packages.yaml rename to configs/sites/tier2/casper/packages.yaml diff --git a/configs/sites/tier2/macos.default/packages.yaml b/configs/sites/tier2/macos.default/packages.yaml index f08498c65..ddb2d126f 100644 --- a/configs/sites/tier2/macos.default/packages.yaml +++ b/configs/sites/tier2/macos.default/packages.yaml @@ -1,9 +1,14 @@ packages: cairo: variants: ~png ~svg + # https://github.com/JCSDA/spack-stack/issues/1211 + fms: + require: + - '+shared' git: buildable: false libiconv: buildable: false wgrib2: - variants: ~openmp + require: + - '~openmp' diff --git a/configs/templates/nco/approved_list.txt b/configs/templates/nco/approved_list.txt new file mode 100644 index 000000000..3fcdd8216 --- /dev/null +++ b/configs/templates/nco/approved_list.txt @@ -0,0 +1,225 @@ +awscli +bacio +boost +bufr +bufr_dump +bufr_tranjb +bufr_util +cairo +cdo +cfitsio +cfp +cmake +cray-parallel-netcdf +crtm +curl +degrib +dumpmb +ecflow +eigen +emapf-c +esmf +fms +g2 +g2c +g2tmpl +gdal +gempak +geos +gfsio +git +grib_util +gsl +hdf5 +imagemagick +ip +ip2 +jasper +lamp_shared +landsfcutil +libdap4 +libgeotiff +libiconv +libjpeg +libjpeg-turbo +libpng +libtiff +libxml2 +libxmlparse +libxrender +madis +manic +mapl +mbedtls +met +metis +metplus +mos_shared +nco +ncview +nemsio +nemsiogfs +netcdf +perl +pigz +pio +pixman +prepobs +prod_envir +prod_util +proj +python +readline +rsync +schism +scotch +sfcio +sigio +sp +subversion +udunits +upp +util_shared +w3emc +w3nco +wgrib2 +wrf_io +xyz2shp +xz +zlib +antlr +apr +apr-util +autoconf +automake +bdftopcf +berkeley-db +bison +bzip2 +ca-certificates-mozilla +diffutils +elfutils +expat +fftw +findutils +flex +font-util +fontconfig +fontsproto +freetype +fribidi +gdbm +gettext +glib +gmake +gnuconfig +gobject-introspection +gperf +harfbuzz +help2man +icu4c +inputproto +json-c +kbproto +libaec +libbsd +libedit +libffi +libfontenc +libice +libidn2 +libmd +libpthread-stubs +libsigsegv +libsm +libtool +libunistring +libwebp +libx11 +libxau +libxaw +libxcb +libxcrypt +libxdmcp +libxext +libxfont +libxmu +libxpm +libxt +libyaml +lz4 +m4 +meson +mkfontdir +mkfontscale +nasm +ncurses +netcdf-cxx4 +ninja +openblas +openssh +openssl +pango +pcre +pcre2 +pkgconf +popt +py-botocore +py-bottleneck +py-cftime +py-colorama +py-cython +py-docutils +py-flit-core +py-jmespath +py-netcdf4 +py-numexpr +py-numpy +py-packaging +py-pandas +py-pip +py-pyasn1 +py-python-dateutil +py-pytz +py-pyyaml +py-rsa +py-s3transfer +py-setuptools +py-setuptools-scm +py-setuptools-scm-git-archive +py-six +py-tomli +py-typing-extensions +py-urllib3 +py-versioneer +py-wheel +py-xarray +qt +re2c +renderproto +sqlite +tar +unixodbc +utf8proc +util-linux-uuid +util-macros +which +xcb-proto +xextproto +xproto +xtrans +xxhash +zstd +atlas +eckit +fckit +gsl-lite +qhull +gftl-shared +nccmp +cray-mpich +gcc-runtime +glibc +parallelio +pkg-config +ecbuild diff --git a/configs/templates/nco/spack.yaml b/configs/templates/nco/spack.yaml new file mode 100644 index 000000000..12787de13 --- /dev/null +++ b/configs/templates/nco/spack.yaml @@ -0,0 +1,140 @@ +spack: + concretizer: + unify: when_possible + + config: + install_tree: + root: $env/install + modules: + default: + roots: + lmod: $env/install/modulefiles + tcl: $env/install/modulefiles + + view: false + include: [] + + specs: + - awscli + - bacio + - boost + - bufr + - cairo + - cdo + - cfitsio + - cmake + - crtm + - curl + - ecflow + - eckit + - ecmwf-atlas + - eigen + - esmf + - fckit + - fms + - g2 + - g2c + - g2tmpl + - gdal + - geos + - gfsio + - gftl-shared + - git + - grib-util + - gsl + - gsl-lite + - hdf5 + - imagemagick + - ip + - ip2 + - jasper + - landsfcutil + - libgeotiff + - libiconv + - libjpeg-turbo + - libpng + - libtiff + - libxml2 + - libxrender + - madis + - mapl + - mbedtls + - met + - metis + - metplus + - nccmp + - gsi-ncdiag + - nco + - ncview + - nemsio + - nemsiogfs + - netcdf-c + - netcdf-fortran + - perl + - pigz + - parallelio + - pixman + - proj + - python + - qhull + - readline + - rsync + - scotch + - sfcio + - sigio + - sp + - subversion + - udunits + - upp + - w3emc + - w3nco + - wgrib2 + - wrf-io + + packages: + # Strictly set virtual package providers: + mpi: + require: cray-mpich + fftw-api: + require: fftw + blas: + require: openblas + lapack: + require: openblas + jpeg: + require: libjpeg-turbo + # Individual package settings. + # Use `require::` to override require's from {common,site}/packages.yaml, + # and `require: - '...'` to add to existing ones. + cdo: + require: + - 'grib2=none' # avoids eccodes/grib-api dependency + curl: + require: + - '~nghttp2' # avoid nghttp2 dependency + ecmwf-atlas: + require:: '@0.33.0 +fckit ~trans ~fftw' # disabling trans and fftw to minimize dependencies; older version because newer eckit doesn't build with intel@19 + esmf: + require: + - 'snapshot=none' # make sure spack-stack doesn't accidentally give us a beta snapshot + fckit: + require:: '@0.10.1 +eckit' # older version needed for older eckit & ecmwf-atlas + glib: + require: '@:2.74.3' # newer versions don't build with intel@19 + mapl: + require: + - '~pflogger ~fargparse ~extdata2g' # minimize dependencies + netcdf-c: + require:: '@4.9.2 +mpi ~parallel-netcdf ~dap ~blosc ~szip build_system=autotools' # disabling dap prevents network access; disabling blosc and szip reduces dependencies + py-pandas: + require:: '~excel' # minimize dependencies + subversion: + require: + - '~serf +pic' # avoid serf dependency + # Explicitly require gcc when needed so Spack won't do anything weird: + boost: + require: '%gcc' + gcc-runtime: + require: '%gcc' + harfbuzz: + require: '%gcc' diff --git a/doc/source/NewSiteConfigs.rst b/doc/source/NewSiteConfigs.rst index b4116538c..036fa41f9 100644 --- a/doc/source/NewSiteConfigs.rst +++ b/doc/source/NewSiteConfigs.rst @@ -152,19 +152,31 @@ These instructions are meant to be a reference that users can follow to set up t brew install libiconv # Note - need to pin to version 5 brew install qt@5 + brew install m4 + # Note - For Sonoma only: install pkg-config, diffutils and clingo + brew install pkg-config + brew install diffutils + brew install clingo # Note - only needed for running JCSDA's # JEDI-Skylab system (using R2D2 localhost) brew install mysql .. note:: - On an Intel based Mac, you will need to also install pkg-config using homebrew. + On an Intel based Mac, regardless of MacOS version, you will need to also install pkg-config using homebrew. This is done to work around an issue where libraries (eg, openssl) cannot be properly found during code compilation. .. code-block:: console brew install pkg-config # Intel based Mac only +.. note:: + For spack-stack-1.8.0 and newer, you must be using cmake 3.26+. + Make sure you upgrade cmake in homebrew. + +.. code-block:: console + brew upgrade cmake + 4. Configure your terminal to use the homebrew installed bash After installing bash with homebrew, you need to change your terminal application's default command to use :code:`$HOMEBREW_ROOT/bin/bash`. @@ -229,7 +241,8 @@ Remember to activate the ``lua`` module environment and have MacTeX in your sear spack external find --scope system \ --exclude bison --exclude openssl \ - --exclude python --exclude gettext + --exclude python --exclude gettext \ + --exclude m4 spack external find --scope system perl spack external find --scope system wget @@ -245,9 +258,12 @@ Remember to activate the ``lua`` module environment and have MacTeX in your sear PATH="$HOMEBREW_ROOT/opt/curl/bin:$PATH" \ spack external find --scope system curl - PATH="$HOMEBREW_ROOT/opt/qt@5/bin:$PATH" \ + PATH="$HOMEBREW_ROOT/opt/qt5/bin:$PATH" \ spack external find --scope system qt + PATH="$HOMEBREW_ROOT/opt/m4/bin:$PATH" \ + spack external find --scope system m4 + # Optional, only if planning to build jedi-tools environment with LaTeX support # The texlive bin directory must have been added to PATH (see above) spack external find --scope system texlive @@ -459,21 +475,15 @@ The following instructions were used to prepare a basic Ubuntu 20.04 or 22.04 LT apt install -y automake apt install -y autopoint apt install -y gettext - apt install -y xterm apt install -y texlive apt install -y libcurl4-openssl-dev apt install -y libssl-dev - apt install -y meson - apt install -y bison # Note - only needed for running JCSDA's # JEDI-Skylab system (using R2D2 localhost) apt install -y mysql-server apt install -y libmysqlclient-dev - # Python - apt install -y python3-dev python3-pip - # Exit root session exit diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 319b53b4a..2ea670ac2 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -3,103 +3,34 @@ Pre-configured sites ************************* -.. note:: - The information on this page usually refers to the latest stable release of spack-stack. There are usually no updates for the latest developmental code for preconfigured sites. Further, current and previous releases of spack-stack are often updated with new packages, and sometimes it is necessary to rebuild spack-stack environments, for example in case an HPC receives a major software update. Also in this case, the documentation available on readthedocs is not updated. Be sure to check the spack-stack Wiki for updates to current and previous releases before using the information below! - -Directory ``configs/sites`` contains site configurations for several HPC systems, as well as minimal configurations for macOS and Linux. The macOS and Linux configurations are **not** meant to be used as is, as user setups and package versions vary considerably. Instructions for adding this information can be found in :numref:`Section %s `. - Pre-configured sites are split into two categories: Tier 1 with officially supported spack-stack installations (see :numref:`Section %s `), and Tier 2 (sites with configuration files that were tested or contributed by others in the past, but that are not officially supported by the spack-stack team; see :numref:`Section %s `). -============================================================= -Officially supported spack-stack 1.7.0 installations (tier 1) -============================================================= - -Ready-to-use spack-stack 1.7.0 installations are available on the following, fully supported platforms. This version supports JEDI-Skylab and various UFS and related applications (UFS Weather Model, EMC Global Workflow, GSI, UFS Short Range Weather Application). Amazon Web Services AMI are available in the US East 1 or 2 regions. - -+---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| Organization | System | Compilers | Location | Maintainers (principal/backup)| -+=====================+==================================+=================+==============================================================================+===============================+ -| **HPC platforms** | -+---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| | Hercules | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.7.0/envs`` | EPIC / JCSDA | -| MSU +----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.7.0/envs`` | EPIC / JCSDA | -+---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| | Discover SCU16 | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/scu16/spack-stack-1.7.0/envs`` | JCSDA | -| NASA +----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| | Discover SCU17 | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-1.7.0/envs`` | JCSDA | -+---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| NCAR-Wyoming + Derecho | GCC, Intel | ``/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.7.0/envs`` | EPIC / JCSDA | -+---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| NOAA (NCEP) | Acorn | Intel | ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.7.0/envs`` | NOAA-EMC | -+---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| | Gaea | Intel | ``/ncrc/proj/epic/spack-stack/spack-stack-1.7.0/envs`` | EPIC / NOAA-EMC | -| +----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| NOAA (RDHPCS) | Hera | GCC, Intel | ``/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.7.0/envs`` | EPIC / NOAA-EMC | -| +----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| | Jet | GCC, Intel | ``/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.7.0/envs`` | EPIC / NOAA-EMC | -+---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| | Narwhal | GCC, Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.7.0/envs`` | NRL | -| U.S. Navy (HPCMP) +----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| | Nautilus | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.7.0/envs`` | NRL | -+---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| Univ. of Wisconsin | S4 | Intel | ``/data/prod/jedi/spack-stack/spack-stack-1.7.0/envs`` | JCSDA | -+---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| **Cloud platforms** | -+---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| | AMI Red Hat 8 | GCC | ``/home/ec2-user/spack-stack/spack-stack-1.7.0/envs`` | JCSDA | -+ Amazon Web Services +----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| | Parallelcluster JCSDA R&D | GCC, Intel | *currently unavailable* | JCSDA | -+---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ -| NOAA (RDHPCS) | RDHPCS Cloud (Parallel Works) | Intel | ``/contrib/spack-stack/spack-stack-1.7.0/envs`` | EPIC / JCSDA | -+---------------------+----------------------------------+-----------------+------------------------------------------------------------------------------+-------------------------------+ - -.. note:: - This release of spack-stack uses different versions of ``mapl`` with different variants, depending on the version of the compiler and whether the system is used for UFS or GEOS. Please see the following table. - -+----------------------------+--------------------------------------+-----------------------------------------------------------------------+ -| Compiler | mapl configuration | Affected systems | -+============================+======================================+=======================================================================+ -| gcc (any) | ``mapl@2.40.3 +pflogger +extdata2g`` | All systems with GCC stacks | -+----------------------------+--------------------------------------+-----------------------------------------------------------------------+ -| intel@2021.6.0 and earlier | ``mapl@2.40.3 +pflogger +extdata2g`` | Discover SCU16, Acorn, Hera, Jet, Narwhal, Nautilus, S4, RDHPCS Cloud | -+----------------------------+--------------------------------------+-----------------------------------------------------------------------+ -| intel@2021.7.0 and later | ``mapl@2.40.3 ~pflogger ~extdata2g`` | Hercules, Orion, Acorn, Gaea and Derecho | -+----------------------------+--------------------------------------+-----------------------------------------------------------------------+ -| intel@2021.7.0 and later | ``mapl@2.43.0 +pflogger +extdata2g`` | Discover SCU17 | -+----------------------------+--------------------------------------+-----------------------------------------------------------------------+ - -.. note:: - We have noted problems on some - not all - platforms with ``intel@2021.5.0`` when we switched from ``zlib`` to ``zlib-ng`` in spack-stack-1.7.0. These issues went away when using a different version of the compiler (anything between 2021.3.0 and 2021.11.0). It is therefore recommended to avoid using ``intel@2021.5.0`` unless it is the only option. - -**To use one of the above installations** via the system default environment module system, and adding certain modules first (see individual sections below), add ``/install/modulefiles/Core`` to the path from the above table and prepend that path to $MODULEPATH, e.g., - -.. code-block:: console +Directories ``configs/sites/tier1`` and ``configs/sites/tier2`` contain site configurations for several HPC systems, as well as minimal configurations for macOS and Linux. The macOS and Linux configurations are **not** meant to be used as is, as user setups and package versions vary considerably. Instructions for adding this information can be found in :numref:`Section %s `. - # On Gaea: - module use /ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core - module load stack-intel - module load bacio netcdf-c ... +As of spack-stack-1.8.0, this page provides general information on the supported platforms, such as the location of the spack-stack installations on tier 1 platforms and instructions on how to set up an environment for **building** spack-stack environments. Information on **using** spack-stack environments for development of downstream applications is available on the spack-stack wiki: https://github.com/JCSDA/spack-stack/wiki -For more information about a specific platform, please see the individual sections below. +.. _EnvironmentNamingConventions: -For questions or problems, please consult the known issues in :numref:`Section %s `, the currently open GitHub `issues `_ and `discussions `_ first. +============================================================= +Environment naming conventions +============================================================= -.. _supplemental_environments: +The following naming conventions are used on all fully-supported (tier 1) sites. Environments are named using an abbreviated prefix that depends on the template/purpose, followed by the compiler name and version: ``prefix-compiler-version``. The following table lists the prefices and gives a few examples. -========================= -Supplemental environments -========================= - -The following is a list of supplemental or "add-on" environments that are maintained through spack-stack. Note that not all are included with every release; see the third column to determine release location and look under ``envs/`` subdirectory (i.e., same parent directory as ``ue-*`` directories per the above table). Check the installation directories to verify which package versions are available before using them. ++----------------------------------+---------------------------------------------------------+-------------------+------------------------------+ +| Template (``configs/templates``) | Description | Prefix | Examples | ++==================================+=========================================================+===================+==============================+ +| ``unified-dev`` | Unified environment for all organizations/applications | ``ue`` | ``ue-intel-2021.10.0`` | ++----------------------------------+---------------------------------------------------------+-------------------+------------------------------+ +| ``skylab-dev`` | JEDI/Skylab environment for JEDI, models, EWOK | ``se`` | ``se-apple-clang-14.0.6`` | ++----------------------------------+---------------------------------------------------------+-------------------+------------------------------+ +| ``neptune-dev`` | NEPTUNE standalone environment (with xNRL Python) | ``ne`` | ``ne-oneapi-2024.2.1`` | ++----------------------------------+---------------------------------------------------------+-------------------+------------------------------+ +| ``gsi-addon-dev`` | GSI addon (chained) environment on top of unified env. | ``gsi`` | ``gsi-gcc-13.3.0`` | ++----------------------------------+---------------------------------------------------------+-------------------+------------------------------+ +| ``unified-dev`` with new ESMF | Unified environment with new ESMF (chained from ``ue``) | ``ue-esmf870b99`` | ``ue-esmf870b99-aocc-4.2.0`` | ++----------------------------------+---------------------------------------------------------+-------------------+------------------------------+ -+------------------+---------------------------------------------------------+------------------------+-------------------------------------------+ -| Environment name | Description | spack-stack release(s) | Platforms | -+==================+=========================================================+========================+===========================================+ -| gsi-addon-* | Supports GSI and related applications | 1.6.0, 1.7.0 | Hera, Hercules, Gaea, Jet, S4 | -+------------------+---------------------------------------------------------+------------------------+-------------------------------------------+ -| ufswm-* | Supports UFS Weather Model with WCOSS2 package versions | 1.6.0 | Acorn, Hera, Hercules, Jet, Orion | -+------------------+---------------------------------------------------------+------------------------+-------------------------------------------+ .. _Preconfigured_Sites_Tier1: @@ -107,70 +38,72 @@ The following is a list of supplemental or "add-on" environments that are mainta Pre-configured sites (tier 1) ============================================================= ++---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| Organization | System | Compilers | Location of top-level spack-stack directory | Maintainers | ++=====================+=======================+====================+========================================================+=================+ +| **HPC platforms** | ++---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| | Hercules | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/hercules/`` | EPIC / JCSDA | +| MSU +-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/orion/`` | EPIC / JCSDA | ++---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| | Discover SCU16 | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/scu16/`` | JCSDA | +| NASA +-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| | Discover SCU17 | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/scu17/`` | JCSDA | ++---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| NCAR-Wyoming + Derecho | GCC, Intel | ``/glade/work/epicufsrt/contrib/spack-stack/derecho/`` | EPIC / JCSDA | ++---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| NOAA (NCEP) | Acorn | Intel | ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/`` | NOAA-EMC | ++---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| | Gaea C5 | Intel | ``/ncrc/proj/epic/spack-stack/`` | EPIC / NOAA-EMC | +| +-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| | Gaea C6 | Intel | ``/ncrc/proj/epic/spack-stack/c6/`` | EPIC / NOAA-EMC | +| NOAA (RDHPCS) +-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| | Hera | GCC, Intel | ``/contrib/spack-stack/`` | EPIC / NOAA-EMC | +| +-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| | Jet | GCC, Intel | ``/contrib/spack-stack`` | EPIC / NOAA-EMC | ++---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| | Narwhal | GCC, Intel, oneAPI | ``/p/app/projects/NEPTUNE/spack-stack/`` | NRL | +| U.S. Navy (HPCMP) +-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| | Nautilus | Intel | ``/p/app/projects/NEPTUNE/spack-stack/`` | NRL | ++---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| Univ. of Wisconsin | S4 | Intel | ``/data/prod/jedi/spack-stack/`` | JCSDA | ++---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| **Cloud platforms** | ++---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| | AMI Red Hat 8 | GCC | ``/home/ec2-user/spack-stack/`` | JCSDA | ++ Amazon Web Services +-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| | Parallelcluster JCSDA | GCC, Intel | *currently unavailable* | JCSDA | ++---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ +| NOAA (RDHPCS) | RDHPCS Parallel Works | Intel | ``/contrib/spack-stack-rocky8/`` | EPIC / JCSDA | ++---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ + .. _Preconfigured_Sites_Orion: ------------------------------ MSU Orion ------------------------------ -The following is required for building new spack environments and for using spack to build and run software. - -.. note:: For spack-stack develop, use module path ``/work/noaa/epic/role-epic/spack-stack/orion/modulefiles-rocky9`` instead of ``/work/noaa/epic/role-epic/spack-stack/orion/modulefiles``. Note also that the ``openmpi`` version for ``gcc@12.2.0`` changes to ``4.1.4``. +The following is required for building new spack environments with any supported compiler on this platform. .. code-block:: console module purge - module use /work/noaa/epic/role-epic/spack-stack/orion/modulefiles - module load ecflow/5.8.4 - -For ``spack-stack-1.7.0`` with Intel, load the following modules after loading miniconda and ecflow: - -.. code-block:: console - - module use /work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.7.0/envs/ue-intel/install/modulefiles/Core - module load stack-intel/2021.9.0 - module load stack-intel-oneapi-mpi/2021.9.0 - module load stack-python/3.10.13 -For ``spack-stack-1.7.0`` with GNU, load the following modules after loading miniconda and ecflow: -.. code-block:: console - - module use /work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.7.0/envs/ue-gcc/install/modulefiles/Core - module load stack-gcc/12.2.0 - module load stack-openmpi/4.1.6 - module load stack-python/3.10.13 +.. _Preconfigured_Sites_Hercules: ------------------------------ MSU Hercules ------------------------------ -The following is required for building new spack environments and for using spack to build and run software. +The following is required for building new spack environments with any supported compiler on this platform. .. code-block:: console module purge - module use /work/noaa/epic/role-epic/spack-stack/hercules/modulefiles - module load ecflow/5.8.4 - module load git-lfs/3.1.2 - -For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: - -.. code-block:: console - - module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.7.0/envs/ue-intel/install/modulefiles/Core - module load stack-intel/2021.9.0 - module load stack-intel-oneapi-mpi/2021.9.0 - module load stack-python/3.10.13 - -For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: + # No need to load modules, spack-stack-1.8.0 have no dependences -.. code-block:: console - - module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.7.0/envs/ue-gcc/install/modulefiles/Core - module load stack-gcc/12.2.0 - module load stack-openmpi/4.1.6 - module load stack-python/3.10.13 .. _Preconfigured_Sites_Discover_SCU16: @@ -178,7 +111,7 @@ For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: NASA Discover SCU16 ------------------------------ -The following is required for building new spack environments and for using spack to build and run software. +The following is required for building new spack environments with any supported compiler on this platform. .. code-block:: console @@ -186,56 +119,20 @@ The following is required for building new spack environments and for using spac module use /discover/swdev/gmao_SIteam/modulefiles-SLES12 module use /discover/swdev/jcsda/spack-stack/scu16/modulefiles module load miniconda/3.9.7 - module load ecflow/5.8.4 - -For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: - -.. code-block:: console - - module use /gpfsm/dswdev/jcsda/spack-stack/scu16/spack-stack-1.7.0/envs/ue-intel-2021.6.0/install/modulefiles/Core - module load stack-intel/2021.6.0 - module load stack-intel-oneapi-mpi/2021.6.0 - module load stack-python/3.10.13 - -For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: - -.. code-block:: console - module use /gpfsm/dswdev/jcsda/spack-stack/scu16/spack-stack-1.7.0/envs/ue-gcc-12.1.0/install/modulefiles/Core - module load stack-gcc/12.1.0 - module load stack-openmpi/4.1.3 - module load stack-python/3.10.13 +.. _Preconfigured_Sites_Discover_SCU17: ------------------------------ NASA Discover SCU17 ------------------------------ -The following is required for building new spack environments and for using spack to build and run software. +The following is required for building new spack environments with any supported compiler on this platform. .. code-block:: console module purge module use /discover/swdev/gmao_SIteam/modulefiles-SLES15 module use /discover/swdev/jcsda/spack-stack/scu17/modulefiles - module load ecflow/5.11.4 - -For ``spack-stack-1.7.0`` with Intel, load the following modules after loading ecflow: - -.. code-block:: console - - module use /gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-1.7.0/envs/ue-intel-2021.10.0/install/modulefiles/Core - module load stack-intel/2021.10.0 - module load stack-intel-oneapi-mpi/2021.10.0 - module load stack-python/3.10.13 - -For ``spack-stack-1.7.0`` with GNU, load the following modules after loading ecflow: - -.. code-block:: console - - module use /gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-1.7.0/envs/ue-gcc-12.3.0/install/modulefiles/Core - module load stack-gcc/12.3.0 - module load stack-openmpi/4.1.6 - module load stack-python/3.10.13 .. _Preconfigured_Sites_Narwhal: @@ -243,7 +140,7 @@ For ``spack-stack-1.7.0`` with GNU, load the following modules after loading ecf NAVY HPCMP Narwhal ------------------------------ -With Intel, the following is required for building new spack environments and for using spack to build and run software. Don't use ``module purge`` on Narwhal! +The following is required for building new spack environments with Intel on this platform.. Don't use ``module purge`` on Narwhal! .. code-block:: console @@ -254,32 +151,20 @@ With Intel, the following is required for building new spack environments and fo module load intel-classic/2023.2.0 module unload cray-mpich module unload craype-network-ofi - module load craype-network-ucx - # Warning. Do not load cray-mpich-ucx/8.1.21 + # Warning. Do not load craype-network-ucx + # or cray-mpich-ucx/8.1.21! # There is a bug in the modulefile that prevents # spack from setting the environment for its # build steps when the module is already # loaded. Instead, let spack load it when the # package requires it. + #module load craype-network-ucx #module load cray-mpich-ucx/8.1.21 module load libfabric/1.12.1.2.2.1 module unload cray-libsci module load cray-libsci/23.05.1.4 -THIS SECTION IS OUT OF DATE, REFER TO 1.7.0 RELEASE DOCUMENTATION - - For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: - -.. code-block:: console - - # These extra steps are required for performance reason, ofi is about 30% slower than ucx - # Note we can't load craype-network-ucx for building spack-stack environments, must do here - module unload craype-network-ofi - module load craype-network-ucx - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.7.0/envs/ue-intel-2021.4.0/install/modulefiles/Core - module load stack-intel/2021.10.0 - module load stack-cray-mpich/8.1.21 - module load stack-python/3.10.13 - -With GNU, the following is required for building new spack environments and for using spack to build and run software. Don't use ``module purge`` on Narwhal! +The following is required for building new spack environments with GNU on this platform.. Don't use ``module purge`` on Narwhal! .. code-block:: console @@ -290,30 +175,19 @@ With GNU, the following is required for building new spack environments and for module load gcc/10.3.0 module unload cray-mpich module unload craype-network-ofi - module load craype-network-ucx - # Warning. Do not load cray-mpich-ucx/8.1.21 + # Warning. Do not load craype-network-ucx + # or cray-mpich-ucx/8.1.21! # There is a bug in the modulefile that prevents # spack from setting the environment for its # build steps when the module is already # loaded. Instead, let spack load it when the # package requires it. + #module load craype-network-ucx #module load cray-mpich-ucx/8.1.21 module load libfabric/1.12.1.2.2.1 module unload cray-libsci module load cray-libsci/23.05.1.4 -THIS SECTION IS OUT OF DATE, REFER TO 1.7.0 RELEASE DOCUMENTATION - For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: - -.. code-block:: console - - # These extra steps are required for performance reason, ofi is about 30% slower than ucx - # Note we can't load craype-network-ucx for building spack-stack environments, must do here - module unload craype-network-ofi - module load craype-network-ucx - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.7.0/envs/ue-gcc-10.3.0/install/modulefiles/Core - module load stack-gcc/10.3.0 - module load stack-cray-mpich/8.1.21 - module load stack-python/3.10.13 .. _Preconfigured_Sites_Nautilus: @@ -321,32 +195,13 @@ THIS SECTION IS OUT OF DATE, REFER TO 1.7.0 RELEASE DOCUMENTATION - For ``spack- NAVY HPCMP Nautilus ------------------------------ -With Intel, the following is required for building new spack environments and for using spack to build and run software. - -.. code-block:: console - - umask 0022 - module purge - -For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: - -.. code-block:: console - - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.7.0/envs/ue-intel-2021.5.0/install/modulefiles/Core - module load stack-intel/2021.5.0 - module load stack-openmpi/4.1.6 - module load stack-python/3.10.13 - -With AMD clang/flang (aocc), the following is required for building new spack environments and for using spack to build and run software. +The following is required for building new spack environments with any supported compiler on this platform. .. code-block:: console umask 0022 module purge -.. note:: - - ``spack-stack-1.7.0`` is not yet supported with the Arm clang/flang compilers. Use Intel instead. .. _Preconfigured_Sites_Derecho: @@ -354,7 +209,9 @@ With AMD clang/flang (aocc), the following is required for building new spack en NCAR-Wyoming Derecho -------------------- -The following is required for building new spack environments and for using spack to build and run software. +The following is required for building new spack environments with any supported compiler on this platform. + +**NEEDS UPDATING** .. code-block:: console @@ -363,28 +220,7 @@ The following is required for building new spack environments and for using spac export LMOD_TMOD_FIND_FIRST=yes module load ncarenv/23.09 module use /glade/work/epicufsrt/contrib/spack-stack/derecho/modulefiles - module load ecflow/5.8.4 - -For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: - -.. code-block:: console - - module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.7.0/envs/ue-intel/install/modulefiles/Core - module load stack-intel/2021.10.0 - module load stack-cray-mpich/8.1.25 - module load stack-python/3.10.13 -For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: - -.. code-block:: console - - module use /glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.7.0/envs/ue-gcc/install/modulefiles/Core - module load stack-gcc/12.2.0 - module load stack-cray-mpich/8.1.25 - module load stack-python/3.10.13 - -.. note:: - CISL restricts the amount of memory available for processes on the login nodes. For example, it is impossible to compile JEDI with even one task (``make -j1``) with the Intel compiles in release mode (``-O2``). We therefore recommend compiling on compute nodes using interactive jobs, if possible. .. _Preconfigured_Sites_Acorn: @@ -392,20 +228,14 @@ For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: NOAA Acorn (WCOSS2 test system) ------------------------------- -For spack-stack-1.7.0, the meta modules are in ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.7.0/envs/ue-intel{19,2022}/modulefiles/Core``. +On WCOSS2 OpenSUSE sets ``CONFIG_SITE`` which causes libraries to be installed in ``lib64``, breaking the ``lib`` assumption made by some packages. Therefore, ``CONFIG_SITE`` should remain set to empty in ``compilers.yaml``. -On WCOSS2 OpenSUSE sets ``CONFIG_SITE`` which causes libraries to be installed in ``lib64``, breaking the ``lib`` assumption made by some packages. Therefore, ``CONFIG_SITE`` should be set to empty in ``compilers.yaml``. Also, don't use ``module purge`` on Acorn! +For official deployments on ``spack-stack`` on Acorn, be mindful of umask and group ownership, as these can be finicky. The umask value should be 002, otherwise various files can be assigned to the wrong group. In any case, running something to the effect of ``chgrp nceplibs -R`` and ``chmod o+rX -R`` after the whole installation is done is a good idea. -When installing an official ``spack-stack`` on Acorn, be mindful of umask and group ownership, as these can be finicky. The umask value should be 002, otherwise various files can be assigned to the wrong group. In any case, running something to the effect of ``chgrp nceplibs -R`` and ``chmod o+rX -R`` after the whole installation is done is a good idea. - -Due to a combined quirk of Cray and Spack, the ``PrgEnv-gnu`` and ``gcc`` modules must be loaded when `ESMF` is being installed with ``gcc``. - -As of spring 2023, there is an inconsistency in ``libstdc++`` versions on Acorn between the login and compute nodes. It is advisable to compile on the compute nodes, which requires running ``spack fetch`` prior to installing through a batch job. - -Note that certain packages, such as recent versions of `py-scipy`, cannot be compiled on compute nodes because their build systems require internet access. +Note that for the installation using Intel 19, the system GCC, 7.5.0, is used on the backend for the Intel compiler. More recent versions of GCC are not reliably compatible. Likewise, for Intel 2022, GCC 10.2.0 is used on the backend. Intel 19 is not reliably compatible with C++17 standards, and Intel 2022 is not reliably compatible with C++20. Without a handful of package version restrictions, certain package builds will break, usually in the configure stage. .. note:: - System-wide ``spack`` software installations are maintained by NCO on this platform which are not associated with spack-stack. The spack-stack official installations use those installations for one dependency (git-lfs). + System-wide ``spack`` software installations are maintained by NCO on this platform, which are not associated with spack-stack. .. _Preconfigured_Sites_Parallel_Works: @@ -413,57 +243,32 @@ Note that certain packages, such as recent versions of `py-scipy`, cannot be com NOAA Parallel Works (AWS, Azure, Gcloud) ---------------------------------------- -The following is required for building new spack environments and for using spack to build and run software. The default module path needs to be removed, otherwise spack detects the system as Cray. +The following is required for building new spack environments with any supported compiler on this platform. The default module path needs to be removed, otherwise spack detects the system as Cray. .. code-block:: console module purge - module unuse /opt/cray/craype/default/modulefiles - module unuse /opt/cray/modulefiles - module use /contrib/spack-stack/modulefiles - module load cmake/3.27.2 - module load ecflow/5.8.4 - module load git-lfs/2.4.1 - -For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: -.. code-block:: console - - module use /contrib/spack-stack/spack-stack-1.7.0/envs/ue-intel-2021.3.0/install/modulefiles/Core - module load stack-intel/2021.3.0 - module load stack-intel-oneapi-mpi/2021.3.0 - module load stack-python/3.10.13 -.. _Preconfigured_Sites_Gaea: +.. _Preconfigured_Sites_Gaea_C5: ------------------------------ -NOAA RDHPCS Gaea +NOAA RDHPCS Gaea C5 ------------------------------ -The following is required for building new spack environments and for using spack to build and run software. Log into a head node, and don't use ``module purge`` on Gaea! +The following is required for building new spack environments with Intel on this platform.. Don't use ``module purge`` on Gaea! .. code-block:: console + # These modules should be loaded by default, if not load (swap) with: module load PrgEnv-intel/8.3.3 module load intel-classic/2023.1.0 module load cray-mpich/8.1.25 module load python/3.9.12 - module use /ncrc/proj/epic/spack-stack/modulefiles - module load ecflow/5.8.4 - -For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: - -.. code-block:: console - - module use /ncrc/proj/epic/spack-stack/spack-stack-1.7.0/envs/ue-intel/install/modulefiles/Core - module load stack-intel/2023.1.0 - module load stack-cray-mpich/8.1.25 - module load stack-python/3.10.13 - module -t available .. note:: - On Gaea, running ``module available`` without the option ``-t`` leads to an error: ``/usr/bin/lua5.3: /opt/cray/pe/lmod/lmod/libexec/Spider.lua:568: stack overflow`` + On Gaea, running ``module available`` without the option ``-t`` can lead to an error: ``/usr/bin/lua5.3: /opt/cray/pe/lmod/lmod/libexec/Spider.lua:568: stack overflow`` .. note:: On Gaea, a current limitation is that any executable that is linked against the MPI library (``cray-mpich``) must be run through ``srun`` on a compute node, even if it is run serially (one process). This is in particular a problem when using ``ctest`` for unit testing created by the ``ecbuild add_test`` macro. A workaround is to use the `cmake` cross-compiling emulator for this: @@ -472,80 +277,73 @@ For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules cmake -DCMAKE_CROSSCOMPILING_EMULATOR="/usr/bin/srun;-n;1" -DMPIEXEC_EXECUTABLE="/usr/bin/srun" -DMPIEXEC_NUMPROC_FLAG="-n" PATH_TO_SOURCE -.. _Preconfigured_Sites_Hera: + +.. _Preconfigured_Sites_Gaea_C6: ------------------------------ -NOAA RDHPCS Hera +NOAA RDHPCS Gaea C6 ------------------------------ -The following is required for building new spack environments and for using spack to build and run software. +The following is required for building new spack environments with Intel on this platform.. Don't use ``module purge`` on Gaea! .. code-block:: console - module purge - module use /scratch1/NCEPDEV/nems/role.epic/modulefiles - module load miniconda3/4.12.0 - module load ecflow/5.8.4 - -For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: + # These modules should be loaded by default, if not load (swap) with: + module load PrgEnv-intel/8.3.3 + module load intel-classic/2023.2.0 + module load cray-mpich/8.1.25 + module load python/3.9.12 -.. code-block:: console - module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.7.0/envs/ue-intel/install/modulefiles/Core - module load stack-intel/2021.5.0 - module load stack-intel-oneapi-mpi/2021.5.1 - module load stack-python/3.10.13 +.. note:: + On Gaea, running ``module available`` without the option ``-t`` can lead to an error: ``/usr/bin/lua5.3: /opt/cray/pe/lmod/lmod/libexec/Spider.lua:568: stack overflow`` -For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: +.. note:: + On Gaea, a current limitation is that any executable that is linked against the MPI library (``cray-mpich``) must be run through ``srun`` on a compute node, even if it is run serially (one process). This is in particular a problem when using ``ctest`` for unit testing created by the ``ecbuild add_test`` macro. A workaround is to use the `cmake` cross-compiling emulator for this: .. code-block:: console - module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.7.0/envs/ue-gcc/install/modulefiles/Core - module load stack-gcc/9.2.0 - module load stack-openmpi/4.1.5 - module load stack-python/3.10.13 + cmake -DCMAKE_CROSSCOMPILING_EMULATOR="/usr/bin/srun;-n;1" -DMPIEXEC_EXECUTABLE="/usr/bin/srun" -DMPIEXEC_NUMPROC_FLAG="-n" PATH_TO_SOURCE -Note that on Hera, a dedicated node exists for ``ecflow`` server jobs (``hecflow01``). Users starting ``ecflow_server`` on the regular login nodes will see their servers being killed every few minutes, and may be barred from accessing the system. -.. _Preconfigured_Sites_Jet: +.. _Preconfigured_Sites_Hera: ------------------------------ -NOAA RDHPCS Jet +NOAA RDHPCS Hera ------------------------------ -The following is required for building new spack environments and for using spack to build and run software. +The following is required for building new spack environments with any supported compiler on this platform. .. code-block:: console module purge - module use /lfs4/HFIP/hfv3gfs/spack-stack/modulefiles - module load miniconda/3.9.12 - module load ecflow/5.5.3 - module use /lfs4/HFIP/hfv3gfs/role.epic/modulefiles -For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: +.. note:: + On Hera, a dedicated node exists for ``ecflow`` server jobs (``hecflow01``). Users starting ``ecflow_server`` on the regular login nodes will see their servers being killed every few minutes, and may be barred from accessing the system. -.. code-block:: console - module use /mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.7.0/envs/ue-intel/install/modulefiles/Core - module load stack-intel/2021.5.0 - module load stack-intel-oneapi-mpi/2021.5.1 - module load stack-python/3.10.8 +.. _Preconfigured_Sites_Jet: + +------------------------------ +NOAA RDHPCS Jet +------------------------------ -For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: +The following is required for building new spack environments with any supported compiler on this platform. .. code-block:: console - module use /mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.7.0/envs/ue-gcc/install/modulefiles/Core - module load stack-gcc/9.2.0 - module load stack-openmpi/3.1.4 - module load stack-python/3.10.8 + module purge + + +.. _Preconfigured_Sites_S4: ------------------------------ UW (Univ. of Wisconsin) S4 ------------------------------ -The following is required for building new spack environments and for using spack to build and run software. +The following is required for building new spack environments with any supported compiler on this platform. + +**NEEDS UPDATING** .. code-block:: console @@ -554,44 +352,28 @@ The following is required for building new spack environments and for using spac module load miniconda/3.9.12 module load ecflow/5.8.4 -For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: - -.. code-block:: console - - module use /data/prod/jedi/spack-stack/spack-stack-1.7.0/envs/ue-intel-2021.5.0/install/modulefiles/Core - module load stack-intel/2021.5.0 - module load stack-intel-oneapi-mpi/2021.5.0 - module load stack-python/3.10.13 - module unuse /opt/apps/modulefiles/Compiler/intel/non-default/22 - module unuse /opt/apps/modulefiles/Compiler/intel/22 - -Note the two `module unuse` commands, that need to be run after the stack metamodules are loaded. Loading the Intel compiler meta module loads the Intel compiler module provided by the sysadmins, which adds those two directories to the module path. These contain duplicate libraries that are not compatible with our stack, such as ``hdf4``. -.. note:: - There is currently no support for GNU on S4, because recent updates to ``hdf5`` require a newer version of ``mpich`` (or other MPI library) than available on the system. Also, for spack-stack-1.7.0, S4 is the only system that uses ``zlib`` instead of ``zlib-ng`` due to the issues described in https://github.com/JCSDA/spack-stack/issues/1055. +.. _Preconfigured_Sites_AWS_Parallelcluster: ------------------------------------------------ Amazon Web Services Parallelcluster Ubuntu 20.04 ------------------------------------------------ +**NEEDS UPDATING** + The JCSDA-managed AWS Parallel Cluster is currently unavailable. ------------------------------ -Amazon Web Services Red Hat 8 ------------------------------ -Use a c6i.4xlarge instance or larger if running out of memory with AMI "skylab-8.0.0-redhat8" (see JEDI documentation at https://jointcenterforsatellitedataassimilation-jedi-docs.readthedocs-hosted.com/en/latest for more information). +.. _Preconfigured_Sites_AWS_SingleNode_RH8: -For ``spack-stack-1.7.0``, run: +----------------------------------------- +Amazon Web Services Single Node Red Hat 8 +----------------------------------------- -.. code-block:: console +**NEEDS UPDATING** + +Use a c6i.4xlarge instance or larger if running out of memory with AMI "skylab-8.0.0-redhat8" (see JEDI documentation at https://jointcenterforsatellitedataassimilation-jedi-docs.readthedocs-hosted.com/en/latest for more information). - ulimit -s unlimited - scl_source enable gcc-toolset-11 - module use /home/ec2-user/spack-stack/spack-stack-1.7.0/envs/unified-env-gcc-11.2.1/install/modulefiles/Core - module load stack-gcc/11.2.1 - module load stack-openmpi/5.0.1 - module load stack-python/3.10.13 .. _Preconfigured_Sites_Tier2: @@ -601,9 +383,15 @@ Pre-configured sites (tier 2) Tier 2 preconfigured site are not officially supported by spack-stack. As such, instructions for these systems may be provided here, in form of a `README.md` in the site directory, or may not be available. Also, these site configs are not updated on the same regular basis as those of the tier 1 systems and therefore may be out of date and/or not working. -The following sites have site configurations in directory `configs/sites/`: -- TACC Frontera (`configs/sites/frontera/`) -- AWS Single Node with Nvidia (NVHPC) compilers (`configs/sites/aws-nvidia/`) + +.. _Preconfigured_Sites_Blackpearl: + +------------------------------ +Blackpearl +------------------------------ + +Blackpearl is an Oracle Linux 9 installation running under Windows Subsystem for Linux (WSL2) on Windows 11. This is the development system of one of the spack-stack developers and maybe useful as an example configuration for users with a similar setup. + .. _Preconfigured_Sites_Casper: @@ -611,7 +399,9 @@ The following sites have site configurations in directory `configs/sites/`: NCAR-Wyoming Casper ------------------------------ -The following is required for building new spack environments and for using spack to build and run software. +The following is required for building new spack environments with any supported compiler on this platform. + +**NEEDS UPDATING** .. code-block:: console @@ -622,6 +412,32 @@ The following is required for building new spack environments and for using spac module use /glade/work/epicufsrt/contrib/spack-stack/casper/modulefiles module load ecflow/5.8.4 + +.. _Preconfigured_Sites_EMC_RHEL: + +------------------------------ +EMC RedHat Enterprise Linux 8 +------------------------------ + +**NEEDS UPDATING** + + +.. _Preconfigured_Sites_Frontera: + +------------------------------ +??? Frontera +------------------------------ + +**NEEDS UPDATING** + + +------------------------------ +Linux/macOS default configs +------------------------------ + +The Linux and macOS configurations are **not** meant to be used as is, as user setups and package versions vary considerably. Instructions for adding this information can be found in :numref:`Section %s `. + + .. _Configurable_Sites_CreateEnv: ======================== @@ -687,6 +503,7 @@ The following instructions install a new spack environment on a pre-configured s spack concretize 2>&1 | tee log.concretize spack install [--verbose] [--fail-fast] 2>&1 | tee log.install + .. _Preconfigured_Sites_ExtendingEnvironments: ====================== diff --git a/doc/source/conf.py b/doc/source/conf.py index 82920fd02..8f33a6da8 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -21,7 +21,7 @@ project = 'spack-stack' copyright = '2024 ' -author = 'Dominikus Heinzeller, Alexander Richert, Cameron Book' +author = 'Dominikus Heinzeller, Alexander Richert, Steve Herbener, Ratko Vasic' # The short X.Y version version = 'dev' diff --git a/spack b/spack index 33eea0479..6f9a6ba4e 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 33eea04798da9317b9e48862291f3dfa9788d0e7 +Subproject commit 6f9a6ba4e5850494ffb7e7d592fe982036c32fb9 diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py index b63dd93f1..83757629a 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py @@ -100,7 +100,7 @@ def __init__(self, **kwargs): if not self.name: # site = self.site if self.site else 'default' - self.name = "{}.{}.{}".format(self.template, self.site, self.compiler) + self.name = "{}.{}.{}".format(self.template, self.site, self.compiler.replace("@", "-")) def env_dir(self): """env_dir is /""" @@ -170,7 +170,7 @@ def _copy_common_includes(self): self._copy_or_merge_includes("modules", modules_yaml_path, modules_yaml_modulesys_path, destination) # Merge or copy common package config(s) packages_yaml_path = os.path.join(common_path, "packages.yaml") - packages_compiler_yaml_path = os.path.join(common_path, f"packages_{self.compiler}.yaml") + packages_compiler_yaml_path = os.path.join(common_path, f"packages_{self.compiler.split('@')[0]}.yaml") destination = os.path.join(env_common_dir, "packages.yaml") self._copy_or_merge_includes("packages", packages_yaml_path, packages_compiler_yaml_path, destination) @@ -203,7 +203,7 @@ def _copy_site_includes(self): self._copy_or_merge_includes("modules", modules_yaml_path, modules_yaml_modulesys_path, destination) # Merge or copy site package config(s) packages_yaml_path = os.path.join(env_path, "packages.yaml") - packages_compiler_yaml_path = os.path.join(env_path, f"packages_{self.compiler}.yaml") + packages_compiler_yaml_path = os.path.join(env_path, f"packages_{self.compiler.split('@')[0]}.yaml") destination = os.path.join(env_site_dir, "packages.yaml") self._copy_or_merge_includes("packages", packages_yaml_path, packages_compiler_yaml_path, destination) @@ -263,11 +263,7 @@ def write(self): target_compiler = f"%{self.compiler}" for i in range(len(definitions)): if "compilers" in definitions[i]: - j = len(definitions[i]["compilers"])-1 - while j>=0: - if not definitions[i]["compilers"][j] == target_compiler: - definitions[i]["compilers"].pop(j) - j -= 1 + definitions[i] = {"compilers": [target_compiler]} spack.config.set("definitions", definitions, scope=env_scope) if self.install_prefix: diff --git a/spack-ext/repos/spack-stack/packages/gmao-swell-env/package.py b/spack-ext/repos/spack-stack/packages/gmao-swell-env/package.py index 092fe668e..ec30399c4 100644 --- a/spack-ext/repos/spack-stack/packages/gmao-swell-env/package.py +++ b/spack-ext/repos/spack-stack/packages/gmao-swell-env/package.py @@ -47,6 +47,8 @@ class GmaoSwellEnv(BundlePackage): depends_on("py-setuptools", type="run") depends_on("py-pycodestyle", type="run") depends_on("py-pyyaml", type="run") + depends_on("py-ruamel-yaml", type="run") + depends_on("py-ruamel-yaml-clib", type="run") # Note that the +delayed option is for compatibility # with the py-xnrl package (this restricts py-dask # to certain versions, since the newest versions diff --git a/spack-ext/repos/spack-stack/packages/neptune-env/package.py b/spack-ext/repos/spack-stack/packages/neptune-env/package.py index 917ea8865..f79968eac 100644 --- a/spack-ext/repos/spack-stack/packages/neptune-env/package.py +++ b/spack-ext/repos/spack-stack/packages/neptune-env/package.py @@ -34,7 +34,11 @@ class NeptuneEnv(BundlePackage): depends_on("p4est", type="run") depends_on("w3emc", type="run") depends_on("w3nco", type="run") - depends_on("ip@5:", type="run") + depends_on("sp", type="run", when="%aocc") + depends_on("ip@5:", type="run", when="%apple-clang") + depends_on("ip@5:", type="run", when="%gcc") + depends_on("ip@5:", type="run", when="%intel") + depends_on("ip@5:", type="run", when="%oneapi") depends_on("esmf", type="run") depends_on("nco", type="run") depends_on("mct", type="run")