From 4a95ab5c9c328d420f8e5e2bc815c2e9258cef91 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 22 Aug 2024 13:40:04 -0600 Subject: [PATCH] Update site configs for Atlantis, Narwhal, Nautilus (#1266) This PR updates the site configs for Atlantis, Narwhal, Nautilus, and also the instructions for Narwhal on how to build and use new environments. Notable changes: Atlantis: Remove external openssl, issues with py-cryptography Narwhal: Update cray-libsci for Intel, and configure GNU without Cray wrappers Nautilus: Use intel-oneapi-mkl as linalg provider with GNU because of problems with using openblas in downstream applications (ELF load command address/offset not properly aligned when loading libopenblas.so; https://github.com/OpenMathLib/OpenBLAS/wiki/Faq#ELFoffset) --- configs/sites/tier1/atlantis/packages.yaml | 9 +++++---- configs/sites/tier1/narwhal/compilers.yaml | 13 +++++++----- configs/sites/tier1/narwhal/packages_gcc.yaml | 2 +- .../sites/tier1/nautilus/packages_gcc.yaml | 20 +++++++++++++++++++ doc/source/PreConfiguredSites.rst | 12 +++++++++-- 5 files changed, 44 insertions(+), 12 deletions(-) diff --git a/configs/sites/tier1/atlantis/packages.yaml b/configs/sites/tier1/atlantis/packages.yaml index 0487117dd..0fb097d39 100644 --- a/configs/sites/tier1/atlantis/packages.yaml +++ b/configs/sites/tier1/atlantis/packages.yaml @@ -122,10 +122,11 @@ packages: externals: - spec: flex@2.6.1+lex prefix: /usr - openssl: - externals: - - spec: openssl@1.1.1k - prefix: /usr + # Can no longer use, issues with py-cryptography + #openssl: + # externals: + # - spec: openssl@1.1.1k + # prefix: /usr gawk: externals: - spec: gawk@4.2.1 diff --git a/configs/sites/tier1/narwhal/compilers.yaml b/configs/sites/tier1/narwhal/compilers.yaml index 22ddd17b4..eb74a5993 100644 --- a/configs/sites/tier1/narwhal/compilers.yaml +++ b/configs/sites/tier1/narwhal/compilers.yaml @@ -14,27 +14,30 @@ compilers:: modules: - PrgEnv-intel/8.3.3 - intel-classic/2023.2.0 + - cray-libsci/23.05.1.4 - libfabric/1.12.1.2.2.1 environment: prepend_path: PATH: '/opt/cray/pe/gcc/10.3.0/snos/bin' CPATH: '/opt/cray/pe/gcc/10.3.0/snos/include' - LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/22.11.1.2/INTEL/19.0/x86_64/lib:/opt/cray/pe/gcc/10.3.0/snos/lib:/opt/cray/pe/gcc/10.3.0/snos/lib64' + LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/23.05.1.4/INTEL/2022.2/x86_64/lib:/opt/cray/pe/gcc/10.3.0/snos/lib:/opt/cray/pe/gcc/10.3.0/snos/lib64' set: CRAYPE_LINK_TYPE: 'dynamic' extra_rpaths: [] - compiler: spec: gcc@10.3.0 paths: - cc: cc - cxx: CC - f77: ftn - fc: ftn + cc: /opt/cray/pe/gcc/10.3.0/bin/gcc + cxx: /opt/cray/pe/gcc/10.3.0/bin/g++ + f77: /opt/cray/pe/gcc/10.3.0/bin/gfortran + fc: /opt/cray/pe/gcc/10.3.0/bin/gfortran flags: {} operating_system: sles15 modules: - PrgEnv-gnu/8.3.3 - gcc/10.3.0 + - cray-libsci/22.11.1.2 + - libfabric/1.12.1.2.2.1 environment: prepend_path: LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/22.11.1.2/GNU/9.1/x86_64/lib' diff --git a/configs/sites/tier1/narwhal/packages_gcc.yaml b/configs/sites/tier1/narwhal/packages_gcc.yaml index 3261960c6..0cc712415 100644 --- a/configs/sites/tier1/narwhal/packages_gcc.yaml +++ b/configs/sites/tier1/narwhal/packages_gcc.yaml @@ -7,7 +7,7 @@ packages: buildable: False cray-mpich: externals: - - spec: cray-mpich@8.1.21%gcc@10.3.0 ~wrappers + - spec: cray-mpich@8.1.21%gcc@10.3.0 +wrappers modules: - cray-mpich-ucx/8.1.21 - craype-network-ucx diff --git a/configs/sites/tier1/nautilus/packages_gcc.yaml b/configs/sites/tier1/nautilus/packages_gcc.yaml index 08bb02aa5..841ff05b1 100644 --- a/configs/sites/tier1/nautilus/packages_gcc.yaml +++ b/configs/sites/tier1/nautilus/packages_gcc.yaml @@ -1,8 +1,17 @@ packages: +# On Nautilus, use intel-oneapi-mkl as provider +# for blas, lapack, fftw-api with GNU because of +# problems using openblas in downstream applications +# (e.g. py-pandas): +# ELF load command address/offset not properly aligned" when loading libopenblas.so +# https://github.com/OpenMathLib/OpenBLAS/wiki/Faq#ELFoffset all: compiler:: [gcc@12.2.1] providers: mpi:: [openmpi@5.0.1] + blas:: [intel-oneapi-mkl] + fftw-api:: [intel-oneapi-mkl] + lapack:: [intel-oneapi-mkl] mpi: buildable: False openmpi: @@ -12,3 +21,14 @@ packages: modules: - penguin/openmpi/5.0.1/gcc-8.5.0 - slurm + openblas: + buildable: False + ectrans: + require:: + - '@1.2.0 +mkl ~fftw' + gsibec: + require:: + - '@1.2.1 +mkl' + py-numpy: + require:: + - '@:1.25 ^intel-oneapi-mkl' diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 4084281e0..b80cd7164 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -253,8 +253,12 @@ With Intel, the following is required for building new spack environments and fo module unload intel module load intel-classic/2023.2.0 module unload cray-mpich - module load cray-mpich/8.1.21 + module unload craype-network-ofi + module load craype-network-ucx + module load cray-mpich-ucx/8.1.21 module load libfabric/1.12.1.2.2.1 + module unload cray-libsci + module load cray-libsci/23.05.1.4 THIS SECTION IS OUT OF DATE, REFER TO 1.7.0 RELEASE DOCUMENTATION - - For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: @@ -279,8 +283,12 @@ With GNU, the following is required for building new spack environments and for module unload gcc module load gcc/10.3.0 module unload cray-mpich - module load cray-mpich/8.1.21 + module unload craype-network-ofi + module load craype-network-ucx + module load cray-mpich-ucx/8.1.21 module load libfabric/1.12.1.2.2.1 + module unload cray-libsci + module load cray-libsci/22.11.1.2 THIS SECTION IS OUT OF DATE, REFER TO 1.7.0 RELEASE DOCUMENTATION - For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: