diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml index 514b01d0d..3087b7f1f 100644 --- a/.github/workflows/macos-ci-aarch64.yaml +++ b/.github/workflows/macos-ci-aarch64.yaml @@ -20,6 +20,7 @@ jobs: run: | pwd ls -lart + find ./* -type d -exec chmod u+xw {} \; rm -fr * - name: checkout @@ -168,7 +169,7 @@ jobs: module use ${ENVDIR}/install/modulefiles/Core module load stack-apple-clang/14.0.3 module load stack-openmpi/5.0.3 - module load stack-python/3.10.13 + module load stack-python/3.11.7 module available module load jedi-ufs-env diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml index 8a60dc15c..13414615c 100644 --- a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml @@ -20,6 +20,7 @@ jobs: run: | pwd ls -lart + find ./* -type d -exec chmod u+xw {} \; rm -fr * - name: checkout @@ -199,7 +200,7 @@ jobs: module use ${ENVDIR}/install/modulefiles/Core module load stack-gcc/11.4.0 module load stack-openmpi/5.0.3 - module load stack-python/3.10.13 + module load stack-python/3.11.7 module available module load jedi-ufs-env diff --git a/.github/workflows/ubuntu-ci-x86_64-intel.yaml b/.github/workflows/ubuntu-ci-x86_64-intel.yaml index cb44dafaf..deacbae25 100644 --- a/.github/workflows/ubuntu-ci-x86_64-intel.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-intel.yaml @@ -20,6 +20,7 @@ jobs: run: | pwd ls -lart + find ./* -type d -exec chmod u+xw {} \; rm -fr * - name: checkout @@ -232,7 +233,7 @@ jobs: module use ${ENVDIR}/install/modulefiles/Core module load stack-intel/2021.10.0 module load stack-intel-oneapi-mpi/2021.10.0 - module load stack-python/3.10.13 + module load stack-python/3.11.7 module available module load jedi-ufs-env diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml index 44eeec3aa..1d20750ce 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml @@ -20,6 +20,7 @@ jobs: run: | pwd ls -lart + find ./* -type d -exec chmod u+xw {} \; rm -fr * - name: checkout @@ -239,7 +240,7 @@ jobs: module use ${ENVDIR}/install/modulefiles/Core module load stack-oneapi/2024.2.0 module load stack-intel-oneapi-mpi/2021.13 - module load stack-python/3.10.13 + module load stack-python/3.11.7 module available module load jedi-ufs-env diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index ddfbd6180..cd1d42635 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -19,7 +19,9 @@ packages: boost: require: '@1.84 ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden' bufr: - require: '@12.0.1 +python' + require: '@12.1.0 +python' + bufr-query: + require: '@0.0.2 +python' cairo: require: '+pic' cdo: @@ -37,7 +39,7 @@ packages: ecflow: require: '@5.11.4 +ui' eckit: - require: '@1.24.5 linalg=eigen,lapack compression=lz4,bzip2' + require: '@1.27.0 linalg=eigen,lapack compression=lz4,bzip2' ecmwf-atlas: require: '@0.38.1 +fckit +trans +tesselation +fftw' ectrans: @@ -183,7 +185,7 @@ packages: proj: require: '~tiff' python: - require: '@3.10.13' + require: '@3.11.7' py-attrs: # https://github.com/JCSDA/spack-stack/issues/740 require: '@21.4.0' diff --git a/configs/containers/specs/jedi-ci.yaml b/configs/containers/specs/jedi-ci.yaml index d72a59e0e..7563ddcdc 100644 --- a/configs/containers/specs/jedi-ci.yaml +++ b/configs/containers/specs/jedi-ci.yaml @@ -1,7 +1,7 @@ ### spack-stack-1.6.0 / skylab-7.x.y containers for fv3-jedi and mpas-jedi (but not for ufs-jedi) specs: [base-env@1.0.0, jedi-base-env@1.0.0, ewok-env@1.0.0, jedi-fv3-env@1.0.0, - jedi-mpas-env@1.0.0, bacio@2.4.1, bison@3.8.2, bufr@12.0.1, ecbuild@3.7.2, eccodes@2.33.0, ecflow@5, - eckit@1.24.5, ecmwf-atlas@0.38.1 +fckit +trans +tesselation +fftw, fiat@1.2.0, ectrans@1.2.0 +fftw, + jedi-mpas-env@1.0.0, bacio@2.4.1, bison@3.8.2, bufr@12.1.0, bufr-query@0.0.2, ecbuild@3.7.2, eccodes@2.33.0, ecflow@5, + eckit@1.27.0, ecmwf-atlas@0.38.1 +fckit +trans +tesselation +fftw, fiat@1.2.0, ectrans@1.2.0 +fftw, eigen@3.4.0, fckit@0.11.0, fms@2024.02, g2@3.5.1, g2tmpl@1.13.0, gsibec@1.2.1, hdf@4.2.15, hdf5@1.14.3, ip@5.0.0, jasper@2.0.32, jedi-cmake@1.4.0, libpng@1.6.37, nccmp@1.9.0.1, netcdf-c@4.9.2, netcdf-cxx4@4.3.1, netcdf-fortran@4.6.1, diff --git a/configs/sites/tier1/atlantis/packages.yaml b/configs/sites/tier1/atlantis/packages.yaml index 0487117dd..0fb097d39 100644 --- a/configs/sites/tier1/atlantis/packages.yaml +++ b/configs/sites/tier1/atlantis/packages.yaml @@ -122,10 +122,11 @@ packages: externals: - spec: flex@2.6.1+lex prefix: /usr - openssl: - externals: - - spec: openssl@1.1.1k - prefix: /usr + # Can no longer use, issues with py-cryptography + #openssl: + # externals: + # - spec: openssl@1.1.1k + # prefix: /usr gawk: externals: - spec: gawk@4.2.1 diff --git a/configs/sites/tier1/narwhal/compilers.yaml b/configs/sites/tier1/narwhal/compilers.yaml index 22ddd17b4..eb74a5993 100644 --- a/configs/sites/tier1/narwhal/compilers.yaml +++ b/configs/sites/tier1/narwhal/compilers.yaml @@ -14,27 +14,30 @@ compilers:: modules: - PrgEnv-intel/8.3.3 - intel-classic/2023.2.0 + - cray-libsci/23.05.1.4 - libfabric/1.12.1.2.2.1 environment: prepend_path: PATH: '/opt/cray/pe/gcc/10.3.0/snos/bin' CPATH: '/opt/cray/pe/gcc/10.3.0/snos/include' - LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/22.11.1.2/INTEL/19.0/x86_64/lib:/opt/cray/pe/gcc/10.3.0/snos/lib:/opt/cray/pe/gcc/10.3.0/snos/lib64' + LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/23.05.1.4/INTEL/2022.2/x86_64/lib:/opt/cray/pe/gcc/10.3.0/snos/lib:/opt/cray/pe/gcc/10.3.0/snos/lib64' set: CRAYPE_LINK_TYPE: 'dynamic' extra_rpaths: [] - compiler: spec: gcc@10.3.0 paths: - cc: cc - cxx: CC - f77: ftn - fc: ftn + cc: /opt/cray/pe/gcc/10.3.0/bin/gcc + cxx: /opt/cray/pe/gcc/10.3.0/bin/g++ + f77: /opt/cray/pe/gcc/10.3.0/bin/gfortran + fc: /opt/cray/pe/gcc/10.3.0/bin/gfortran flags: {} operating_system: sles15 modules: - PrgEnv-gnu/8.3.3 - gcc/10.3.0 + - cray-libsci/22.11.1.2 + - libfabric/1.12.1.2.2.1 environment: prepend_path: LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/22.11.1.2/GNU/9.1/x86_64/lib' diff --git a/configs/sites/tier1/narwhal/packages_gcc.yaml b/configs/sites/tier1/narwhal/packages_gcc.yaml index 3261960c6..0cc712415 100644 --- a/configs/sites/tier1/narwhal/packages_gcc.yaml +++ b/configs/sites/tier1/narwhal/packages_gcc.yaml @@ -7,7 +7,7 @@ packages: buildable: False cray-mpich: externals: - - spec: cray-mpich@8.1.21%gcc@10.3.0 ~wrappers + - spec: cray-mpich@8.1.21%gcc@10.3.0 +wrappers modules: - cray-mpich-ucx/8.1.21 - craype-network-ucx diff --git a/configs/sites/tier1/nautilus/packages_gcc.yaml b/configs/sites/tier1/nautilus/packages_gcc.yaml index 08bb02aa5..841ff05b1 100644 --- a/configs/sites/tier1/nautilus/packages_gcc.yaml +++ b/configs/sites/tier1/nautilus/packages_gcc.yaml @@ -1,8 +1,17 @@ packages: +# On Nautilus, use intel-oneapi-mkl as provider +# for blas, lapack, fftw-api with GNU because of +# problems using openblas in downstream applications +# (e.g. py-pandas): +# ELF load command address/offset not properly aligned" when loading libopenblas.so +# https://github.com/OpenMathLib/OpenBLAS/wiki/Faq#ELFoffset all: compiler:: [gcc@12.2.1] providers: mpi:: [openmpi@5.0.1] + blas:: [intel-oneapi-mkl] + fftw-api:: [intel-oneapi-mkl] + lapack:: [intel-oneapi-mkl] mpi: buildable: False openmpi: @@ -12,3 +21,14 @@ packages: modules: - penguin/openmpi/5.0.1/gcc-8.5.0 - slurm + openblas: + buildable: False + ectrans: + require:: + - '@1.2.0 +mkl ~fftw' + gsibec: + require:: + - '@1.2.1 +mkl' + py-numpy: + require:: + - '@:1.25 ^intel-oneapi-mkl' diff --git a/configs/templates/gsi-addon-dev/spack.yaml b/configs/templates/gsi-addon-dev/spack.yaml index 0a8aedf6d..cecc52bae 100644 --- a/configs/templates/gsi-addon-dev/spack.yaml +++ b/configs/templates/gsi-addon-dev/spack.yaml @@ -15,7 +15,7 @@ spack: # Note: Set 'compilers' manually; must match upstream list - compilers: [] - packages: - - global-workflow-env ^bufr@11.7.0 ^metplus@3.1.1 ^met@9.1.3 + - global-workflow-env ^metplus@3.1.1 ^met@9.1.3 - ufs-weather-model-env - gsi-env diff --git a/configs/templates/neptune-dev/spack.yaml b/configs/templates/neptune-dev/spack.yaml index be3b8d816..5f3332f79 100644 --- a/configs/templates/neptune-dev/spack.yaml +++ b/configs/templates/neptune-dev/spack.yaml @@ -8,7 +8,7 @@ spack: definitions: - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel', '%oneapi'] - packages: - - neptune-env +python ^esmf@8.7.0b11 snapshot=b11 + - neptune-env +espc +python +xnrl ^esmf@8.7.0b11 snapshot=b11 specs: - matrix: diff --git a/configs/templates/skylab-dev/spack.yaml b/configs/templates/skylab-dev/spack.yaml index 75523e091..1ba272a63 100644 --- a/configs/templates/skylab-dev/spack.yaml +++ b/configs/templates/skylab-dev/spack.yaml @@ -17,6 +17,7 @@ spack: - jedi-neptune-env ^esmf@=8.7.0b11 snapshot=b11 - jedi-ufs-env ^esmf@=8.6.1 - jedi-um-env + - neptune-env ~espc +python ~xnrl ^esmf@=8.7.0b11 snapshot=b11 - soca-env # Various crtm tags (list all to avoid duplicate packages) diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index 46a6f3037..a05ed4a75 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -21,7 +21,7 @@ spack: - jedi-tools-env - jedi-ufs-env ^esmf@=8.6.1 - jedi-um-env - - neptune-env ~python ^esmf@=8.7.0b11 snapshot=b11 + - neptune-env ~espc +python ~xnrl ^esmf@=8.7.0b11 snapshot=b11 - soca-env - ufs-srw-app-env ^esmf@=8.6.1 - ufs-weather-model-env ^esmf@=8.6.1 diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 4084281e0..b80cd7164 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -253,8 +253,12 @@ With Intel, the following is required for building new spack environments and fo module unload intel module load intel-classic/2023.2.0 module unload cray-mpich - module load cray-mpich/8.1.21 + module unload craype-network-ofi + module load craype-network-ucx + module load cray-mpich-ucx/8.1.21 module load libfabric/1.12.1.2.2.1 + module unload cray-libsci + module load cray-libsci/23.05.1.4 THIS SECTION IS OUT OF DATE, REFER TO 1.7.0 RELEASE DOCUMENTATION - - For ``spack-stack-1.7.0`` with Intel, proceed with loading the following modules: @@ -279,8 +283,12 @@ With GNU, the following is required for building new spack environments and for module unload gcc module load gcc/10.3.0 module unload cray-mpich - module load cray-mpich/8.1.21 + module unload craype-network-ofi + module load craype-network-ucx + module load cray-mpich-ucx/8.1.21 module load libfabric/1.12.1.2.2.1 + module unload cray-libsci + module load cray-libsci/22.11.1.2 THIS SECTION IS OUT OF DATE, REFER TO 1.7.0 RELEASE DOCUMENTATION - For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: diff --git a/spack b/spack index cef497611..d8de455f3 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit cef497611df30f3cbf42c6b1c2a9196265f69edd +Subproject commit d8de455f35c23ecbb2e53eacd661b7060145c122 diff --git a/spack-ext/repos/spack-stack/packages/jedi-base-env/package.py b/spack-ext/repos/spack-stack/packages/jedi-base-env/package.py index a1f8726e9..9f09724b7 100644 --- a/spack-ext/repos/spack-stack/packages/jedi-base-env/package.py +++ b/spack-ext/repos/spack-stack/packages/jedi-base-env/package.py @@ -20,12 +20,14 @@ class JediBaseEnv(BundlePackage): # Need to find a free fftw provider for fftw-api ... variant("fftw", default=True, description="Build fftw") variant("hdf4", default=True, description="Build hdf4 library and python hdf module") + variant("bufrquery", default=True, description="Build bufr-query library") depends_on("base-env", type="run") depends_on("bison", type="run") depends_on("blas", type="run") depends_on("boost", type="run") depends_on("bufr", type="run") + depends_on("bufr-query", when="+bufrquery", type="run") # Force users to load manually # depends_on("crtm@v2.4.1-jedi", type="run") depends_on("ecbuild", type="run") diff --git a/spack-ext/repos/spack-stack/packages/neptune-env/package.py b/spack-ext/repos/spack-stack/packages/neptune-env/package.py index 1d902454e..917ea8865 100644 --- a/spack-ext/repos/spack-stack/packages/neptune-env/package.py +++ b/spack-ext/repos/spack-stack/packages/neptune-env/package.py @@ -19,7 +19,9 @@ class NeptuneEnv(BundlePackage): version("1.4.0") - variant("python", default=True, description="Build Python libraries") + variant("python", default=True, description="Build Python dependencies") + variant("espc", default=True, description="Build ESPC dependencies") + variant("xnrl", default=True, description="Build XNRL and its extra Python dependencies") depends_on("base-env", type="run") @@ -37,9 +39,11 @@ class NeptuneEnv(BundlePackage): depends_on("nco", type="run") depends_on("mct", type="run") - # Required by ESPC - depends_on("fftw", type="build") - depends_on("netlib-lapack", type="build") + conflicts("+xnrl", when="~python", msg="Variant xnrl requires variant python") + + with when("+espc"): + depends_on("fftw", type="build") + depends_on("netlib-lapack", type="build") with when("+python"): depends_on("py-f90nml", type="run") @@ -53,8 +57,10 @@ class NeptuneEnv(BundlePackage): depends_on("py-pyyaml", type="run") depends_on("py-scipy", type="run") depends_on("py-xarray", type="run") - depends_on("py-xnrl", type="run") depends_on("py-pytest", type="run") depends_on("py-fortranformat", type="run") + with when("+xnrl"): + depends_on("py-xnrl", type="run") + # There is no need for install() since there is no code.