From 399df7843b3fdba1f766a2eb9142e4b77a2c30f9 Mon Sep 17 00:00:00 2001 From: Jessica Meixner Date: Tue, 2 Apr 2024 12:37:55 -0400 Subject: [PATCH] For NCEP regtests, add option for gnu compiler and new machine Hercules (#1145) --- model/src/wminitmd.F90 | 6 ++ regtests/bin/matrix_cmake_ncep | 108 +++++++++++++++++++++++++-------- 2 files changed, 88 insertions(+), 26 deletions(-) diff --git a/model/src/wminitmd.F90 b/model/src/wminitmd.F90 index 956490b5e..768b6d91e 100644 --- a/model/src/wminitmd.F90 +++ b/model/src/wminitmd.F90 @@ -1273,6 +1273,12 @@ SUBROUTINE WMINIT ( IDSI, IDSO, IDSS, IDST, IDSE, IFNAME, & IF (WORDS(6) .EQ. 'T') THEN CALL NEXTLN ( COMSTR , MDSI , MDSE2 ) READ (MDSI,*,END=2001,ERR=2002)(ODAT(I,1),I=5*(8-1)+1,5*8) + ELSE + ODAT(5*(8-1)+1,1)=0 + ODAT(5*(8-1)+2,1)=0 + ODAT(5*(8-1)+3,1)=0 + ODAT(5*(8-1)+4,1)=0 + ODAT(5*8,1)=0 END IF ELSE READ (MDSI,*,END=2001,ERR=2002)(ODAT(I,1),I=5*(J-1)+1,5*J) diff --git a/regtests/bin/matrix_cmake_ncep b/regtests/bin/matrix_cmake_ncep index 7d0d26bec..2eafd8986 100755 --- a/regtests/bin/matrix_cmake_ncep +++ b/regtests/bin/matrix_cmake_ncep @@ -22,28 +22,37 @@ usage () { cat 2>&1 << EOF - Usage: $myname model_dir + Usage: $myname model_dir compiler Required: model_dir : path to model dir of WW3 source + Optional: + compiler : intel (default) or gnu EOF } - # Get required arguments if [ ! $# = 0 ] then main_dir="$1" ; shift + if [ ! $# = 0 ] + then + compiler="$1"; shift + else + compiler='intel' + fi else usage exit 1 fi + + # Convert main_dir to absolute path main_dir="`cd $main_dir 1>/dev/null 2>&1 && pwd`" # Module Versions from spack-stack that are common for all platforms modnetcdfc='netcdf-c/4.9.2' - modnetcdff='netcdf-fortran/4.6.0' + modnetcdff='netcdf-fortran/4.6.1' modjasper='jasper/2.0.32' modzlib='zlib/1.2.13' modpng='libpng/1.6.37' @@ -51,30 +60,71 @@ EOF modbacio='bacio/2.4.1' modg2='g2/3.4.5' modw3emc='w3emc/2.10.0' - modesmf='esmf/8.4.2' + modesmf='esmf/8.5.0' modscotch='scotch/7.0.4' # Set batchq queue, choose modules and other custom variables to fit system and # to define headers etc (default to original version if empty) ishera=`hostname | grep hfe` isorion=`hostname | grep Orion` + ishercules=`hostname | grep hercules` if [ $ishera ] then - # If no other h, assuming Hera batchq='slurm' - spackstackpath='/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core' - modcomp='stack-intel/2021.5.0' - modmpi='stack-intel-oneapi-mpi/2021.5.1' - metispath='/scratch1/NCEPDEV/climate/Matthew.Masarik/waves/opt/spack-stack/1.5.0/parmetis-4.0.3/install' - modcmake='cmake/3.23.1' + if [ $compiler = "intel" ] + then + spackstackpath='/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core' + modcomp='stack-intel/2021.5.0' + modmpi='stack-intel-oneapi-mpi/2021.5.1' + metispath='/scratch1/NCEPDEV/climate/Matthew.Masarik/waves/opt/hera/intel/spack-stack/1.6.0/parmetis-4.0.3/install' + modcmake='cmake/3.23.1' + elif [ $compiler = "gnu" ] + then + spackstackpath='/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core' + modcomp='stack-gcc/9.2.0' + modmpi='stack-openmpi/4.1.5' + metispath='/scratch1/NCEPDEV/climate/Matthew.Masarik/waves/opt/hera/gnu/spack-stack/1.6.0/parmetis-4.0.3/install' + modcmake='cmake/3.23.1' + else + echo "Compiler $compiler not supported on hera" + exit 1 + fi elif [ $isorion ] then + if [ $compiler = "intel" ] + then + batchq='slurm' + spackstackpath='/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core' + modcomp='stack-intel/2022.0.2' + modmpi='stack-intel-oneapi-mpi/2021.5.1' + metispath='/work/noaa/marine/Matthew.Masarik/waves/opt/orion/intel/spack-stack/1.6.0/parmetis-4.0.3/install' + modcmake='cmake/3.23.1' + else + echo "Compiler $compiler not supported on orion" + exit 1 + fi + elif [ $ishercules ] + then batchq='slurm' - spackstackpath='/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core' - modcomp='stack-intel/2022.0.2' - modmpi='stack-intel-oneapi-mpi/2021.5.1' - metispath='/work/noaa/marine/Matthew.Masarik/waves/opt/spack-stack/1.5.0/parmetis-4.0.3/install' - modcmake='cmake/3.23.1' + if [ $compiler = "intel" ] + then + spackstackpath='/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core' + modcomp='stack-intel/2021.9.0' + modmpi='stack-intel-oneapi-mpi/2021.9.0' + metispath='/work/noaa/marine/Matthew.Masarik/waves/opt/hercules/intel/spack-stack/1.6.0/parmetis-4.0.3/install' + modcmake='cmake/3.23.1' + elif [ $compiler = "gnu" ] + then + spackstackpath='/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core' + spackstackpath2='/work/noaa/epic/role-epic/spack-stack/hercules/modulefiles' + modcomp='stack-gcc/12.2.0' + modmpi='stack-mvapich2/2.3.7' + metispath='/work/noaa/marine/Matthew.Masarik/waves/opt/hercules/gnu/spack-stack/1.6.0/parmetis-4.0.3/install' + modcmake='cmake/3.23.1' + else + echo "Compiler $compiler not supported on hercules" + exit 1 + fi else batchq= fi @@ -105,10 +155,19 @@ EOF echo '#SBATCH --exclusive' >> matrix.head echo ' ' >> matrix.head echo 'ulimit -s unlimited' >> matrix.head - echo 'ulimit -c 0' >> matrix.head - echo 'export KMP_STACKSIZE=2G' >> matrix.head - echo 'export FI_OFI_RXM_BUFFER_SIZE=128000' >> matrix.head - echo 'export FI_OFI_RXM_RX_SIZE=64000' >> matrix.head + elif [ $batchq = "slurm" ] && [ $ishercules ] + then + echo "#SBATCH -n ${np}" >> matrix.head + echo "##SBATCH --cpus-per-task=${nth}" >> matrix.head + echo '#SBATCH -q batch' >> matrix.head + echo '#SBATCH -t 08:00:00' >> matrix.head + echo '#SBATCH -A marine-cpu' >> matrix.head + echo '#SBATCH -J ww3_regtest' >> matrix.head + echo '#SBATCH -o matrix.out' >> matrix.head + echo '#SBATCH -p hercules' >> matrix.head + echo '#SBATCH --exclusive' >> matrix.head + echo ' ' >> matrix.head + echo 'ulimit -s unlimited' >> matrix.head elif [ $batchq = "slurm" ] then echo "#SBATCH -n ${np}" >> matrix.head @@ -133,13 +192,10 @@ EOF # Netcdf, Parmetis and SCOTCH modules & variables echo " module purge" >> matrix.head - if [ ! -z $basemodcomp ]; then - echo " module load $basemodcomp" >> matrix.head - fi - if [ ! -z $basemodmpi ]; then - echo " module load $basemodmpi" >> matrix.head - fi - echo " module use $spackstackpath" >> matrix.head + echo " module use $spackstackpath" >> matrix.head + if [ ! -z $spackstackpath2 ]; then + echo " module use $spackstackpath2" >> matrix.head + fi echo " module load $modcomp" >> matrix.head echo " module load $modmpi" >> matrix.head echo " module load $modcmake" >> matrix.head