Revert Gaea support.

DavidHuber-NOAA · Feb 12, 2024 · 4c923ab · 4c923ab
1 parent 6e085fb
commit 4c923ab
Show file tree

Hide file tree

Showing 5 changed files with 50 additions and 35 deletions.
diff --git a/modulefiles/gsi_gaea.intel.lua b/modulefiles/gsi_gaea.intel.lua
@@ -1,16 +1,17 @@
 help([[
 ]])
 
-unload("intel-classic")
+unload("intel")
 unload("cray-mpich")
 unload("cray-python")
-unload("darshan-runtime")
+unload("darshan")
 
-prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/spack-stack-1.4.1-c4/envs/unified-env/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/lustre/f2/pdata/esrl/gsd/spack-stack/modulefiles")
 
-local stack_python_ver=os.getenv("stack_python_ver") or "3.11.6"
-local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0"
-local stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.25"
+local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
+local stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2"
+local stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "7.7.20"
 local cmake_ver=os.getenv("cmake_ver") or "3.23.1"
 
 load(pathJoin("stack-intel", stack_intel_ver))
@@ -20,14 +21,22 @@ load(pathJoin("cmake", cmake_ver))
 
 load("gsi_common")
 
-local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1"
-load(pathJoin("prod_util", prod_util_ver))
+local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2"
+load(pathJoin("prod-util", prod_util_ver))
 
-pushenv("GSI_BINARY_SOURCE_DIR", "/gpfs/f5/epic/world-shared/GSI_data/fix/20230911")
+-- Needed at runtime:
+load("alps")
+
+local MKLROOT="/opt/intel/oneapi/mkl/2022.0.2/"
+prepend_path("LD_LIBRARY_PATH",pathJoin(MKLROOT,"lib/intel64"))
+pushenv("MKLROOT", MKLROOT)
+
+pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix/20230911")
 
 setenv("CC","cc")
 setenv("FC","ftn")
 setenv("CXX","CC")
 pushenv("CRAYPE_LINK_TYPE","dynamic")
 
 whatis("Description: GSI environment on Gaea with Intel Compilers")
+
diff --git a/regression/regression_param.sh b/regression/regression_param.sh
@@ -26,8 +26,8 @@ case $machine in
  ;;
  Gaea)
  sub_cmd="sub_gaea"
- memnode=251
- numcore=128
+ memnode=64
+ numcore=36
  ;;
  wcoss2)
  sub_cmd="sub_wcoss2"
@@ -69,8 +69,8 @@ case $regtest in
  topts[1]="0:30:00" ; popts[1]="48/2" ; ropts[1]="/1"
  topts[2]="0:30:00" ; popts[2]="60/3" ; ropts[2]="/2"
  elif [[ "$machine" = "Gaea" ]]; then
- topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1"
- topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2"
+ topts[1]="0:10:00" ; popts[1]="18/8/" ; ropts[1]="/1"
+ topts[2]="0:10:00" ; popts[2]="18/10/" ; ropts[2]="/2"
  elif [[ "$machine" = "wcoss2" ]]; then
  topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1"
  topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2"
@@ -99,8 +99,8 @@ case $regtest in
  topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1"
  topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1"
  elif [[ "$machine" = "Gaea" ]]; then
- topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1"
- topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1"
+ topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1"
+ topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1"
  elif [[ "$machine" = "wcoss2" ]]; then
  topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1"
  topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1"
@@ -129,8 +129,8 @@ case $regtest in
  topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1"
  topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1"
  elif [[ "$machine" = "Gaea" ]]; then
- topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1"
- topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1"
+ topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1"
+ topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1"
  elif [[ "$machine" = "wcoss2" ]]; then
  topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1"
  topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1"
@@ -158,8 +158,8 @@ case $regtest in
  topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1"
  topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1"
  elif [[ "$machine" = "Gaea" ]]; then
- topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1"
- topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1"
+ topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1"
+ topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1"
  elif [[ "$machine" = "wcoss2" ]]; then
  topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1"
  topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1"
@@ -188,8 +188,8 @@ case $regtest in
  topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1"
  topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1"
  elif [[ "$machine" = "Gaea" ]]; then
- topts[1]="0:15:00" ; popts[1]="28/1/" ; ropts[1]="/1"
- topts[2]="0:15:00" ; popts[2]="28/2/" ; ropts[2]="/1"
+ topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1"
+ topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1"
  elif [[ "$machine" = "wcoss2" ]]; then
  topts[1]="0:15:00" ; popts[1]="28/1/" ; ropts[1]="/1"
  topts[2]="0:15:00" ; popts[2]="28/2/" ; ropts[2]="/1"
@@ -218,8 +218,8 @@ case $regtest in
  topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1"
  topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1"
  elif [[ "$machine" = "Gaea" ]]; then
- topts[1]="0:30:00" ; popts[1]="14/8/" ; ropts[1]="/1"
- topts[2]="0:30:00" ; popts[2]="14/14/" ; ropts[2]="/1"
+ topts[1]="0:30:00" ; popts[1]="8/6/" ; ropts[1]="/1"
+ topts[2]="0:30:00" ; popts[2]="8/8/" ; ropts[2]="/1"
  elif [[ "$machine" = "wcoss2" ]]; then
  topts[1]="0:30:00" ; popts[1]="14/8/" ; ropts[1]="/1"
  topts[2]="0:30:00" ; popts[2]="14/14/" ; ropts[2]="/2"
@@ -248,8 +248,8 @@ case $regtest in
  topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1"
  topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2"
  elif [[ "$machine" = "Gaea" ]]; then
- topts[1]="0:10:00" ; popts[1]="16/2/" ; ropts[1]="/1"
- topts[2]="0:10:00" ; popts[2]="16/4/" ; ropts[2]="/2"
+ topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1"
+ topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2"
  elif [[ "$machine" = "wcoss2" ]]; then
  topts[1]="0:10:00" ; popts[1]="16/2/" ; ropts[1]="/1"
  topts[2]="0:10:00" ; popts[2]="16/4/" ; ropts[2]="/2"
@@ -315,7 +315,7 @@ elif [[ "$machine" = "Gaea" ]]; then
  export MPI_BUFS_PER_PROC=256
  export MPI_BUFS_PER_HOST=256
  export MPI_GROUP_MAX=256
- export APRUN="srun -n \$ntasks --cpus-per-task=\$threads"
+ export APRUN="srun --export=ALL --mpi=pmi2 -n \$ntasks"
 elif [[ "$machine" = "wcoss2" ]]; then
  export OMP_PLACES=cores
  export OMP_STACKSIZE=2G

diff --git a/regression/regression_var.sh b/regression/regression_var.sh
@@ -52,13 +52,13 @@ echo "Running Regression Tests on '$machine'";
 case $machine in
  Gaea)
  export queue="normal"
- export noscrub="/gpfs/f5/scratch/$LOGNAME/gsi_tmp/noscrub"
- export ptmp="/gpfs/f5/scratch/$LOGNAME/gsi_tmp/ptmp"
- export casesdir="/gpfs/f5/epic/world-shared/GSI_data/CASES/regtest"
+ export noscrub="/lustre/f2/scratch/$LOGNAME/gsi_tmp/noscrub"
+ export ptmp="/lustre/f2/scratch/$LOGNAME/gsi_tmp/ptmp"
+ export casesdir="/lustre/f2/dev/role.epic/contrib/GSI_data/CASES/regtest"
 
  export group="global"
  if [[ "$cmaketest" = "false" ]]; then
- export basedir="/gpfs/f5/dev/$LOGNAME/sandbox/GSI"
+ export basedir="/lustre/f2/dev/$LOGNAME/sandbox/GSI"
  fi
 
  export check_resource="no"

diff --git a/ush/module-setup.sh b/ush/module-setup.sh
@@ -59,7 +59,7 @@ elif [[ $MACHINE_ID = gaea* ]] ; then
  __ms_source_etc_profile=yes
  fi
 
- module reset
+ source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh
 
 elif [[ $MACHINE_ID = expanse* ]]; then
  # We are on SDSC Expanse

diff --git a/ush/sub_gaea b/ush/sub_gaea
@@ -88,8 +88,8 @@ output=${output:-$jobname.out}
 myuser=$LOGNAME
 myhost=$(hostname)
 
-if [ -d /gpfs/f5/scratch/$LOGNAME ]; then
- DATA=/gpfs/f5/scratch/$LOGNAME/tmp
+if [ -d /lustre/f2/scratch/$LOGNAME ]; then
+ DATA=/lustre/f2/scratch/$LOGNAME/tmp
 fi
 DATA=${DATA:-$ptmp/tmp}
 
@@ -110,18 +110,24 @@ echo ""
 echo "#SBATCH --output=$output" >> $cfile
 echo "#SBATCH --job-name=$jobname" >> $cfile
 echo "#SBATCH --qos=$queue" >> $cfile
+echo "#SBATCH --clusters=c4" >> $cfile
 echo "#SBATCH --time=$timew" >> $cfile
 echo "#SBATCH --nodes=$nodes --ntasks-per-node=$procs --cpus-per-task=$threads" >> $cfile
 echo "#SBATCH --account=$accnt" >> $cfile
 echo "#SBATCH --mem=0" >> $cfile
 
 echo "" >>$cfile
+echo "export ntasks=$(( $nodes * $procs ))" >> $cfile
+echo "export ppn=$procs" >> $cfile
+echo "export threads=$threads" >> $cfile
 echo "export OMP_NUM_THREADS=$threads" >> $cfile
+echo "ulimit -s unlimited" >> $cfile
+
 echo "" >>$cfile
 echo ". "$(awk '{ print $1, $2, $3, $4, $5, $6, $7, $8, $9 }' $regdir/regression_var.out) >>$cfile
 echo "" >>$cfile
 
-echo "module purge" >> $cfile
+echo "source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh" >> $cfile
 echo "module use $modulefiles" >> $cfile
 echo "module load gsi_gaea.intel" >> $cfile
 echo "module list" >> $cfile
@@ -152,7 +158,7 @@ sbatch=${sbatch:-sbatch}
 ofile=$DATA/subout$$
 >$ofile
 chmod 777 $ofile
-$sbatch $cfile >$ofile
+$sbatch  --export=ALL $cfile >$ofile
 rc=$?
 cat $ofile
 if [[ -w $SUBLOG ]];then