Merge branch 'main' into derecho_updates_080224

config_inputdata.xml

@@ -7,28 +7,16 @@
   <!-- inputdata_chksum.dat is found on the server in the directory above inputdata -->
   <!-- it will be searched for filename and chksum of each downloaded file.  -->
   <!-- see the file ftp://ftp.cgd.ucar.edu/cesm/inputdata_chksum.dat for proper format. -->
-  <server>
-    <comment>grid ftp requires the globus-url-copy tool on the client side </comment>
-    <protocol>gftp</protocol>
-    <address>ftp://gridanon.cgd.ucar.edu:2811/cesm/inputdata/</address>
-    <checksum>../inputdata_checksum.dat</checksum>
-  </server>
-
   <server>
     <protocol>wget</protocol>
-    <address>ftp://ftp.cgd.ucar.edu/cesm/inputdata/</address>
-    <user>anonymous</user>
-    <password>[email protected]</password>
+    <address>https://ftp.cgd.ucar.edu/cesm/inputdata/</address>
     <checksum>../inputdata_checksum.dat</checksum>
   </server>
 
   <server>
-    <comment> ftp requires the python package ftplib </comment>
-    <protocol>ftp</protocol>
-    <address>ftp.cgd.ucar.edu/cesm/inputdata</address>
-    <user>anonymous</user>
-    <password>[email protected]</password>
-    <checksum>../inputdata_checksum.dat</checksum>
+    <comment> wget access to subversion </comment>
+    <protocol>wget</protocol>
+    <address>https://svn-ccsm-inputdata.cgd.ucar.edu/trunk/inputdata/</address>
   </server>
 
   <server>

machines/Depends.intel-oneapi

@@ -7,11 +7,15 @@ bndry_mod.o \
 prim_advance_mod.o
 
 # FMS netcdf_io gives ICE in intel-oneapi/2024.0.2
+
 REDUCED_OPT_OBJS=\
 netcdf_io.o \
 fms_netcdf_domain_io.o \
 fms_netcdf_unstructured_domain_io.o
 
+REDUCED_DEBUG_OBJS=\
+unicon.o
+
 # shr_wv_sat_mod does not need to have better than ~0.1% precision, and benefits
 # enormously from a lower precision in the vector functions.
 REDUCED_PRECISION_OBJS=\
@@ -49,5 +53,8 @@ ifeq ($(DEBUG),FALSE)
 	  $(CC) -c $(INCLDIR) $(INCS) $(CFLAGS) -O3 -fp-model fast $<
   $(PUMAS_MG_OBJS): %.o: %.F90
 	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -no-fma -ftz -no-prec-sqrt -qoverride-limits -no-inline-max-total-size -inline-factor=200 -qopt-report=5 $<
+else
+  $(REDUCED_DEBUG_OBJS): %.o: %.F90
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O0 -check nobounds -check nopointers $<
 
 endif

machines/Depends.nvhpc

@@ -13,6 +13,18 @@ REDUCED_OPT_OBJS=\
 SatellitePhenologyMod.o \
 esm.o
 
+REDUCED_ERRORCHKS_IN_DEBUG=\
+mpp.o \
+horiz_interp_conserve.o \
+data_override.o \
+MOM_io_infra.o \
+dynColumnStateUpdaterMod.o \
+prim_advection_mod.o \
+MARBL_forcing_mod.o \
+MOM_internal_tide_input.o \
+MOM_ice_shelf.o \
+mom_surface_forcing_nuopc.o
+
 REDUCED_PRECISION_OBJS=\
 shr_wv_sat_mod.o
 
@@ -154,9 +166,12 @@ ifeq ($(DEBUG),FALSE)
 #  $(SHR_RANDNUM_C_OBJS): %.o: %.c
 #	  $(CC) -c $(INCLDIR) $(INCS) $(CFLAGS) -O3 -fast $<
   $(PUMAS_OBJS): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt $(GPUFLAGS) $<
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt $(OPENACC_GPU_FLAGS) $<
   $(RRTMGP_OBJS): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(GPUFLAGS) $<
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(OPENACC_GPU_FLAGS) $<
   $(CLUBB_OBJS): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(GPUFLAGS) $<
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(OPENACC_GPU_FLAGS) $<
+else
+  $(REDUCED_ERRORCHKS_IN_DEBUG): %.o: %.F90
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -Mnobounds -Ktrap=none -Knoieee $<
 endif

machines/aleph/config_batch.xml

@@ -0,0 +1,10 @@
+<batch_system MACH="aleph" type="pbs" >
+  <directives>
+    <directive>-l nodes={{ num_nodes }}</directive>
+    <directive>-q iccp</directive>
+    <directive> -V </directive>
+  </directives>
+  <queues>
+    <queue walltimemax="24:00:00" default="true" >iccp</queue>
+  </queues>
+</batch_system>

machines/athena/config_batch.xml

@@ -0,0 +1,20 @@
+<!-- athena is lsf -->
+<batch_system MACH="athena" type="lsf">
+  <submit_args>
+    <argument> -q $JOB_QUEUE </argument>
+    <argument> -W $JOB_WALLCLOCK_TIME </argument>
+    <argument> -P $PROJECT </argument>
+  </submit_args>
+  <directives>
+    <directive                       > -R "span[ptile={{ tasks_per_node }}]"</directive>
+    <directive                       > -N  </directive>
+    <directive default="poe"         > -a {{ poe }} </directive>
+  </directives>
+  <queues>
+    <queue walltimemin="00:00" walltimemax="02:00">poe_short</queue>
+    <queue walltimemin="02:00" walltimemax="04:00" default="true">poe_medium</queue>
+    <queue walltimemin="04:00" walltimemax="08:00">poe_long</queue>
+  </queues>
+</batch_system>
+
+

machines/aws-hpc6a/config_batch.xml

@@ -0,0 +1,12 @@
+<batch_system MACH="aws-hpc6a" type="slurm">
+  <batch_submit>sbatch</batch_submit>
+  <submit_args>
+    <argument> --time $JOB_WALLCLOCK_TIME </argument>
+    <argument> -p $JOB_QUEUE </argument>
+  </submit_args>
+  <queues>
+    <queue walltimemax="144:00:00" nodemin="1" nodemax="96">regular</queue>
+    <queue walltimemax="1:00:00"   nodemin="1" nodemax="1">build</queue>
+    <queue default="true" walltimemax="4:00:00"   jobmin="1" jobmax="1">serial</queue>
+  </queues>
+</batch_system>

machines/casper/config_batch.xml

@@ -0,0 +1,21 @@
+<!-- casper pbs -->
+<batch_system MACH="casper" type="pbs">
+  <batch_submit>qsub</batch_submit>
+  <submit_args>
+    <argument> -l gpu_type=$GPU_TYPE </argument>
+  </submit_args>
+  <directives queue="casper" compiler="nvhpc" gpu_enabled="true">
+    <directive default="/bin/bash" > -S {{ shell }} </directive>
+    <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }}:mps=1 </directive>
+  </directives>
+  <directives queue="casper" gpu_enabled="false">
+    <directive default="/bin/bash" > -S {{ shell }} </directive>
+    <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=300GB:ngpus={{ ngpus_per_node }} </directive>
+  </directives>
+  <!-- Unknown queues use the batch directives for the regular queue -->
+  <unknown_queue_directives>casper</unknown_queue_directives>
+  <queues>
+    <queue walltimemax="12:00:00" nodemin="1" nodemax="10">casper</queue>
+  </queues>
+</batch_system>
+

machines/cmake_macros/intel-oneapi.cmake

@@ -0,0 +1,63 @@
+string(APPEND CFLAGS " -qno-opt-dynamic-align -fp-model precise -std=gnu99")
+if (compile_threaded)
+  string(APPEND CFLAGS " -qopenmp")
+endif()
+if (NOT DEBUG)
+  string(APPEND CFLAGS " -O2 -debug minimal")
+endif()
+if (DEBUG)
+  string(APPEND CFLAGS " -O0 -g")
+endif()
+string(APPEND CFLAGS " -no-fma")
+string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DCPRINTEL")
+string(APPEND CXX_LDFLAGS " -cxxlib")
+set(CXX_LINKER "FORTRAN")
+set(FC_AUTO_R8 "-r8")
+string(APPEND FFLAGS " -qno-opt-dynamic-align  -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model source")
+if (compile_threaded)
+  string(APPEND FFLAGS " -qopenmp")
+endif()
+if (DEBUG)
+  string(APPEND FFLAGS " -O0 -g -check bounds -check pointers -fpe0 -check noarg_temp_created")
+endif()
+if (NOT DEBUG)
+  string(APPEND FFLAGS " -O2 -debug minimal")
+endif()
+string(APPEND FFLAGS " -no-fma")
+set(FFLAGS_NOOPT "-O0")
+set(FIXEDFLAGS "-fixed")
+set(FREEFLAGS "-free")
+if (compile_threaded)
+  string(APPEND LDFLAGS " -qopenmp")
+endif()
+set(MPICC "mpicc")
+set(MPICXX "mpicxx")
+set(MPIFC "mpif90")
+set(SCC "icc")
+set(SCXX "icpc")
+set(SFC "ifort")
+if (MPILIB STREQUAL mpich)
+  string(APPEND SLIBS " -mkl=cluster")
+endif()
+if (MPILIB STREQUAL mpich2)
+  string(APPEND SLIBS " -mkl=cluster")
+endif()
+if (MPILIB STREQUAL mvapich)
+  string(APPEND SLIBS " -mkl=cluster")
+endif()
+if (MPILIB STREQUAL mvapich2)
+  string(APPEND SLIBS " -mkl=cluster")
+endif()
+if (MPILIB STREQUAL mpt)
+  string(APPEND SLIBS " -mkl=cluster")
+endif()
+if (MPILIB STREQUAL openmpi)
+  string(APPEND SLIBS " -mkl=cluster")
+endif()
+if (MPILIB STREQUAL impi)
+  string(APPEND SLIBS " -mkl=cluster")
+endif()
+if (MPILIB STREQUAL mpi-serial)
+  string(APPEND SLIBS " -mkl")
+endif()
+set(SUPPORTS_CXX "TRUE")