Merge branch 'update/derecho_support' into 'ew-develop' (PR ESMCI#5)

Advance EarthWorksOrg external to include changes from the ESMCI/ccs_config_cesm 'ccs_config_cesm0.0.72' tag.
jedwards4b · Sep 7, 2023 · caeefbd · caeefbd
2 parents 710afc3 + 05de0d0
commit caeefbd
Show file tree

Hide file tree

Showing 5 changed files with 152 additions and 2 deletions.
diff --git a/machines/cmake_macros/derecho.cmake b/machines/cmake_macros/derecho.cmake
@@ -0,0 +1,11 @@
+if (COMP_NAME STREQUAL gptl)
+  string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_VPRINTF -DHAVE_BACKTRACE -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
+endif()
+set(NETCDF_PATH "$ENV{NETCDF}")
+set(PIO_FILESYSTEM_HINTS "lustre")
+set(PNETCDF_PATH "$ENV{PNETCDF}")
+# If we want to use cray-libsci instead of mkl uncomment this line as well as the module in config_machines.xml
+string(REPLACE "-mkl=cluster" "" SLIBS "${SLIBS}")
+#string(REPLACE "-mkl=cluster" "-qmkl=cluster" SLIBS "${SLIBS}")
+#string(APPEND CPPDEFS " -DNO_SHR_VMATH ")
+string(APPEND CPPDEFS " -DHAVE_GETTID")
diff --git a/machines/cmake_macros/intel_derecho.cmake b/machines/cmake_macros/intel_derecho.cmake
@@ -0,0 +1,11 @@
+set(CONFIG_ARGS "--host=cray")
+string(APPEND CFLAGS " -march=core-avx2")
+string(APPEND FFLAGS " -march=core-avx2")
+if (COMP_NAME STREQUAL gptl)
+  string(APPEND CPPDEFS " -DHAVE_SLASHPROC")
+endif()
+if (COMP_NAME STREQUAL mpi-serial)
+  string(APPEND CFLAGS " -std=c89 ")
+endif()	
+set(SCC icx)
+set(SFC ifort)
diff --git a/machines/cmake_macros/nvhpc_derecho.cmake b/machines/cmake_macros/nvhpc_derecho.cmake
@@ -0,0 +1,6 @@
+if (COMP_NAME STREQUAL mpi-serial)
+   string(APPEND CFLAGS " -std=c89 ")
+endif()
+if (NOT DEBUG)
+  string(APPEND FFLAGS " -target=zen3")
+endif()
diff --git a/machines/config_batch.xml b/machines/config_batch.xml
@@ -376,6 +376,17 @@
     </queues>
   </batch_system>
 
+  <batch_system MACH="derecho" type="pbs" >
+    <batch_submit>qsub</batch_submit>
+    <directives>
+      <directive default="/bin/bash" > -S {{ shell }}  </directive>
+      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}</directive>
+    </directives>
+    <queues>
+      <queue walltimemax="4:00:00" nodemin="1" nodemax="2488" >main</queue>
+    </queues>
+  </batch_system>
+
   <batch_system MACH="eastwind" type="slurm" >
     <batch_submit>sbatch</batch_submit>
     <submit_args>

diff --git a/machines/config_machines.xml b/machines/config_machines.xml
@@ -1284,6 +1284,117 @@ This allows using a different mpirun command to launch unit tests
     </environment_variables>
   </machine>
 
+  <machine MACH="derecho">
+    <DESC>NCAR AMD EPYC system</DESC>
+    <NODENAME_REGEX>de.*.hpc.ucar.edu</NODENAME_REGEX>
+    <OS>CNL</OS>
+    <COMPILERS>intel,gnu,cray,nvhpc,intel-oneapi,intel-classic</COMPILERS>
+    <MPILIBS>mpich</MPILIBS>
+    <CIME_OUTPUT_ROOT>$ENV{SCRATCH}</CIME_OUTPUT_ROOT>
+    <DIN_LOC_ROOT>$ENV{CESMDATAROOT}/inputdata</DIN_LOC_ROOT>
+    <DIN_LOC_ROOT_CLMFORC>$ENV{CESMDATAROOT}/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
+    <DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
+    <BASELINE_ROOT>$ENV{CESMDATAROOT}/ccsm_baselines</BASELINE_ROOT>
+    <CCSM_CPRNC>$ENV{CESMDATAROOT}/cprnc/cprnc</CCSM_CPRNC>
+    <GMAKE_J>16</GMAKE_J>
+    <BATCH_SYSTEM>pbs</BATCH_SYSTEM>
+    <SUPPORTED_BY>cseg</SUPPORTED_BY>
+    <MAX_TASKS_PER_NODE>128</MAX_TASKS_PER_NODE>
+    <MAX_MPITASKS_PER_NODE>128</MAX_MPITASKS_PER_NODE>
+    <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
+    <mpirun mpilib="default">
+      <executable>mpiexec</executable>
+      <arguments>
+        <arg name="label"> --label</arg>
+        <arg name="buffer"> --line-buffer</arg>
+        <arg name="num_tasks" > -n {{ total_tasks }}</arg>
+      </arguments>
+    </mpirun>
+    <module_system type="module" allow_error="true">
+      <init_path lang="perl">$ENV{LMOD_ROOT}/lmod/init/perl</init_path>
+      <init_path lang="python">$ENV{LMOD_ROOT}/lmod/init/env_modules_python.py</init_path>
+      <init_path lang="sh">$ENV{LMOD_ROOT}/lmod/init/sh</init_path>
+      <init_path lang="csh">$ENV{LMOD_ROOT}/lmod/init/csh</init_path>
+      <cmd_path lang="perl">$ENV{LMOD_ROOT}/lmod/libexec/lmod perl</cmd_path>
+      <cmd_path lang="python">$ENV{LMOD_ROOT}/lmod/libexec/lmod python</cmd_path>
+      <cmd_path lang="sh">module</cmd_path>
+      <cmd_path lang="csh">module</cmd_path>
+      <modules>
+	<command name="load">cesmdev/1.0</command>
+	<command name="load">ncarenv/23.06</command>
+        <command name="purge"/>
+	<command name="load">craype</command>
+      </modules>
+      <modules compiler="intel">
+        <command name="load">intel/2023.0.0</command>
+	<command name="load">mkl</command>
+      </modules>
+      <modules compiler="intel-oneapi">
+        <command name="load">intel-oneapi/2023.0.0</command>
+	<command name="load">mkl</command>
+      </modules>
+      <modules compiler="intel-classic">
+        <command name="load">intel-classic/2023.0.0</command>
+	<command name="load">mkl</command>
+      </modules>
+      <modules compiler="cray">
+        <command name="load">cce/15.0.1</command>
+        <command name="load">cray-libsci/23.02.1.1</command>
+      </modules>
+      <modules compiler="gnu">
+        <command name="load">gcc/12.2.0</command>
+        <command name="load">cray-libsci/23.02.1.1</command>
+      </modules>
+      <modules compiler="nvhpc">
+        <command name="load">nvhpc/23.1</command>
+      </modules>
+      <modules>
+	<command name="load">ncarcompilers/1.0.0</command>
+        <command name="load">cmake</command>
+      </modules>
+      <modules mpilib="mpich">
+	<command name="load">cray-mpich/8.1.25</command>
+      </modules>
+      <modules mpilib="mpi-serial">
+        <command name="load">mpi-serial/2.3.0</command>
+      </modules>
+
+      <modules mpilib="mpi-serial">
+        <command name="load">netcdf/4.9.2</command>
+      </modules>
+
+      <modules mpilib="!mpi-serial">
+        <command name="load">netcdf-mpi/4.9.2</command>
+        <command name="load">parallel-netcdf/1.12.3</command>
+      </modules>
+
+      <modules DEBUG="TRUE">
+	<command name="load">parallelio/2.6.0-debug</command>
+	<command name="load">esmf/8.5.0b23-debug</command>
+      </modules>
+
+      <modules DEBUG="FALSE">
+	<command name="load">parallelio/2.6.0</command>
+	<command name="load">esmf/8.5.0b23</command>
+        </modules>
+    </module_system>
+
+    <environment_variables>
+      <env name="OMP_STACKSIZE">64M</env>
+      <env name="FI_CXI_RX_MATCH_MODE">hybrid</env>
+    </environment_variables>
+    <!-- derecho has both gpfs and lustre file systems so I think this setting may cause issues -->
+    <!--    <environment_variables mpilib="mpich">
+         <env name="MPICH_MPIIO_HINTS">*:romio_cb_read=enable:romio_cb_write=enable:striping_factor=2</env>
+         </environment_variables>
+      -->
+    <environment_variables comp_interface="nuopc">
+      <!-- required on all systems for timing file output --> 
+      <env name="ESMF_RUNTIME_PROFILE">ON</env>
+      <env name="ESMF_RUNTIME_PROFILE_OUTPUT">SUMMARY</env>
+    </environment_variables>
+  </machine>
+
   <machine MACH="eastwind">
     <DESC>PNL IBM Xeon cluster, os is Linux (pgi), batch system is SLURM</DESC>
     <OS>LINUX</OS>
@@ -1919,12 +2030,12 @@ This allows using a different mpirun command to launch unit tests
 
       <modules DEBUG="TRUE">
 	<command name="load">parallelio/2.5.10-debug</command>
-	<command name="load">esmf/8.5.0b19-debug</command>
+	<command name="load">esmf/8.5.0b21-debug</command> 
       </modules>
 
       <modules DEBUG="FALSE">
 	<command name="load">parallelio/2.5.10</command>
-	<command name="load">esmf/8.5.0b19</command>
+	<command name="load">esmf/8.5.0b21</command>
       </modules>
     </module_system>