Skip to content

Commit

Permalink
Merge branch 'main' into derecho_batch_update_121523
Browse files Browse the repository at this point in the history
  • Loading branch information
jedwards4b authored Dec 15, 2023
2 parents 3cd3a88 + 08843e4 commit 3055562
Show file tree
Hide file tree
Showing 47 changed files with 3,083 additions and 3,682 deletions.
64 changes: 64 additions & 0 deletions machines/aleph/config_machines.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
<machine MACH="aleph">
<DESC>XC50 SkyLake, os is CNL, 40 pes/node, batch system is PBSPro</DESC>
<OS>CNL</OS>
<COMPILERS>intel,gnu,cray</COMPILERS>
<MPILIBS>mpt,mpi-serial</MPILIBS>
<CIME_OUTPUT_ROOT>/proj/$ENV{USER}</CIME_OUTPUT_ROOT>
<DIN_LOC_ROOT>$ENV{DIN_LOC_ROOT}</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>$DIN_LOC_ROOT</DIN_LOC_ROOT_CLMFORC>
<DOUT_S_ROOT>${CIME_OUTPUT_ROOT}/archive/$CASE</DOUT_S_ROOT>
<BASELINE_ROOT>${CIME_OUTPUT_ROOT}/cesm_baselines</BASELINE_ROOT>
<GMAKE_J>8</GMAKE_J>
<BATCH_SYSTEM>pbs</BATCH_SYSTEM>
<SUPPORTED_BY> @ pusan.ac.kr</SUPPORTED_BY>
<MAX_TASKS_PER_NODE>40</MAX_TASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE>40</MAX_MPITASKS_PER_NODE>
<mpirun mpilib="default">
<executable>aprun</executable>
<arguments>
<arg name="hyperthreading" default="1"> -j {{ hyperthreading }}</arg>
<arg name="num_tasks"> -n {{ total_tasks }}</arg>
<arg name="tasks_per_node"> -N $MAX_MPITASKS_PER_NODE</arg>
<arg name="tasks_per_numa" > -S {{ tasks_per_numa }}</arg>
<arg name="thread_count"> -d $ENV{OMP_NUM_THREADS}</arg>
<arg name="env_thread_count">--mpmd-env OMP_NUM_THREADS=$OMP_NUM_THREADS</arg>
</arguments>
</mpirun>
<module_system type="module">
<init_path lang="perl">/opt/modules/default/init/perl.pm</init_path>
<init_path lang="python">/opt/modules/default/init/python.py</init_path>
<init_path lang="sh">/opt/modules/default/init/sh</init_path>
<init_path lang="csh">/opt/modules/default/init/csh</init_path>
<cmd_path lang="perl">/opt/modules/default/bin/modulecmd perl</cmd_path>
<cmd_path lang="python">/opt/modules/default/bin/modulecmd python</cmd_path>
<cmd_path lang="sh">module</cmd_path>
<cmd_path lang="csh">module</cmd_path>
<modules>
<command name="rm">craype-x86-skylake</command>
<command name="rm">PrgEnv-pgi</command>
<command name="rm">PrgEnv-intel</command>
<command name="rm">PrgEnv-cray</command>
<command name="rm">PrgEnv-gnu</command>
<command name="rm">cray-netcdf</command>
<command name="rm">cray-hdf5</command>
<command name="rm">cray-parallel-netcdf</command>
<command name="rm">papi</command>
</modules>
<modules compiler="intel">
<command name="load">PrgEnv-intel</command>
<command name="load">craype-x86-skylake</command>
<command name="load">craype-hugepages2M</command>
<command name="rm">perftools-base/7.0.4</command>
<command name="load">cray-netcdf/4.6.1.3</command>
<command name="load">cray-hdf5/1.10.2.0</command>
<command name="load">cray-parallel-netcdf/1.11.1.1</command>
<command name="load">papi/5.6.0.4</command>
<command name="load">gridftp/6.0</command>
<command name="load">cray-python/3.6.5.1</command>
</modules>
</module_system>
<environment_variables>
<env name="OMP_STACKSIZE">256M</env>
<env name="POSTPROCESS_PATH">/home/jedwards/workflow/CESM_postprocessing</env>
</environment_variables>
</machine>
84 changes: 84 additions & 0 deletions machines/athena/config_machines.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@

<machine MACH="athena">
<DESC> CMCC IBM iDataPlex, os is Linux, 16 pes/node, batch system is LSFd mpich</DESC>
<OS>LINUX</OS>
<COMPILERS>intel,intel15</COMPILERS>
<MPILIBS>mpich2</MPILIBS>
<CIME_OUTPUT_ROOT>/work/$USER/CESM2</CIME_OUTPUT_ROOT>
<DIN_LOC_ROOT>/users/home/dp16116/CESM2/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>$DIN_LOC_ROOT/atm/datm7</DIN_LOC_ROOT_CLMFORC>
<DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
<BASELINE_ROOT>$ENV{CESMDATAROOT}/ccsm_baselines</BASELINE_ROOT>
<CCSM_CPRNC>/users/home/dp16116/CESM2/cesm2.0.1/cime/tools/cprnc/cprnc</CCSM_CPRNC>
<PERL5LIB>/usr/lib64/perl5:/usr/share/perl5</PERL5LIB>
<GMAKE_J>8</GMAKE_J>
<BATCH_SYSTEM>lsf</BATCH_SYSTEM>
<SUPPORTED_BY> </SUPPORTED_BY>
<MAX_TASKS_PER_NODE>30</MAX_TASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE>15</MAX_MPITASKS_PER_NODE>
<PROJECT_REQUIRED>FALSE</PROJECT_REQUIRED>
<mpirun mpilib="default">
<executable> mpirun_Impi5 </executable>
</mpirun>
<module_system type="module">
<init_path lang="perl">/usr/share/Modules/init/perl.pm</init_path>
<init_path lang="python">/usr/share/Modules/init/python.py</init_path>
<init_path lang="csh">/usr/share/Modules/init/csh</init_path>
<init_path lang="sh">/usr/share/Modules/init/sh</init_path>
<cmd_path lang="perl">/usr/bin/modulecmd perl</cmd_path>
<cmd_path lang="python">/usr/bin/modulecmd python </cmd_path>
<cmd_path lang="sh">module</cmd_path>
<cmd_path lang="csh">module</cmd_path>
<modules>
<command name="purge"/>
</modules>
<modules compiler="intel">
<command name="load">ANACONDA2/python2.7</command>
<command name="load">INTEL/intel_xe_2015.3.187</command>
<command name="load">SZIP/szip-2.1_int15</command>
</modules>
<modules compiler="intel" mpilib="!mpi-serial" DEBUG="TRUE">
<command name="load">ESMF/esmf-6.3.0rp1-intelmpi-64-g_int15</command>
</modules>
<modules compiler="intel" mpilib="!mpi-serial" DEBUG="FALSE">
<command name="load">ESMF/esmf-6.3.0rp1-intelmpi-64-O_int15</command>
</modules>
<modules compiler="intel" mpilib="mpi-serial" DEBUG="TRUE">
<command name="load">ESMF/esmf-6.3.0rp1-mpiuni-64-g_int15</command>
</modules>
<modules compiler="intel" mpilib="mpi-serial" DEBUG="FALSE">
<command name="load">ESMF/esmf-6.3.0rp1-mpiuni-64-O_int15</command>
</modules>
<modules mpilib="mpi-serial">
<command name="load">HDF5/hdf5-1.8.15-patch1</command>
<command name="load">NETCDF/netcdf-C_4.3.3.1-F_4.4.2_C++_4.2.1</command>
</modules>
<modules mpilib="!mpi-serial">
<command name="load">HDF5/hdf5-1.8.15-patch1_parallel</command>
<command name="load">NETCDF/netcdf-C_4.3.3.1-F_4.4.2_C++_4.2.1_parallel</command>
<command name="load">PARALLEL_NETCDF/parallel-netcdf-1.6.1</command>
</modules>
<modules>
<command name="load">CMAKE/cmake-3.3.0-rc1</command>
</modules>
<modules compiler="intel">
<command name="unload">INTEL/intel_xe_2013.5.192</command>
<command name="unload">INTEL/intel_xe_2013</command>
<command name="unload">HDF5/hdf5-1.8.10-patch1</command>
<command name="load">INTEL/intel_xe_2015.3.187</command>
</modules>
</module_system>
<environment_variables>
<env name="OMP_STACKSIZE">256M</env>
</environment_variables>
<environment_variables compiler="intel">
<env name="I_MPI_EXTRA_FILESYSTEM_LIST">gpfs</env>
<env name="I_MPI_EXTRA_FILESYSTEM">on</env>
<env name="I_MPI_PLATFORM">snb</env>
<env name="I_MPI_HYDRA_BOOTSTRAP">lsf</env>
<env name="I_MPI_LSF_USE_COLLECTIVE_LAUNCH">1</env>
<env name="I_MPI_DAPL_UD">on</env>
<env name="I_MPI_DAPL_SCALABLE_PROGRESS">on</env>
<env name="XIOS_PATH">/users/home/models/nemo/xios-cmip6/intel_xe_2013</env>
</environment_variables>
</machine>
49 changes: 49 additions & 0 deletions machines/aws-hpc6a/config_machines.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
<machine MACH="aws-hpc6a">
<DESC>AWS HPC6a (96-core AMD) Nodes</DESC>
<OS>LINUX</OS>
<COMPILERS>intel</COMPILERS>
<MPILIBS>impi</MPILIBS>
<CIME_OUTPUT_ROOT>/scratch/$USER</CIME_OUTPUT_ROOT>
<DIN_LOC_ROOT>/scratch/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>$DIN_LOC_ROOT/atm/datm7</DIN_LOC_ROOT_CLMFORC>
<DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
<BASELINE_ROOT>$ENV{CESMDATAROOT}/cesm_baselines</BASELINE_ROOT>
<CCSM_CPRNC>$ENV{CESMDATAROOT}/tools/cime/tools/cprnc/cprnc</CCSM_CPRNC>
<GMAKE_J>8</GMAKE_J>
<BATCH_SYSTEM>slurm</BATCH_SYSTEM>
<SUPPORTED_BY>cseg</SUPPORTED_BY>
<!-- have not seen any performance benefit in smt -->
<MAX_TASKS_PER_NODE>96</MAX_TASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE>96</MAX_MPITASKS_PER_NODE>
<PROJECT_REQUIRED>FALSE</PROJECT_REQUIRED>
<mpirun mpilib="default">
<executable>scontrol show hostnames $SLURM_JOB_NODELIST > hostfile ; mpirun -f hostfile </executable>
<arguments>
<arg name="num_tasks">-n {{ total_tasks }}</arg>
</arguments>
</mpirun>
<mpirun mpilib="default" queue="serial">
<executable>env -u I_MPI_OFI_PROVIDER mpirun </executable>
<arguments>
<arg name="num_tasks">-n {{ total_tasks }}</arg>
</arguments>
</mpirun>
<module_system type="none"/>
<environment_variables>
<env name="OMP_STACKSIZE">256M</env>
<env name="TMPDIR">/scratch/$USER</env>
<env name="I_MPI_CC">icc</env>
<env name="I_MPI_CXX">icpc</env>
<env name="I_MPI_FC">ifort</env>
<env name="I_MPI_F90">ifort</env>
<env name="NETCDF_PATH">/opt/ncar/software</env>
<env name="ESMFMKFILE">/opt/ncar/esmf/lib/esmf.mk</env>
</environment_variables>
<environment_variables comp_interface="nuopc">
<env name="ESMF_RUNTIME_PROFILE">ON</env>
<env name="ESMF_RUNTIME_PROFILE_OUTPUT">SUMMARY</env>
</environment_variables>
<resource_limits>
<resource name="RLIMIT_STACK">-1</resource>
</resource_limits>
</machine>
125 changes: 125 additions & 0 deletions machines/casper/config_machines.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
<machine MACH="casper">
<DESC>NCAR GPU platform, os is Linux, 36 pes/node, batch system is pbs</DESC>
<OS>LINUX</OS>
<COMPILERS>nvhpc,intel</COMPILERS>
<MPILIBS>openmpi</MPILIBS>
<CIME_OUTPUT_ROOT>/glade/scratch/$USER</CIME_OUTPUT_ROOT>
<DIN_LOC_ROOT>$ENV{CESMDATAROOT}/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>/glade/p/cgd/tss/CTSM_datm_forcing_data</DIN_LOC_ROOT_CLMFORC>
<DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
<BASELINE_ROOT>$ENV{CESMDATAROOT}/cesm_baselines</BASELINE_ROOT>
<CCSM_CPRNC>$ENV{CESMDATAROOT}/tools/cime/tools/cprnc/cprnc</CCSM_CPRNC>
<GMAKE_J>8</GMAKE_J>
<BATCH_SYSTEM>pbs</BATCH_SYSTEM>
<SUPPORTED_BY>ASAP/CISL</SUPPORTED_BY>
<MAX_TASKS_PER_NODE>36</MAX_TASKS_PER_NODE>
<MAX_GPUS_PER_NODE compiler="nvhpc">8</MAX_GPUS_PER_NODE>
<MAX_MPITASKS_PER_NODE>36</MAX_MPITASKS_PER_NODE>
<MAX_CPUTASKS_PER_GPU_NODE>36</MAX_CPUTASKS_PER_GPU_NODE>
<GPU_TYPE>none,v100</GPU_TYPE>
<GPU_OFFLOAD>none,openacc,openmp,combined</GPU_OFFLOAD>
<PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
<mpirun mpilib="default">
<executable>mpirun</executable>
<arguments>
<arg name="anum_tasks"> -np {{ total_tasks }}</arg>
</arguments>
</mpirun>
<mpirun mpilib="openmpi">
<executable>mpirun</executable>
<arguments>
<arg name="anum_tasks"> -np {{ total_tasks }}</arg>
</arguments>
</mpirun>
<module_system type="module" allow_error="true">
<init_path lang="perl">/glade/u/apps/casper/23.10/spack/opt/spack/lmod/8.7.24/gcc/7.5.0/m4jx/lmod/lmod/init/perl</init_path>
<init_path lang="python">/glade/u/apps/casper/23.10/spack/opt/spack/lmod/8.7.24/gcc/7.5.0/m4jx/lmod/lmod/init/env_modules_python.py</init_path>
<init_path lang="sh">/glade/u/apps/casper/23.10/spack/opt/spack/lmod/8.7.24/gcc/7.5.0/m4jx/lmod/lmod/init/sh</init_path>
<init_path lang="csh">/glade/u/apps/casper/23.10/spack/opt/spack/lmod/8.7.24/gcc/7.5.0/m4jx/lmod/lmod/init/csh</init_path>
<cmd_path lang="perl">/glade/u/apps/casper/23.10/spack/opt/spack/lmod/8.7.24/gcc/7.5.0/m4jx/lmod/lmod/libexec/lmod perl</cmd_path>
<cmd_path lang="python">/glade/u/apps/casper/23.10/spack/opt/spack/lmod/8.7.24/gcc/7.5.0/m4jx/lmod/lmod/libexec/lmod python</cmd_path>
<cmd_path lang="sh">module</cmd_path>
<cmd_path lang="csh">module</cmd_path>
<modules>
<command name="purge"/>
<command name="load">ncarenv/23.10</command>
<command name="load">cmake/3.26.3</command>
</modules>
<modules compiler="nvhpc">
<command name="load">nvhpc/23.7</command>
</modules>
<modules compiler="intel">
<command name="load">intel/2023.2.1</command>
<command name="load">mkl/2023.2.0</command>
</modules>
<modules mpilib="openmpi" compiler="nvhpc" gpu_offload="!none">
<command name="load">cuda/12.2.1</command>
</modules>
<modules mpilib="openmpi" compiler="nvhpc">
<command name="load">openmpi/4.1.6</command>
<command name="load">netcdf-mpi/4.9.2</command>
<command name="load">parallel-netcdf/1.12.3</command>
</modules>
<modules mpilib="mpi-serial" compiler="nvhpc">
<command name="load">netcdf/4.9.2</command>
</modules>
<modules mpilib="openmpi" compiler="intel">
<command name="load">openmpi/4.1.6</command>
<command name="load">netcdf-mpi/4.9.2</command>
<command name="load">parallel-netcdf/1.12.3</command>
</modules>
<modules mpilib="mpi-serial" compiler="intel">
<command name="load">netcdf/4.9.2</command>
</modules>
<!-- prebuild ESMF lib for NUOPC driver -->
<modules compiler="intel" mpilib="openmpi" DEBUG="TRUE">
<command name="use">/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/</command>
<command name="load">esmf-8.4.0b08_casper-ncdfio-openmpi-g</command>
</modules>
<modules compiler="intel" mpilib="openmpi" DEBUG="FALSE">
<command name="use">/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/</command>
<command name="load">esmf-8.4.0b08_casper-ncdfio-openmpi-O</command>
</modules>
<modules compiler="nvhpc" mpilib="openmpi" DEBUG="TRUE">
<command name="use">/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2/</command>
<command name="load">esmf-8.4.1b01-ncdfio-openmpi-g</command>
</modules>
<modules compiler="nvhpc" mpilib="openmpi" DEBUG="FALSE">
<command name="use">/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2/</command>
<command name="load">esmf-8.4.1b01-ncdfio-openmpi-O</command>
</modules>
<modules compiler="pgi" mpilib="openmpi" DEBUG="TRUE">
<command name="use">/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/</command>
<command name="load">esmf-8.4.0b08_casper-ncdfio-openmpi-g</command>
</modules>
<modules compiler="pgi" mpilib="openmpi" DEBUG="FALSE">
<command name="use">/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/</command>
<command name="load">esmf-8.2.0b11_casper-ncdfio-openmpi-O</command>
</modules>
<modules>
<command name="load">ncarcompilers/0.5.0</command>
</modules>
<modules>
<command name="load">parallelio/2.6.2</command>
<command name="load">esmf/8.5.0</command>
<command name="load">ncarcompilers/1.0.0</command>
</modules>
</module_system>
<environment_variables>
<env name="MODULEPATH">/glade/u/apps/dav/modulefiles/default/compilers:/glade/u/apps/dav/modulefiles/default/idep</env>
<env name="OMP_STACKSIZE">256M</env>
<env name="TMPDIR">/glade/scratch/$USER</env>
<env name="CESMDATAROOT">/glade/p/cesmdata/cseg</env>
<env name="NETCDF_PATH">$ENV{NETCDF}</env>
</environment_variables>
<environment_variables comp_interface="nuopc">
<env name="ESMF_RUNTIME_PROFILE">ON</env>
<env name="ESMF_RUNTIME_PROFILE_OUTPUT">SUMMARY</env>
</environment_variables>
<environment_variables compiler="nvhpc" gpu_offload="!none">
<env name="RTE_KERNELS">openacc</env>
</environment_variables>
<resource_limits>
<resource name="RLIMIT_STACK">-1</resource>
</resource_limits>
</machine>
61 changes: 61 additions & 0 deletions machines/centos7-linux/config_machines.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
<machine MACH="centos7-linux">
<DESC>
Example port to centos7 linux system with gcc, netcdf, pnetcdf and mpich
using modules from http://www.admin-magazine.com/HPC/Articles/Environment-Modules
</DESC>
<!-- Moved to top level config_machines.xml
<NODENAME_REGEX>regex.expression.matching.your.machine</NODENAME_REGEX>
-->
<OS>LINUX</OS>
<PROXY> https://howto.get.out </PROXY>
<COMPILERS>gnu</COMPILERS>
<MPILIBS>mpich</MPILIBS>
<PROJECT>none</PROJECT>
<SAVE_TIMING_DIR> </SAVE_TIMING_DIR>
<CIME_OUTPUT_ROOT>$ENV{HOME}/cesm/scratch</CIME_OUTPUT_ROOT>
<DIN_LOC_ROOT>$ENV{HOME}/cesm/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>$ENV{HOME}/cesm/inputdata/lmwg</DIN_LOC_ROOT_CLMFORC>
<DOUT_S_ROOT>$ENV{HOME}/cesm/archive/$CASE</DOUT_S_ROOT>
<BASELINE_ROOT>$ENV{HOME}/cesm/cesm_baselines</BASELINE_ROOT>
<CCSM_CPRNC>$ENV{HOME}/cesm/tools/cime/tools/cprnc/cprnc</CCSM_CPRNC>
<GMAKE>make</GMAKE>
<GMAKE_J>8</GMAKE_J>
<BATCH_SYSTEM>none</BATCH_SYSTEM>
<SUPPORTED_BY>[email protected]</SUPPORTED_BY>
<MAX_TASKS_PER_NODE>8</MAX_TASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE>8</MAX_MPITASKS_PER_NODE>
<PROJECT_REQUIRED>FALSE</PROJECT_REQUIRED>
<mpirun mpilib="default">
<executable>mpiexec</executable>
<arguments>
<arg name="ntasks"> -np {{ total_tasks }} </arg>
</arguments>
</mpirun>
<module_system type="module" allow_error="true">
<init_path lang="perl">/usr/share/Modules/init/perl.pm</init_path>
<init_path lang="python">/usr/share/Modules/init/python.py</init_path>
<init_path lang="csh">/usr/share/Modules/init/csh</init_path>
<init_path lang="sh">/usr/share/Modules/init/sh</init_path>
<cmd_path lang="perl">/usr/bin/modulecmd perl</cmd_path>
<cmd_path lang="python">/usr/bin/modulecmd python</cmd_path>
<cmd_path lang="sh">module</cmd_path>
<cmd_path lang="csh">module</cmd_path>
<modules>
<command name="purge"/>
<command name="load">lang/python/3.7.0</command>
</modules>
<modules compiler="gnu">
<command name="load">compiler/gnu/8.2.0</command>
<command name="load">mpi/3.3/gnu-8.2.0</command>
<command name="load">tool/netcdf/4.7.4/gcc-8.2.0</command>
<command name="load">tool/parallel-netcdf/1.12.1/mpich</command>
</modules>
</module_system>
<environment_variables>
<env name="OMP_STACKSIZE">256M</env>
<env name="ESMFMKFILE">$ENV{HOME}/ESMF_8_2_0b22/lib/libg/Linux.gfortran.64.mvapich2.default/esmf.mk</env>
</environment_variables>
<resource_limits>
<resource name="RLIMIT_STACK">-1</resource>
</resource_limits>
</machine>
Loading

0 comments on commit 3055562

Please sign in to comment.