Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/master' into whannah/atm/add-zon…
Browse files Browse the repository at this point in the history
…al-mean-mod
  • Loading branch information
whannah1 committed Mar 27, 2024
2 parents 1b9b9a5 + b170268 commit b958311
Show file tree
Hide file tree
Showing 80 changed files with 3,587 additions and 505 deletions.
28 changes: 28 additions & 0 deletions cime_config/allactive/config_pesall.xml
Original file line number Diff line number Diff line change
Expand Up @@ -1379,6 +1379,34 @@
</nthrds>
</pes>
</mach>
<mach name="improv">
<pes compset="any" pesize="any">
<comment>improv: any compset on ne30np4 grid</comment>
<ntasks>
<ntasks_atm>-4</ntasks_atm>
<ntasks_lnd>-4</ntasks_lnd>
<ntasks_rof>-4</ntasks_rof>
<ntasks_ice>-4</ntasks_ice>
<ntasks_ocn>-4</ntasks_ocn>
<ntasks_cpl>-4</ntasks_cpl>
<ntasks_glc>-4</ntasks_glc>
<ntasks_wav>-4</ntasks_wav>
</ntasks>
</pes>
<pes compset=".*EAM.+ELM.+MPASSI.+SGLC.+SWAV.+BGC.*" pesize="any">
<comment>improv: BGC compset on ne30np4 grid</comment>
<ntasks>
<ntasks_atm>-8</ntasks_atm>
<ntasks_lnd>-8</ntasks_lnd>
<ntasks_rof>-8</ntasks_rof>
<ntasks_ice>-8</ntasks_ice>
<ntasks_ocn>-8</ntasks_ocn>
<ntasks_cpl>-8</ntasks_cpl>
<ntasks_glc>-8</ntasks_glc>
<ntasks_wav>-8</ntasks_wav>
</ntasks>
</pes>
</mach>
</grid>
<grid name="a%ne30np4.pg2_l%.+_oi%oEC60to30v3">
<mach name="compy">
Expand Down
128 changes: 128 additions & 0 deletions cime_config/config_grids.xml
Original file line number Diff line number Diff line change
Expand Up @@ -509,6 +509,46 @@
<mask>SOwISC12to60E2r4</mask>
</model_grid>

<model_grid alias="TL319_FRISwISC08to60E3r1" compset="(DATM|XATM|SATM)">
<grid name="atm">TL319</grid>
<grid name="lnd">TL319</grid>
<grid name="ocnice">FRISwISC08to60E3r1</grid>
<grid name="rof">JRA025</grid>
<grid name="glc">null</grid>
<grid name="wav">null</grid>
<mask>FRISwISC08to60E3r1</mask>
</model_grid>

<model_grid alias="TL319_FRISwISC04to60E3r1" compset="(DATM|XATM|SATM)">
<grid name="atm">TL319</grid>
<grid name="lnd">TL319</grid>
<grid name="ocnice">FRISwISC04to60E3r1</grid>
<grid name="rof">JRA025</grid>
<grid name="glc">null</grid>
<grid name="wav">null</grid>
<mask>FRISwISC04to60E3r1</mask>
</model_grid>

<model_grid alias="TL319_FRISwISC02to60E3r1" compset="(DATM|XATM|SATM)">
<grid name="atm">TL319</grid>
<grid name="lnd">TL319</grid>
<grid name="ocnice">FRISwISC02to60E3r1</grid>
<grid name="rof">JRA025</grid>
<grid name="glc">null</grid>
<grid name="wav">null</grid>
<mask>FRISwISC02to60E3r1</mask>
</model_grid>

<model_grid alias="TL319_FRISwISC01to60E3r1" compset="(DATM|XATM|SATM)">
<grid name="atm">TL319</grid>
<grid name="lnd">TL319</grid>
<grid name="ocnice">FRISwISC01to60E3r1</grid>
<grid name="rof">JRA025</grid>
<grid name="glc">null</grid>
<grid name="wav">null</grid>
<mask>FRISwISC01to60E3r1</mask>
</model_grid>

<model_grid alias="TL319_ECwISC30to60E2r1" compset="(DATM|XATM|SATM)">
<grid name="atm">TL319</grid>
<grid name="lnd">TL319</grid>
Expand Down Expand Up @@ -2414,6 +2454,14 @@
<file grid="ice|ocn" mask="WCAtl12to45E2r4">$DIN_LOC_ROOT/share/domains/domain.ocn.TL319_WCAtl12to45E2r4.210318.nc</file>
<file grid="atm|lnd" mask="SOwISC12to60E2r4">$DIN_LOC_ROOT/share/domains/domain.lnd.TL319_SOwISC12to60E2r4.210119.nc</file>
<file grid="ice|ocn" mask="SOwISC12to60E2r4">$DIN_LOC_ROOT/share/domains/domain.ocn.TL319_SOwISC12to60E2r4.210119.nc</file>
<file grid="atm|lnd" mask="FRISwISC08to60E3r1">$DIN_LOC_ROOT/share/domains/domain.lnd.TL319_FRISwISC08to60E3r1.240214.nc</file>
<file grid="ice|ocn" mask="FRISwISC08to60E3r1">$DIN_LOC_ROOT/share/domains/domain.ocn.TL319_FRISwISC08to60E3r1.240214.nc</file>
<file grid="atm|lnd" mask="FRISwISC04to60E3r1">$DIN_LOC_ROOT/share/domains/domain.lnd.TL319_FRISwISC04to60E3r1.240214.nc</file>
<file grid="ice|ocn" mask="FRISwISC04to60E3r1">$DIN_LOC_ROOT/share/domains/domain.ocn.TL319_FRISwISC04to60E3r1.240214.nc</file>
<file grid="atm|lnd" mask="FRISwISC02to60E3r1">$DIN_LOC_ROOT/share/domains/domain.lnd.TL319_FRISwISC02to60E3r1.240214.nc</file>
<file grid="ice|ocn" mask="FRISwISC02to60E3r1">$DIN_LOC_ROOT/share/domains/domain.ocn.TL319_FRISwISC02to60E3r1.240214.nc</file>
<file grid="atm|lnd" mask="FRISwISC01to60E3r1">$DIN_LOC_ROOT/share/domains/domain.lnd.TL319_FRISwISC01to60E3r1.240216.nc</file>
<file grid="ice|ocn" mask="FRISwISC01to60E3r1">$DIN_LOC_ROOT/share/domains/domain.ocn.TL319_FRISwISC01to60E3r1.240216.nc</file>
<file grid="atm|lnd" mask="ECwISC30to60E2r1">$DIN_LOC_ROOT/share/domains/domain.lnd.TL319_ECwISC30to60E2r1.201007.nc</file>
<file grid="ice|ocn" mask="ECwISC30to60E2r1">$DIN_LOC_ROOT/share/domains/domain.ocn.TL319_ECwISC30to60E2r1.201007.nc</file>
<file grid="atm|lnd" mask="IcoswISC30E3r5">$DIN_LOC_ROOT/share/domains/domain.lnd.TL319_IcoswISC30E3r5.231121.nc</file>
Expand Down Expand Up @@ -2787,6 +2835,34 @@
<desc>SOwISC12to60E2r4 is a MPAS ice/ocean grid with enhanced resolution of 12km in the Southern Ocean around Antarctica. The high resolution regions smoothly transition to the background resolution of the standard low resolution 60to30km grid:</desc>
</domain>

<domain name="FRISwISC08to60E3r1">
<nx>605169</nx>
<ny>1</ny>
<file grid="ice|ocn">$DIN_LOC_ROOT/share/domains/domain.ocn.FRISwISC08to60E3r1.240214.nc</file>
<desc>FRISwISC08to60E3r1 is a MPAS ice/ocean grid with enhanced resolution of 12km in the Southern Ocean around Antarctica and 4 km beneath the Filchner-Ronne Ice Shelf. The high resolution regions smoothly transition to the background resolution of the standard low resolution 60to30km grid:</desc>
</domain>

<domain name="FRISwISC04to60E3r1">
<nx>718054</nx>
<ny>1</ny>
<file grid="ice|ocn">$DIN_LOC_ROOT/share/domains/domain.ocn.FRISwISC04to60E3r1.240214.nc</file>
<desc>FRISwISC04to60E3r1 is a MPAS ice/ocean grid with enhanced resolution of 12km in the Southern Ocean around Antarctica and 4 km beneath the Filchner-Ronne Ice Shelf. The high resolution regions smoothly transition to the background resolution of the standard low resolution 60to30km grid:</desc>
</domain>

<domain name="FRISwISC02to60E3r1">
<nx>1055045</nx>
<ny>1</ny>
<file grid="ice|ocn">$DIN_LOC_ROOT/share/domains/domain.ocn.FRISwISC02to60E3r1.240214.nc</file>
<desc>FRISwISC02to60E3r1 is a MPAS ice/ocean grid with enhanced resolution of 12km in the Southern Ocean around Antarctica and 4 km beneath the Filchner-Ronne Ice Shelf. The high resolution regions smoothly transition to the background resolution of the standard low resolution 60to30km grid:</desc>
</domain>

<domain name="FRISwISC01to60E3r1">
<nx>2162616</nx>
<ny>1</ny>
<file grid="ice|ocn">$DIN_LOC_ROOT/share/domains/domain.ocn.FRISwISC01to60E3r1.240216.nc</file>
<desc>FRISwISC01to60E3r1 is a MPAS ice/ocean grid with enhanced resolution of 12km in the Southern Ocean around Antarctica and 4 km beneath the Filchner-Ronne Ice Shelf. The high resolution regions smoothly transition to the background resolution of the standard low resolution 60to30km grid:</desc>
</domain>

<domain name="ECwISC30to60E2r1">
<nx>237984</nx>
<ny>1</ny>
Expand Down Expand Up @@ -4106,6 +4182,38 @@
<map name="OCN2ATM_SMAPNAME">cpl/gridmaps/SOwISC12to60E2r4/map_SOwISC12to60E2r4_to_TL319_aave.210119.nc</map>
</gridmap>

<gridmap atm_grid="TL319" ocn_grid="FRISwISC08to60E3r1">
<map name="ATM2OCN_FMAPNAME">cpl/gridmaps/TL319/map_TL319_to_FRISwISC08to60E3r1_traave.20240214.nc</map>
<map name="ATM2OCN_SMAPNAME">cpl/gridmaps/TL319/map_TL319_to_FRISwISC08to60E3r1-nomask_trbilin.20240214.nc</map>
<map name="ATM2OCN_VMAPNAME">cpl/gridmaps/TL319/map_TL319_to_FRISwISC08to60E3r1_esmfpatch.20240214.nc</map>
<map name="OCN2ATM_FMAPNAME">cpl/gridmaps/FRISwISC08to60E3r1/map_FRISwISC08to60E3r1_to_TL319_traave.20240214.nc</map>
<map name="OCN2ATM_SMAPNAME">cpl/gridmaps/FRISwISC08to60E3r1/map_FRISwISC08to60E3r1_to_TL319_traave.20240214.nc</map>
</gridmap>

<gridmap atm_grid="TL319" ocn_grid="FRISwISC04to60E3r1">
<map name="ATM2OCN_FMAPNAME">cpl/gridmaps/TL319/map_TL319_to_FRISwISC04to60E3r1_traave.20240214.nc</map>
<map name="ATM2OCN_SMAPNAME">cpl/gridmaps/TL319/map_TL319_to_FRISwISC04to60E3r1-nomask_trbilin.20240214.nc</map>
<map name="ATM2OCN_VMAPNAME">cpl/gridmaps/TL319/map_TL319_to_FRISwISC04to60E3r1_esmfpatch.20240214.nc</map>
<map name="OCN2ATM_FMAPNAME">cpl/gridmaps/FRISwISC04to60E3r1/map_FRISwISC04to60E3r1_to_TL319_traave.20240214.nc</map>
<map name="OCN2ATM_SMAPNAME">cpl/gridmaps/FRISwISC04to60E3r1/map_FRISwISC04to60E3r1_to_TL319_traave.20240214.nc</map>
</gridmap>

<gridmap atm_grid="TL319" ocn_grid="FRISwISC02to60E3r1">
<map name="ATM2OCN_FMAPNAME">cpl/gridmaps/TL319/map_TL319_to_FRISwISC02to60E3r1_traave.20240214.nc</map>
<map name="ATM2OCN_SMAPNAME">cpl/gridmaps/TL319/map_TL319_to_FRISwISC02to60E3r1-nomask_trbilin.20240214.nc</map>
<map name="ATM2OCN_VMAPNAME">cpl/gridmaps/TL319/map_TL319_to_FRISwISC02to60E3r1_esmfpatch.20240214.nc</map>
<map name="OCN2ATM_FMAPNAME">cpl/gridmaps/FRISwISC02to60E3r1/map_FRISwISC02to60E3r1_to_TL319_traave.20240214.nc</map>
<map name="OCN2ATM_SMAPNAME">cpl/gridmaps/FRISwISC02to60E3r1/map_FRISwISC02to60E3r1_to_TL319_traave.20240214.nc</map>
</gridmap>

<gridmap atm_grid="TL319" ocn_grid="FRISwISC01to60E3r1">
<map name="ATM2OCN_FMAPNAME">cpl/gridmaps/TL319/map_TL319_to_FRISwISC01to60E3r1_traave.20240216.nc</map>
<map name="ATM2OCN_SMAPNAME">cpl/gridmaps/TL319/map_TL319_to_FRISwISC01to60E3r1-nomask_trbilin.20240216.nc</map>
<map name="ATM2OCN_VMAPNAME">cpl/gridmaps/TL319/map_TL319_to_FRISwISC01to60E3r1_esmfpatch.20240216.nc</map>
<map name="OCN2ATM_FMAPNAME">cpl/gridmaps/FRISwISC01to60E3r1/map_FRISwISC01to60E3r1_to_TL319_traave.20240216.nc</map>
<map name="OCN2ATM_SMAPNAME">cpl/gridmaps/FRISwISC01to60E3r1/map_FRISwISC01to60E3r1_to_TL319_traave.20240216.nc</map>
</gridmap>

<gridmap atm_grid="TL319" ocn_grid="ECwISC30to60E2r1">
<map name="ATM2OCN_FMAPNAME">cpl/gridmaps/TL319/map_TL319_to_ECwISC30to60E2r1_aave.201006.nc</map>
<map name="ATM2OCN_SMAPNAME">cpl/gridmaps/TL319/map_TL319_to_ECwISC30to60E2r1-nomask_bilin.201006.nc</map>
Expand Down Expand Up @@ -4637,6 +4745,26 @@
<map name="ROF2OCN_LIQ_RMAPNAME">cpl/cpl6/map_JRA025_to_SOwISC12to60E2r4_smoothed.r150e300.210119.nc</map>
</gridmap>

<gridmap ocn_grid="FRISwISC08to60E3r1" rof_grid="JRA025">
<map name="ROF2OCN_ICE_RMAPNAME">cpl/cpl6/map_JRA025_to_FRISwISC08to60E3r1_cstmnn.r150e300.20230926.nc</map>
<map name="ROF2OCN_LIQ_RMAPNAME">cpl/cpl6/map_JRA025_to_FRISwISC08to60E3r1_cstmnn.r150e300.20230926.nc</map>
</gridmap>

<gridmap ocn_grid="FRISwISC04to60E3r1" rof_grid="JRA025">
<map name="ROF2OCN_ICE_RMAPNAME">cpl/cpl6/map_JRA025_to_FRISwISC04to60E3r1_cstmnn.r150e300.20230912.nc</map>
<map name="ROF2OCN_LIQ_RMAPNAME">cpl/cpl6/map_JRA025_to_FRISwISC04to60E3r1_cstmnn.r150e300.20230912.nc</map>
</gridmap>

<gridmap ocn_grid="FRISwISC02to60E3r1" rof_grid="JRA025">
<map name="ROF2OCN_ICE_RMAPNAME">cpl/cpl6/map_JRA025_to_FRISwISC02to60E3r1_cstmnn.r150e300.20230926.nc</map>
<map name="ROF2OCN_LIQ_RMAPNAME">cpl/cpl6/map_JRA025_to_FRISwISC02to60E3r1_cstmnn.r150e300.20230926.nc</map>
</gridmap>

<gridmap ocn_grid="FRISwISC01to60E3r1" rof_grid="JRA025">
<map name="ROF2OCN_ICE_RMAPNAME">cpl/cpl6/map_JRA025_to_FRISwISC01to60E3r1_cstmnn.r150e300.20230926.nc</map>
<map name="ROF2OCN_LIQ_RMAPNAME">cpl/cpl6/map_JRA025_to_FRISwISC01to60E3r1_cstmnn.r150e300.20230926.nc</map>
</gridmap>

<gridmap ocn_grid="ECwISC30to60E2r1" rof_grid="JRA025">
<map name="ROF2OCN_ICE_RMAPNAME">cpl/cpl6/map_JRA025_to_ECwISC30to60E2r1_smoothed.r150e300.201006.nc</map>
<map name="ROF2OCN_LIQ_RMAPNAME">cpl/cpl6/map_JRA025_to_ECwISC30to60E2r1_smoothed.r150e300.201006.nc</map>
Expand Down
13 changes: 13 additions & 0 deletions cime_config/machines/Depends.chicoma-cpu.gnu.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# For this file, fixes non-BFB behavior of stealth feature on pm-cpu with -O2
set(NOOPT
eam/src/physics/cam/zm_conv.F90)

if (NOT DEBUG)
foreach(ITEM IN LISTS NOOPT)
e3sm_deoptimize_file("${ITEM}")
endforeach()
endif()




10 changes: 10 additions & 0 deletions cime_config/machines/Depends.chicoma-gpu.nvidia.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
list(APPEND REDUCE_OPT_LIST
homme/src/share/derivative_mod_base.F90
)

# Can use this flag to avoid internal compiler error for this file (with nvidia/21.11)
if (NOT DEBUG)
foreach(ITEM IN LISTS REDUCE_OPT_LIST)
e3sm_add_flags("${ITEM}" " -Mnovect")
endforeach()
endif()
10 changes: 10 additions & 0 deletions cime_config/machines/Depends.chicoma-gpu.nvidiagpu.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
list(APPEND REDUCE_OPT_LIST
homme/src/share/derivative_mod_base.F90
)

# Can use this flag to avoid internal compiler error for this file (with nvidia/21.11)
if (NOT DEBUG)
foreach(ITEM IN LISTS REDUCE_OPT_LIST)
e3sm_add_flags("${ITEM}" " -Mnovect")
endforeach()
endif()
4 changes: 3 additions & 1 deletion cime_config/machines/Depends.oneapi-ifx.cmake
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@

# compile mpas_seaice_core_interface.f90 with ifort, not ifx
e3sm_add_flags("${CMAKE_BINARY_DIR}/core_seaice/model_forward/mpas_seaice_core_interface.f90" "-fc=ifort")
if (NOT MPILIB STREQUAL "openmpi")
e3sm_add_flags("${CMAKE_BINARY_DIR}/core_seaice/model_forward/mpas_seaice_core_interface.f90" "-fc=ifort")
endif()
19 changes: 19 additions & 0 deletions cime_config/machines/cmake_macros/gnu_chicoma-gpu.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
string(APPEND CONFIG_ARGS " --host=cray")
if (COMP_NAME STREQUAL gptl)
string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY")
endif()
string(APPEND SLIBS " -lblas -llapack")
set(CXX_LINKER "FORTRAN")
if (NOT DEBUG)
string(APPEND CFLAGS " -O2 -g")
endif()
if (NOT DEBUG)
string(APPEND FFLAGS " -O2 -g")
endif()
string(APPEND CXX_LIBS " -lstdc++")
set(MPICC "cc")
set(MPICXX "CC")
set(MPIFC "ftn")
set(SCC "gcc")
set(SCXX "g++")
set(SFC "gfortran")
8 changes: 8 additions & 0 deletions cime_config/machines/cmake_macros/gnu_improv.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
if (COMP_NAME STREQUAL gptl)
string(APPEND CPPDEFS " -DHAVE_SLASHPROC")
endif()
string(APPEND CMAKE_C_FLAGS_RELEASE " -O2")
string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2")
set(PIO_FILESYSTEM_HINTS "gpfs")
string(APPEND SLIBS " /lcrc/group/e3sm/soft/improv/netlib-lapack/3.12.0/gcc-12.3.0/liblapack.a /lcrc/group/e3sm/soft/improv/netlib-lapack/3.12.0/gcc-12.3.0/libblas.a")

20 changes: 20 additions & 0 deletions cime_config/machines/cmake_macros/gnugpu_chicoma-gpu.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
string(APPEND CONFIG_ARGS " --host=cray")
set(USE_CUDA "TRUE")
string(APPEND CPPDEFS " -DGPU")
if (COMP_NAME STREQUAL gptl)
string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY")
endif()
string(APPEND CPPDEFS " -DTHRUST_IGNORE_CUB_VERSION_CHECK")
string(APPEND SLIBS " -lblas -llapack")
string(APPEND CUDA_FLAGS " -ccbin CC -O2 -arch sm_80 --use_fast_math")
string(APPEND KOKKOS_OPTIONS " -DKokkos_ARCH_AMPERE80=On -DKokkos_ENABLE_CUDA=On -DKokkos_ENABLE_CUDA_LAMBDA=On -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_OPENMP=Off")
if (NOT DEBUG)
string(APPEND CFLAGS " -O2")
string(APPEND FFLAGS " -O2")
endif()
set(MPICC "cc")
set(MPICXX "CC")
set(MPIFC "ftn")
set(SCC "cc")
set(SCXX "CC")
set(SFC "ftn")
24 changes: 24 additions & 0 deletions cime_config/machines/cmake_macros/nvidia_chicoma-gpu.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
string(APPEND CONFIG_ARGS " --host=cray")
if (COMP_NAME STREQUAL gptl)
string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY")
endif()
string(APPEND SLIBS " -lblas -llapack")
set(CXX_LINKER "FORTRAN")
if (NOT DEBUG)
string(APPEND CFLAGS " -O2")
endif()
if (NOT DEBUG)
string(APPEND CXXFLAGS " -O2")
endif()
if (NOT DEBUG)
string(APPEND FFLAGS " -O2")
endif()
if (compile_threaded)
string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_OPENMP=Off") # work-around for nvidia as kokkos is not passing "-mp" for threaded build
endif()
set(MPICC "cc")
set(MPICXX "CC")
set(MPIFC "ftn")
set(SCC "cc")
set(SCXX "CC")
set(SFC "ftn")
13 changes: 13 additions & 0 deletions cime_config/machines/cmake_macros/nvidiagpu_chicoma-gpu.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
string(APPEND CONFIG_ARGS " --host=cray")
set(USE_CUDA "TRUE")
string(APPEND CPPDEFS " -DGPU")
if (COMP_NAME STREQUAL gptl)
string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY")
endif()
string(APPEND CPPDEFS " -DTHRUST_IGNORE_CUB_VERSION_CHECK")
string(APPEND CUDA_FLAGS " -ccbin CC -O2 -arch sm_80 --use_fast_math")
string(APPEND SLIBS " -lblas -llapack")
set(CXX_LINKER "FORTRAN")
set(SCC "cc")
set(SCXX "CC")
set(SFC "ftn")
Loading

0 comments on commit b958311

Please sign in to comment.