Merge branch 'jgfouca/eamxx_downstream_merge_2024_06_26' into master …

…(PR E3SM-Project#6489)

Downstream merge of eamxx/scream.

[BFB]

.github/workflows/e3sm-gh-md-linter.yml

@@ -7,6 +7,8 @@ on:
     branches: ["master"]
     paths:
         - '**/*.md'
+        # for now let's not lint files in eamxx
+        - '!components/eamxx/**/*.md'
 
 jobs:
   linter:

.github/workflows/eamxx-gh-pages.yml

@@ -53,14 +53,14 @@ jobs:
         run: |
           echo "= The job was automatically triggered by a ${{github.event_name}} event."
 
-      - name: Set up Python 3.10
+      - name: Set up Python 3.11
         uses: actions/[email protected]
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
       - name: Install Python deps
         run: |
-          pip install mkdocs pymdown-extensions mkdocs-material mdutils
+          pip install mkdocs pymdown-extensions mkdocs-material mdutils mkdocs-bibtex
   
       - name: Generate EAMxx params docs
         working-directory: components/eamxx/scripts

.gitignore

@@ -9,6 +9,10 @@
 *.la
 *.a
 
+# Shared libraries
+*.so
+*.so.*
+
 # Executables
 *.exe
 
@@ -41,3 +45,9 @@ site
 components/eamxx/site/*
 # Ignore auto-generated eamxx_params.md file
 components/eamxx/docs/common/eamxx_params.md
+
+# Python packaging
+*.egg-info
+components/eamxx/src/python/build
+components/eamxx/src/python/build_src
+components/eamxx/src/python/dist

cime_config/machines/config_machines.xml

@@ -277,7 +277,6 @@
       <env name="BLA_VENDOR">Generic</env>
       <env name="Albany_ROOT">$SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/albany-e3sm-serial-release-gcc-cmake-fix; else echo "$Albany_ROOT"; fi}</env>
       <env name="Trilinos_ROOT">$SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/trilinos-e3sm-serial-release-gcc; else echo "$Trilinos_ROOT"; fi}</env>
-      <env name="Kokkos_ROOT">$SHELL{if [ -z "$Kokkos_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/trilinos-e3sm-serial-release-gcc; else echo "$Kokkos_ROOT"; fi}</env>
     </environment_variables>
     <environment_variables compiler="nvidia" mpilib="mpich">
       <env name="ADIOS2_ROOT">$SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /global/cfs/cdirs/e3sm/3rdparty/adios2/2.9.1/cray-mpich-8.1.25/nvidia-22.7; else echo "$ADIOS2_ROOT"; fi}</env>
@@ -1444,7 +1443,7 @@
         <command name="reset"></command>
         <command name="load">PrgEnv-cray</command>
         <command name="load">craype-accel-amd-gfx90a</command>
-        <command name="load">rocm/5.1.0</command>
+        <command name="load">rocm/5.4.0</command>
         <command name="load">libunwind/1.6.2</command>
       </modules>
       <modules>
@@ -1483,6 +1482,10 @@
       <env name="OMP_PROC_BIND">spread</env>
       <env name="OMP_PLACES">threads</env>
     </environment_variables>
+
+    <environment_variables compiler="crayclang-scream" mpilib="mpich">
+      <env name="ADIOS2_ROOT">$SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.9.1/cray-mpich-8.1.26/crayclang-scream-14.0.0; else echo "$ADIOS2_ROOT"; fi}</env>
+    </environment_variables>
   </machine>
 
   <!-- Skylake nodes of Stampede2 at TACC -->
@@ -2656,7 +2659,6 @@
     <environment_variables compiler="gnu">
       <env name="Albany_ROOT">$SHELL{if [ -z "$Albany_ROOT" ]; then echo /lcrc/group/e3sm/ac.jwatkins/LandIce/AlbanyBuilds/build-gcc-sfad12-e3sm/install; else echo "$Albany_ROOT"; fi}</env>
       <env name="Trilinos_ROOT">$SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /lcrc/group/e3sm/ac.jwatkins/LandIce/TrilinosBuilds/build-gcc-e3sm/install; else echo "$Trilinos_ROOT"; fi}</env>
-      <env name="Kokkos_ROOT">$SHELL{if [ -z "$Kokkos_ROOT" ]; then echo /lcrc/group/e3sm/ac.jwatkins/LandIce/TrilinosBuilds/build-gcc-e3sm/install; else echo "$Kokkos_ROOT"; fi}</env>
     </environment_variables>
   </machine>
 

cime_config/machines/config_workflow.xml

@@ -37,7 +37,7 @@
       <template>template.case.test</template>
       <prereq>$BUILD_COMPLETE and $TEST</prereq>
     </job>
-    <job name="case.post_run_io">
+    <!--job name="case.post_run_io">
       <template>template.post_run_io</template>
       <dependency>case.run</dependency>
       <prereq>case.get_value("PIO_TYPENAME_ATM") == 'adios' or \
@@ -53,11 +53,12 @@
       <runtime_parameters>
         <walltime>0:30:00</walltime>
       </runtime_parameters>
-    </job>
+    </job-->
     <job name="case.st_archive">
       <template>template.st_archive</template>
       <!-- If DOUT_S is true and case.run (or case.test) exits successfully then run st_archive-->
-      <dependency>(case.run and case.post_run_io) or case.test</dependency>
+      <!--dependency>(case.run and case.post_run_io) or case.test</dependency-->
+      <dependency>case.run or case.test</dependency>
       <prereq>$DOUT_S</prereq>
       <runtime_parameters>
         <task_count>1</task_count>

cime_config/tests.py

@@ -723,6 +723,7 @@
         "time"  : "01:00:00",
         "tests" : (
             "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-optics",
+            "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-aci",
         )
     },
 

components/cmake/build_model.cmake

@@ -137,6 +137,9 @@ macro(build_model COMP_CLASS COMP_NAME)
     # Add rrtmgp++ source code if asked for
     if (USE_RRTMGPXX)
       message(STATUS "Building RRTMGPXX")
+      # For now, hardcode YAKL mode in rrtmgp
+      set(RRTMGP_ENABLE_YAKL On)
+      add_definitions("-DRRTMGP_ENABLE_YAKL")
       # Build the static rrtmgpxx library
       set(RRTMGPXX_BIN ${CMAKE_CURRENT_BINARY_DIR}/rrtmgp)
       add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../../eam/src/physics/rrtmgp/external/cpp ${RRTMGPXX_BIN})

components/eam/src/dynamics/se/se_iop_intr_mod.F90

@@ -422,10 +422,6 @@ subroutine apply_iop_forcing(elem,hvcoord,hybrid,tl,n,t_before_advance,nets,nete
       call outfld('QDIFF',qdiff_dyn,plon,begchunk)
     endif
 
-    if (iop_coriolis) then
-      call iop_apply_coriolis(elem,t1,nelemd_todo,np_todo,dt)
-    endif
-
     call outfld('TOBS',tobs,plon,begchunk)
     call outfld('QOBS',qobs,plon,begchunk)
     call outfld('DIVQ',divq,plon,begchunk)
@@ -441,6 +437,10 @@ subroutine apply_iop_forcing(elem,hvcoord,hybrid,tl,n,t_before_advance,nets,nete
 
   enddo
 
+  if (iop_coriolis) then
+    call iop_apply_coriolis(elem,t1,nelemd_todo,np_todo,dt)
+  endif
+
   if ((iop_nudge_tq .or. iop_nudge_uv) .and. dp_crm) then
     ! If running in a doubly periodic CRM mode, then nudge the domain
     !   based on the domain mean and observed quantities of T, Q, u, and v

components/eam/src/physics/rrtmgp/cpp/rrtmgp_interface.cpp

@@ -49,7 +49,7 @@ GasOpticsRRTMGP k_dist_sw;
 GasOpticsRRTMGP k_dist_lw;
 
 // Vector of strings to hold active gas names. 
-string1d active_gases;
+string1dv active_gases;
 
 extern "C" void rrtmgp_initialize_cxx(int ngas, char *gas_names[], char const *coefficients_file_sw, char const *coefficients_file_lw) {
     // First, make sure yakl has been initialized
@@ -66,9 +66,9 @@ extern "C" void rrtmgp_initialize_cxx(int ngas, char *gas_names[], char const *c
     // impossible from this initialization routine because I do not think the
     // rad_cnst objects are setup yet.
     // the other tasks!
-    active_gases = string1d("active_gases", ngas);
+    active_gases = string1dv(ngas);
     for (int igas=0; igas<ngas; igas++) {
-        active_gases(igas+1) = gas_names[igas];
+        active_gases[igas] = gas_names[igas];
     }
     GasConcs available_gases;
     available_gases.init(active_gases, 1, 1);
@@ -236,11 +236,11 @@ extern "C" void rrtmgp_run_sw (
     gas_concs.init(active_gases, ncol, nlay);
     real2d tmp2d;
     tmp2d = real2d("tmp", ncol, nlay);
-    for (int igas = 1; igas <= ngas; igas++) {
+    for (int igas = 0; igas < ngas; igas++) {
         parallel_for(SimpleBounds<2>(nlay,ncol), YAKL_LAMBDA(int ilay, int icol) {
-            tmp2d(icol,ilay) = gas_vmr(igas,icol,ilay);
+            tmp2d(icol,ilay) = gas_vmr(igas+1,icol,ilay);
         });
-        gas_concs.set_vmr(active_gases(igas), tmp2d);
+        gas_concs.set_vmr(active_gases[igas], tmp2d);
     }
 
     // Do gas optics
@@ -460,11 +460,11 @@ extern "C" void rrtmgp_run_lw (
     gas_concs.init(active_gases, ncol, nlay);
     real2d tmp2d;
     tmp2d = real2d("tmp", ncol, nlay);
-    for (int igas = 1; igas <= ngas; igas++) {
+    for (int igas = 0; igas < ngas; igas++) {
         parallel_for(SimpleBounds<2>(nlay,ncol), YAKL_LAMBDA(int ilay, int icol) {
-            tmp2d(icol,ilay) = gas_vmr(igas,icol,ilay);
+            tmp2d(icol,ilay) = gas_vmr(igas+1,icol,ilay);
         });
-        gas_concs.set_vmr(active_gases(igas), tmp2d);
+        gas_concs.set_vmr(active_gases[igas], tmp2d);
     }
 
     //  Boundary conditions

components/eamxx/CMakeLists.txt

@@ -203,13 +203,24 @@ if (NOT SCREAM_SMALL_KERNELS)
   set(EKAT_DISABLE_WORKSPACE_SHARING TRUE CACHE STRING "")
 endif()
 
-# Add RRTMGP debug checks. Note, we might consider also adding RRTMGP_EXPENSIVE_CHECKS
+# Add RRTMGP settings. Note, we might consider also adding RRTMGP_EXPENSIVE_CHECKS
 # to turn on the RRTMGP internal checks here as well, via
 #     option (RRTMGP_EXPENSIVE_CHECKS "Turn on internal RRTMGP error checking" ${SCREAM_DEBUG})
 # and then adding to scream_config.h:
 #     #cmakedefine RRTMGP_EXPENSIVE_CHECKS
 option (SCREAM_RRTMGP_DEBUG "Turn on extra debug checks in RRTMGP" ${SCREAM_DEBUG})
 
+option(SCREAM_RRTMGP_ENABLE_YAKL "Use YAKL under rrtmgp" TRUE)
+option(SCREAM_RRTMGP_ENABLE_KOKKOS "Use Kokkos under rrtmgp" FALSE)
+if (SCREAM_RRTMGP_ENABLE_YAKL)
+  add_definitions("-DRRTMGP_ENABLE_YAKL")
+endif()
+
+if (SCREAM_RRTMGP_ENABLE_KOKKOS)
+  add_definitions("-DRRTMGP_ENABLE_KOKKOS")
+endif()
+
+
 set(SCREAM_DOUBLE_PRECISION TRUE CACHE BOOL "Set to double precision (default True)")
 
 # For now, only used in share/grid/remap/refining_remapper_rma.*pp
@@ -261,6 +272,12 @@ set(SCREAM_BASE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 set(SCREAM_SRC_DIR  ${CMAKE_CURRENT_SOURCE_DIR}/src)
 set(SCREAM_BIN_DIR  ${CMAKE_CURRENT_BINARY_DIR})
 
+option (EAMXX_ENABLE_PYBIND "Whether to enable python interface to eamxx, via pybind11" OFF)
+if (EAMXX_ENABLE_PYBIND)
+  # Pybind11 requires shared libraries
+  set (BUILD_SHARED_LIBS ON)
+endif()
+
 ####################################################################
 #                    Packs-related settings                        #
 ####################################################################
@@ -618,4 +635,3 @@ print_var(SCREAM_TEST_THREAD_INC)
 print_var(SCREAM_TEST_MAX_RANKS)
 
 message ("**************************************************")
-

components/eamxx/cime_config/namelist_defaults_scream.xml

@@ -234,6 +234,12 @@ be lost if SCREAM_HACK_XML is not enabled.
       <Ckm type="real" doc="Eddy diffusivity coefficient for momentum">0.1</Ckm>
     </shoc>
 
+    <!-- MAM4xx-ACI -->
+    <mam4_aci inherit="atm_proc_base">
+      <wsubmin type="real" doc="Minimum diagnostic sub-grid vertical velocity">0.001</wsubmin>
+      <top_level_mam4xx type="integer" doc="Top level for MAM4xx">6</top_level_mam4xx>
+    </mam4_aci>
+
     <!-- CLD fraction -->
     <cldFraction inherit="atm_proc_base"/>
 
@@ -481,6 +487,33 @@ be lost if SCREAM_HACK_XML is not enabled.
     <surf_sens_flux    >0.0</surf_sens_flux>
     <surf_lw_flux_up   >0.0</surf_lw_flux_up>
     <surf_mom_flux     type="array(real)">0.0,0.0</surf_mom_flux>
+    <!-- Note: MAM4xx-ACI needs dry aerosol diameter (dgnum), currently set to aitken model nominal diameter, remove it once all of MAM4xx is integrated -->
+    <dgnum type="real" doc="Dry aerosol particles diameter in meters">2.6e-08</dgnum>
+        <bc_c1 type="real">0.0</bc_c1>
+    <bc_c3 type="real">0.0</bc_c3>
+    <bc_c4 type="real">0.0</bc_c4>
+    <dst_c1 type="real">0.0</dst_c1>
+    <dst_c3 type="real">0.0</dst_c3>
+    <so4_c1 type="real">0.0</so4_c1>
+    <so4_c2 type="real">0.0</so4_c2>
+    <so4_c3 type="real">0.0</so4_c3>
+    <pom_c1 type="real">0.0</pom_c1>
+    <pom_c3 type="real">0.0</pom_c3>
+    <pom_c4 type="real">0.0</pom_c4>
+    <soa_c1 type="real">0.0</soa_c1>
+    <soa_c2 type="real">0.0</soa_c2>
+    <soa_c3 type="real">0.0</soa_c3>
+    <nacl_c1 type="real">0.0</nacl_c1>
+    <nacl_c2 type="real">0.0</nacl_c2>
+    <nacl_c3 type="real">0.0</nacl_c3>
+    <mom_c1 type="real">0.0</mom_c1>
+    <mom_c2 type="real">0.0</mom_c2>
+    <mom_c3 type="real">0.0</mom_c3>
+    <mom_c4 type="real">0.0</mom_c4>
+    <num_c1 type="real">0.0</num_c1>
+    <num_c2 type="real">0.0</num_c2>
+    <num_c3 type="real">0.0</num_c3>
+    <num_c4 type="real">0.0</num_c4>
     <!-- default ne1024 initial condition files do not have these, so init to zero here -->
     <!-- TODO: delete this once we can tell the AD that some fields can be inited by procs -->
     <qc hgrid="ne256np4|ne1024np4">0.0</qc>
@@ -503,6 +536,7 @@ be lost if SCREAM_HACK_XML is not enabled.
     <output_yaml_files type="array(string)"/>
     <model_restart>
       <filename_prefix>./${CASE}.scream</filename_prefix>
+      <iotype>default</iotype>
       <output_control locked="true">
         <Frequency>${REST_N}</Frequency>
         <frequency_units>${REST_OPTION}</frequency_units>
@@ -551,6 +585,12 @@ be lost if SCREAM_HACK_XML is not enabled.
     <iop_dosubsidence COMPSET=".*DYCOMSrf01">true</iop_dosubsidence>
     <iop_srf_prop type="logical" doc="Use IOP file specified surface values in surface coupling importer.">false</iop_srf_prop>
     <iop_srf_prop COMPSET=".*DYCOMSrf01">true</iop_srf_prop>
+    <iop_nudge_uv type="logical" doc="Nudge winds to those in the IOP file.">false</iop_nudge_uv>
+    <iop_nudge_tq type="logical" doc="Nudge thermodynamics to those in the IOP file.">false</iop_nudge_tq>
+    <iop_nudge_tq_low type="real" doc="Lowest layer to apply nudging for t and q (pressure in hPa).">1100</iop_nudge_tq_low>
+    <iop_nudge_tq_high type="real" doc="Highest layer to apply nudging for t and q (pressure in hPa).">0</iop_nudge_tq_high>
+    <iop_nudge_tscale type="real" doc="Time scale to nudge thermodynamics or winds to.">10800</iop_nudge_tscale>
+    <iop_coriolis type="logical" doc="Apply coriolis forcing to winds based on large scale winds in IOP file.">false</iop_coriolis>
   </iop_options>
 
   <!-- Homme control namelist -->

components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/aci/shell_commands

@@ -0,0 +1,23 @@
+
+#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer
+#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options
+./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41"
+
+#modify initial condition file to get aerosol species ICs
+$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b
+
+# Add spa as RRTMG needs spa
+$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,spa,rrtmgp" -b
+
+# Replace spa with mam4_aci to invoke mam4 aci scheme
+$CIMEROOT/../components/eamxx/scripts/atmchange mac_aero_mic::atm_procs_list="tms,shoc,cldFraction,mam4_aci,p3" -b
+
+#Set precribed ccn to false so that P3 uses input from ACI
+$CIMEROOT/../components/eamxx/scripts/atmchange p3::do_prescribed_ccn=false -b
+
+#Set predicted ccn to true so that P3 uses input from ACI
+$CIMEROOT/../components/eamxx/scripts/atmchange p3::do_predict_nc=true -b
+
+
+
+

components/eamxx/cmake/CompareNCFiles.cmake

@@ -13,19 +13,19 @@ function(CompareNCFiles)
   set (argsMv LABELS FIXTURES_REQUIRED)
 
   cmake_parse_arguments(PARSE "${options}" "${args1v}" "${argsMv}" ${ARGN})
-  CheckMacroArgs(CompareNCFilesFamily PARSE "${options}" "${args1v}" "${argsMv}")
+  CheckMacroArgs(CompareNCFiles PARSE "${options}" "${args1v}" "${argsMv}")
 
   # Sanity checks
   if (NOT PARSE_TEST_NAME)
-    message ("Error! CompareNCFilesPair requires the keyword argument TEST_NAME")
+    message ("Error! CompareNCFiles requires the keyword argument TEST_NAME")
     message (FATAL_ERROR "Aborting...")
   endif()
   if (NOT PARSE_SRC_FILE)
-    message ("Error! CompareNCFilesPair requires the keyword argument SRC_FILE")
+    message ("Error! CompareNCFiles requires the keyword argument SRC_FILE")
     message (FATAL_ERROR "Aborting...")
   endif()
   if (NOT PARSE_TGT_FILE)
-    message ("Error! CompareNCFilesPair requires the keyword argument TGT_FILE")
+    message ("Error! CompareNCFiles requires the keyword argument TGT_FILE")
     message (FATAL_ERROR "Aborting...")
   endif()
 

components/eamxx/cmake/Findmpi4py.cmake

@@ -0,0 +1,36 @@
+# - FindMPI4PY
+# Find mpi4py includes
+# This module defines:
+# MPI4PY_INCLUDE_DIR, where to find mpi4py.h, etc.
+# MPI4PY_FOUND
+
+function (SetMpi4pyIncludeDir)
+endfunction()
+
+if (NOT TARGET mpi4py)
+  # If user provided an include dir, we will use that, otherwise we'll ask python to find it
+  if (NOT MPI4PY_INCLUDE_DIR)
+    execute_process(COMMAND
+      "${PYTHON_EXECUTABLE}" "-c" "import mpi4py; print (mpi4py.get_include())"
+      OUTPUT_VARIABLE OUTPUT
+      RESULT_VARIABLE RESULT
+      OUTPUT_STRIP_TRAILING_WHITESPACE)
+    if (RESULT)
+      set(MPI4PY_FOUND FALSE)
+    else ()
+      set (MPI4PY_INCLUDE_DIR ${OUTPUT} CACHE PATH "Path to mpi4py include directory" FORCE)
+    endif()
+  endif()
+
+  # If we still don't have an include dir, it means we have no mpi4py installed
+  if (NOT MPI4PY_INCLUDE_DIR)
+    set(MPI4PY_FOUND FALSE)
+  else ()
+    add_library(mpi4py INTERFACE)
+    target_include_directories(mpi4py INTERFACE SYSTEM ${MPI4PY_INCLUDE_DIR})
+    set(MPI4PY_FOUND TRUE)
+  endif()
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(mpi4py DEFAULT_MSG MPI4PY_INCLUDE_DIR)