Merge branch 'ufs-community:develop' into RT_bmark_gefs

CDEPS-interface/cdeps_files.cmake

@@ -56,6 +56,7 @@ list(APPEND cdeps_datm_files
 list(APPEND cdeps_dice_files
   CDEPS/dice/dice_datamode_ssmi_mod.F90
   CDEPS/dice/dice_flux_atmice_mod.F90
+  CDEPS/dice/dice_datamode_cplhist_mod.F90
   CDEPS/dice/ice_comp_nuopc.F90
 )
 

CICE-interface/CMakeLists.txt

@@ -51,7 +51,8 @@ list(APPEND lib_src_files
   ${cice_shared_files_c}
   ${icepack_files}
   ${cice_mpi_comm_files}
-  ${cice_nuopc_cmeps_driver_files})
+  ${cice_nuopc_cmeps_driver_files}
+  ${cice_cdeps_inline_files})
 
 list(APPEND _cice_defs FORTRANUNDERSCORE
                        coupled)
@@ -84,6 +85,11 @@ if(OpenMP_Fortran_FOUND)
   target_link_libraries(cice PRIVATE OpenMP::OpenMP_Fortran)
 endif()
 
+# ice prescribed
+add_dependencies(cice cdeps::cdeps)
+target_compile_definitions(cice PUBLIC "DISABLE_FoX")
+target_link_libraries(cice PUBLIC cdeps::cdeps)
+
 ###############################################################################
 ### Install
 ###############################################################################

CICE-interface/cice_files.cmake

@@ -150,3 +150,22 @@ list(APPEND cice_nuopc_cmeps_driver_files
   CICE/cicecore/drivers/nuopc/cmeps/ice_shr_methods.F90
   CICE/cicecore/drivers/nuopc/cmeps/ice_mesh_mod.F90
 )
+
+#-- Using ice prescribed ifndef cesmcoupled
+list(APPEND cice_cdeps_inline_files
+  ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_orb_mod.F90
+  ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_const_mod.F90
+  ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_abort_mod.F90
+  ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_strconvert_mod.F90
+  ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_log_mod.F90
+  ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_sys_mod.F90
+  ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_kind_mod.F90
+  ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_nl_mod.F90
+  ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_cal_mod.F90
+  ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_string_mod.F90
+  ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_timer_mod.F90
+  ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/streams/dshr_strdata_mod.F90
+  ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/streams/dshr_stream_mod.F90
+  ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/streams/dshr_methods_mod.F90
+  ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/dshr/dshr_mod.F90
+)

CMakeLists.txt

@@ -16,7 +16,7 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/CMakeModules/Modules)
 ###############################################################################
 
 # Valid applications and choices
-list(APPEND VALID_APPS ATM ATMAERO ATMAQ ATMW ATMWM ATML LND S2S S2SA S2SW S2SWA S2SWAL HAFS HAFSW HAFS-MOM6 HAFS-MOM6W HAFS-ALL NG-GODAS)
+list(APPEND VALID_APPS ATM ATMAERO ATMAQ ATMW ATMWM ATML LND S2S S2SA S2SW S2SWA S2SWAL ATM_DS2S ATM_DS2S-PCICE HAFS HAFSW HAFS-MOM6 HAFS-MOM6W HAFS-ALL NG-GODAS)
 set(APP NONE CACHE BOOL "Application Name")
 if(NOT (APP IN_LIST VALID_APPS))
   message(FATAL_ERROR "${APP} is not a valid application.\nValid Applications are: ${VALID_APPS}")
@@ -152,7 +152,7 @@ if(FMS)
   find_package(FMS 2022.04 REQUIRED COMPONENTS R4 R8)
   if(APP MATCHES "^(HAFSW)$")
     add_library(fms ALIAS FMS::fms_r4)
-  elseif (APP MATCHES "^(S2S|S2SA|S2SW|S2SWA|S2SWAL|NG-GODAS|HAFS-MOM6|HAFS-MOM6W)$")
+  elseif (APP MATCHES "^(S2S|S2SA|S2SW|S2SWA|S2SWAL|ATM_DS2S|ATM_DS2S-PCICE|NG-GODAS|HAFS-MOM6|HAFS-MOM6W)$")
     add_library(fms ALIAS FMS::fms_r8)
   endif()
   if(APP MATCHES "^(ATM|ATMAERO|ATMAQ|ATMWM|ATMW|ATML|HAFS|HAFS-ALL)$")
@@ -329,6 +329,7 @@ if(CDEPS)
   add_dependencies(ufs cdeps::cdeps)
   list(APPEND _ufs_defs_private FRONT_CDEPS_DATM=cdeps_datm_comp)
   list(APPEND _ufs_defs_private FRONT_CDEPS_DOCN=cdeps_docn_comp)
+  list(APPEND _ufs_defs_private FRONT_CDEPS_DICE=cdeps_dice_comp)
   target_link_libraries(ufs PUBLIC cdeps::cdeps)
 endif()
 

cmake/configure_apps.cmake

@@ -53,6 +53,7 @@ if(APP MATCHES "^(S2S|S2SA|S2SW|S2SWA|S2SWAL)$")
   set(FV3        ON  CACHE BOOL "Enable FV3"                 FORCE)
   set(MOM6       ON  CACHE BOOL "Enable MOM6"                FORCE)
   set(CICE6      ON  CACHE BOOL "Enable CICE6"               FORCE)
+  set(CDEPS      ON  CACHE BOOL "Enable CDEPS"               FORCE)
   set(STOCH_PHYS ON  CACHE BOOL "Enable Stochastic Physics"  FORCE)
   if(APP MATCHES "^(S2SW|S2SWA|S2SWAL)")
     set(WW3      ON  CACHE BOOL "Enable WAVEWATCH III"       FORCE)
@@ -68,6 +69,20 @@ if(APP MATCHES "^(S2S|S2SA|S2SW|S2SWA|S2SWAL)$")
   message("${APP_MSG} mode")
 endif()
 
+if(APP MATCHES "^(ATM_DS2S|ATM_DS2S-PCICE)$")
+  set(APP_MSG "Configuring UFS app in ATM_DS2S with CDEPS data")
+  set(CMEPS      ON  CACHE BOOL "Enable CMEPS"               FORCE)
+  set(FMS        ON  CACHE BOOL "Enable FMS"                 FORCE)
+  set(FV3        ON  CACHE BOOL "Enable FV3"                 FORCE)
+  set(STOCH_PHYS ON  CACHE BOOL "Enable Stochastic Physics"  FORCE)
+  set(CDEPS      ON  CACHE BOOL "Enable CDEPS"               FORCE)
+  if (APP MATCHES  "^(ATM_DS2S-PCICE)$")
+    set(CICE6      ON  CACHE BOOL "Enable CICE6"               FORCE)
+    string(CONCAT APP_MSG ${APP_MSG} " with prescribed ice sea ice")
+  endif()
+  message("${APP_MSG} mode")
+endif()
+
 if(APP MATCHES "^(HAFS|HAFSW|HAFS-MOM6|HAFS-MOM6W|HAFS-ALL)$")
   set(CMEPS      ON  CACHE BOOL "Enable CMEPS"               FORCE)
   set(CDEPS      ON  CACHE BOOL "Enable CDEPS"               FORCE)

cmake/configure_noaacloud.intel.cmake

@@ -0,0 +1 @@
+set(PARALLEL_NETCDF ON CACHE BOOL "Enable parallel NetCDF" FORCE)

doc/UsersGuide/source/CodeOverview.rst

@@ -108,7 +108,8 @@ The umbrella repository for the UFS WM is named ``ufs-weather-model``. Under thi
     ├── CDEPS-interface
     │     └── CDEPS
     │         ├── (datm)                         -------- CDEPS DATM
-    │         └── (docn)                         -------- CDEPS DOCN
+    │         ├── (docn)                         -------- CDEPS DOCN
+    │         └── (dice)                         -------- CDEPS DICE
     ├── CICE-interface
     │    └── CICE                                -------- CICE6 sea ice model
     │        ├── (icepack)                       -------- Sea ice column physics
@@ -161,4 +162,4 @@ The umbrella repository for the UFS WM is named ``ufs-weather-model``. Under thi
     │   └── fv3_conf   
     └── WW3
          └── (model)                             -------- WW3 model
-             └── (src)                           -------- NUOPC WW3 caps
+             └── (src)                           -------- NUOPC WW3 caps

driver/UFSDriver.F90

@@ -61,6 +61,9 @@ MODULE UFSDriver
 #endif
 #ifdef FRONT_CDEPS_DOCN
       use FRONT_CDEPS_DOCN, only: DOCN_SS  => SetServices
+#endif
+#ifdef FRONT_CDEPS_DICE
+      use FRONT_CDEPS_DICE, only: DICE_SS  => SetServices
 #endif
   ! - Handle build time ICE options:
 #ifdef FRONT_CICE6
@@ -410,6 +413,23 @@ subroutine SetModelServices(driver, rc)
             found_comp = .true.
           end if
 #endif
+#ifdef FRONT_CDEPS_DICE
+          if (trim(model) == "dice") then
+            !TODO: Remove bail code and pass info and SetVM to DriverAddComp
+            !TODO: once component supports threading.
+            if (ompNumThreads > 1) then
+              write (msg, *) "ESMF-aware threading NOT implemented for model: "//&
+                trim(model)
+              call ESMF_LogSetError(ESMF_RC_NOT_VALID, msg=msg,line=__LINE__, &
+                file=__FILE__, rcToReturn=rc)
+              return  ! bail out
+            endif
+            call NUOPC_DriverAddComp(driver, trim(prefix), DICE_SS, &
+              petList=petList, comp=comp, rc=rc)
+            if (ChkErr(rc,__LINE__,u_FILE_u)) return
+            found_comp = .true.
+          end if
+#endif
 #ifdef FRONT_CICE6
           if (trim(model) == "cice6") then
             call NUOPC_DriverAddComp(driver, trim(prefix), CICE6_SS, &

modulefiles/ufs_gaea.intel.lua

@@ -5,13 +5,7 @@ help([[
 
 whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea ]===])
 
-load("PrgEnv-intel/8.3.3")
-load("intel-classic/2023.1.0")
-load("cray-mpich/8.1.25")
-load("python/3.9.12")
-
 prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")
-prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/modulefiles")
 
 stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0"
 load(pathJoin("stack-intel", stack_intel_ver))
@@ -22,8 +16,13 @@ load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
 stack_python_ver=os.getenv("stack_python_ver") or "3.10.13"
 load(pathJoin("stack-python", stack_python_ver))
 
+cmake_ver=os.getenv("cmake_ver") or "3.23.1"
+load(pathJoin("cmake", cmake_ver))
+
 load("ufs_common")
-load("nccmp/1.9.0.1")
+
+nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
+load(pathJoin("nccmp", nccmp_ver))
 
 unload("darshan-runtime")
 unload("cray-libsci")

tests/bl_date.conf

@@ -1 +1 @@
-export BL_DATE=20240603
+export BL_DATE=20240607

tests/default_vars.sh

@@ -419,6 +419,7 @@ export CMP_DATAONLY=false
 # Defaults for ufs.configure
 export esmf_logkind="ESMF_LOGKIND_MULTI"
 export DumpFields="false"
+export MED_history_n=1000000
 
 export_fv3_v16 ()
 {
@@ -498,9 +499,12 @@ export HAFS=false
 export AQM=false
 export DATM_CDEPS=false
 export DOCN_CDEPS=false
+export DICE_CDEPS=false
+export CICE_PRESCRIBED=false
 export CDEPS_INLINE=false
 export POSTAPP='global'
 export USE_MERRA2=.true.
+export NESTED=.false.
 
 export NTILES=6
 export INPES=${INPES_dflt}
@@ -904,6 +908,7 @@ export_cice6() {
   export DT_CICE=${DT_ATMOS}
   export CICE_NPT=999
   export CICE_RUNTYPE=initial
+  export CICE_ICE_IC='cice_model.res.nc'
   export CICE_RUNID=unknown
   export CICE_USE_RESTART_TIME=.false.
   export CICE_RESTART_DIR=./RESTART/
@@ -954,6 +959,12 @@ export_cice6() {
   export CICE_BLCKX
   export CICE_BLCKY
   export CICE_DECOMP=slenderX2
+
+  #ds2s
+  export MESH_DICE=none
+  export stream_files_dice=none
+  export CICE_PRESCRIBED=false
+  export DICE_CDEPS=false
 }
 
 # Defaults for the MOM6 model namelist, mx100
@@ -1056,6 +1067,8 @@ export HAFS=false
 export AQM=false
 export DATM_CDEPS=false
 export DOCN_CDEPS=false
+export DICE_CDEPS=false
+export CICE_PRESCRIBED=false
 export CDEPS_INLINE=false
 export FV3BMIC='p8c'
 export BMIC=.false.
@@ -1222,6 +1235,10 @@ export AOD_FRQ=060000
 export RESTART_FILE_PREFIX=''
 export RESTART_FILE_SUFFIX_SECS=''
 export RT35D=''
+
+#CDEPS ds2s
+export MESH_DICE=none
+export stream_files_dice=none
 }
 export_35d_run ()
 {

tests/fv3_conf/control_run.IN

@@ -10,14 +10,17 @@ elif [ $NPX = 385 ]; then
 elif [ $NPX = 769 ]; then
   inputdir=FV3_input_data768
 fi
-echo "inputdir=$inputdir,NPX=$NPX"
+
+echo "inputdir=$inputdir,NPX=$NPX,NESTED=$NESTED"
 
 OPNREQ_TEST=${OPNREQ_TEST:-false}
 V2_SFC_FILE=${V2_SFC_FILE:-false}
 SUFFIX=${RT_SUFFIX}
 if [ $WARM_START = .false. ]; then
   mkdir INPUT RESTART
-  if [ "$V2_SFC_FILE" = "true" ]; then
+  if [ "$NESTED" = .true. ]; then
+    cp -r @[INPUTDATA_ROOT]/${inputdir}/INPUT_L127_nested/* ./INPUT/.
+  elif [ "$V2_SFC_FILE" = "true" ]; then
     cp -r @[INPUTDATA_ROOT]/${inputdir}/INPUT_L127_v2_sfc/* ./INPUT/.
   else
     cp -r @[INPUTDATA_ROOT]/${inputdir}/INPUT_L127/* ./INPUT/.

tests/fv3_conf/cpld_docn_dice.IN

@@ -0,0 +1,15 @@
+mkdir -p INPUT
+
+SUFFIX=${RT_SUFFIX}
+if [[ ${OPNREQ_TEST} == true ]]; then
+  SUFFIX=${BL_SUFFIX}
+fi
+
+#CDEPS data files
+#cp @[INPUTDATA_ROOT]/DOCN_DICE_cplhist/ufs.cpld.cpl*.nc INPUT
+module load nco
+ncrcat ../${DEP_RUN}${SUFFIX}/ufs.cpld.cpl.hi.ice*nc ./INPUT/ufs.cpld.cpl.hi.ice.nc
+ncrcat ../${DEP_RUN}${SUFFIX}/ufs.cpld.cpl.hi.ocn*nc ./INPUT/ufs.cpld.cpl.hi.ocn.nc
+
+#CDEPS fix files - same for ocn/ice
+cp @[INPUTDATA_ROOT]/CICE_FIX/@[OCNRES]/mesh.mx@[OCNRES].nc INPUT

tests/fv3_conf/cpld_docn_pcice.IN

@@ -0,0 +1,4 @@
+mkdir -p INPUT
+
+#For consistency, same CDEPS docn and CICE ice prescribed input files (data and ESMF_mesh)
+cp @[INPUTDATA_ROOT]/DOCN_DICE_ERA5/* INPUT