Merge pull request #109 from grantfirl/ufs-dev-PR98

UFS-dev PR#98
NCAR · Jan 24, 2024 · f3b9c2c · f3b9c2c
2 parents c96f7fe + 8132b91
commit f3b9c2c
Show file tree

Hide file tree

Showing 142 changed files with 2,839 additions and 6,482 deletions.
diff --git a/CICE-interface/CICE b/CICE-interface/CICE
diff --git a/CICE-interface/cice_files.cmake b/CICE-interface/cice_files.cmake
@@ -86,7 +86,6 @@ list(APPEND icepack_files
   CICE/icepack/columnphysics/icepack_wavefracspec.F90
   CICE/icepack/columnphysics/icepack_zbgc.F90
   CICE/icepack/columnphysics/icepack_zbgc_shared.F90
-  CICE/icepack/columnphysics/icepack_zsalinity.F90
 )
 
 list(APPEND cice_shared_files_c

diff --git a/CMEPS-interface/CMEPS b/CMEPS-interface/CMEPS
diff --git a/CMEPS-interface/CMakeLists.txt b/CMEPS-interface/CMakeLists.txt
@@ -31,6 +31,8 @@ list(APPEND _nems_util_files
   CMEPS/ufs/ufs_kind_mod.F90
   CMEPS/ufs/perf_mod.F90
   CMEPS/ufs/ufs_const_mod.F90
+  ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_orb_mod.F90
+  ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_const_mod.F90
   ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_abort_mod.F90
   ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_strconvert_mod.F90
   ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_log_mod.F90

diff --git a/FV3 b/FV3
diff --git a/MOM6-interface/MOM6 b/MOM6-interface/MOM6
diff --git a/doc/UsersGuide/build/.gitignore b/doc/UsersGuide/build/.gitignore
diff --git a/doc/UsersGuide/source/Configurations.rst b/doc/UsersGuide/source/Configurations.rst
diff --git a/doc/UsersGuide/source/Glossary.rst b/doc/UsersGuide/source/Glossary.rst
@@ -54,6 +54,9 @@ Glossary
    cron table
       Cron is a job scheduler accessed through the command-line on UNIX-like operating systems. It is useful for automating tasks such as regression testing. Cron periodically checks a cron table (aka crontab) to see if any tasks are are ready to execute. If so, it runs them. 
 
+   data assimilation
+      Data assimilation is the process of combining observations, model data, and error statistics to achieve the best estimate of the state of a system. One of the major sources of error in weather and climate forecasts is uncertainty related to the initial conditions that are used to generate future predictions. Even the most precise instruments have a small range of unavoidable measurement error, which means that tiny measurement errors (e.g., related to atmospheric conditions and instrument location) can compound over time. These small differences result in very similar forecasts in the short term (i.e., minutes, hours), but they cause widely divergent forecasts in the long term. Errors in weather and climate forecasts can also arise because models are imperfect representations of reality. Data assimilation systems seek to mitigate these problems by combining the most timely observational data with a "first guess" of the atmospheric state (usually a previous forecast) and other sources of data to provide a "best guess" analysis of the atmospheric state to start a weather or climate simulation. When combined with an "ensemble" of model runs (many forecasts with slightly different conditions), data assimilation helps predict a range of possible atmospheric states, giving an overall measure of uncertainty in a given forecast.
+
    DATM
       DATM is the *Data Atmosphere* component of :term:`CDEPS`. It uses static atmospheric forcing files (derived from observations or previous atmospheric model runs) instead of output from an active atmospheric model. This reduces the complexity and computational cost associated with coupling to an active atmospheric model. The *Data Atmosphere* component is particularly useful when employing computationally intensive Data Assimilation (DA) techniques to update ocean and/or sea ice fields in a coupled model. In general, use of DATM in place of :term:`ATM` can be appropriate when users are running a coupled model and only want certain components of the model to be active. More information about DATM is available in the `CDEPS Documentation <https://escomp.github.io/CDEPS/versions/master/html/datm.html>`__.
 

diff --git a/modulefiles/ufs_acorn.intel.lua b/modulefiles/ufs_acorn.intel.lua
@@ -2,30 +2,20 @@ help([[
 Load environment to build UFS on Acorn with Intel compiler
 ]])
 
-PrgEnv_intel_ver=os.getenv("PrgEnv_intel_ver") or "8.1.0"
-load(pathJoin("PrgEnv-intel", PrgEnv_intel_ver))
+prepend_path("MODULEPATH", "/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.4.1/envs/ufs-pio-2.5.10/install/modulefiles/Core")
 
-intel_ver=os.getenv("intel_ver") or "19.1.3.304"
-load(pathJoin("intel", intel_ver))
+load("stack-intel")
+load("stack-cray-mpich")
+load("stack-python")
 
-craype_ver=os.getenv("craype_ver") or "2.7.13"
-load(pathJoin("craype", craype_ver))
-
-cray_mpich_ver=os.getenv("cray_mpich_ver") or "8.1.9"
-load(pathJoin("cray-mpich", cray_mpich_ver))
-
-cmake_ver=os.getenv("cmake_ver") or "3.20.2"
-load(pathJoin("cmake", cmake_ver))
-
-prepend_path("MODULEPATH", "/lfs/h1/emc/nceplibs/noscrub/hpc-stack/libs/hpc-stack/modulefiles/stack")
-
-hpc_ver=os.getenv("hpc_ver") or "1.2.0"
-hpc_intel_ver=os.getenv("hpc_intel_ver") or "19.1.3.304"
-hpc_cray_mpich_ver=os.getenv("hpc_cray_mpich_ver") or "8.1.9"
-load(pathJoin("hpc", hpc_ver))
-load(pathJoin("hpc-intel", hpc_intel_ver))
-load(pathJoin("hpc-cray-mpich", hpc_cray_mpich_ver))
+--Avoid production installations; use spack-stack only:
+remove_path("MODULEPATH", "/apps/ops/prod/libs/modulefiles/compiler/intel/19.1.3.304")
+remove_path("MODULEPATH", "/apps/ops/prod/libs/modulefiles/mpi/intel/19.1.3.304/cray-mpich/8.1.4")
+remove_path("MODULEPATH", "/apps/ops/prod/libs/modulefiles/mpi/intel/19.1.3.304/cray-mpich/8.1.7")
+remove_path("MODULEPATH", "/apps/prod/lmodules/intel/19.1.3.304")
+remove_path("MODULEPATH", "/apps/prod/lmodules/INTEL_cray_mpich/19.1.3.304/cray-mpich/8.1.4")
 
+load("cmake")
 load("ufs_common")
 
 prepend_path("MODULEPATH", "/lfs/h1/emc/nceplibs/noscrub/UPP_IFI/modulefiles")

diff --git a/modulefiles/ufs_cheyenne.gnu.lua b/modulefiles/ufs_cheyenne.gnu.lua
@@ -33,10 +33,24 @@ load(pathJoin("hpc-gnu", hpc_gnu_ver))
 hpc_mpt_ver=os.getenv("hpc_mpt_ver") or "2.25"
 load(pathJoin("hpc-mpt", hpc_mpt_ver))
 
-scotch_ver=os.getenv("scotch_ver") or "7.0.3"
-load(pathJoin("scotch", scotch_ver))
+load("jasper/2.0.25")
+load("zlib/1.2.11")
+load("hdf5/1.14.0")
+load("netcdf/4.9.2")
+load("pio/2.5.10")
+load("esmf/8.4.2")
+load("fms/2023.01")
+load("bacio/2.4.1")
+load("crtm/2.4.0")
+load("g2/3.4.5")
+load("g2tmpl/1.10.2")
+load("ip/3.3.3")
+load("sp/2.3.3")
+load("w3emc/2.9.2")
+load("gftl-shared/v1.5.0")
+load("mapl/2.35.2-esmf-8.4.2")
+load("scotch/7.0.3")
 
-load("ufs_common")
 
 setenv("CC", "mpicc")
 setenv("CXX", "mpicxx")

diff --git a/modulefiles/ufs_cheyenne.intel.lua b/modulefiles/ufs_cheyenne.intel.lua
@@ -34,10 +34,24 @@ load(pathJoin("hpc-intel", hpc_intel_ver))
 hpc_mpt_ver=os.getenv("hpc_mpt_ver") or "2.25"
 load(pathJoin("hpc-mpt", hpc_mpt_ver))
 
-scotch_ver=os.getenv("scotch_ver") or "7.0.3"
-load(pathJoin("scotch", scotch_ver))
+load("jasper/2.0.25")
+load("zlib/1.2.11")
+load("hdf5/1.14.0")
+load("netcdf/4.9.2")
+load("pio/2.5.10")
+load("esmf/8.4.2")
+load("fms/2023.01")
+load("bacio/2.4.1")
+load("crtm/2.4.0")
+load("g2/3.4.5")
+load("g2tmpl/1.10.2")
+load("ip/3.3.3")
+load("sp/2.3.3")
+load("w3emc/2.9.2")
+load("gftl-shared/v1.5.0")
+load("mapl/2.35.2-esmf-8.4.2")
+load("scotch/7.0.3")
 
-load("ufs_common")
 
 setenv("CC", "mpicc")
 setenv("CXX", "mpicxx")

diff --git a/modulefiles/ufs_common.lua b/modulefiles/ufs_common.lua
@@ -3,23 +3,25 @@ whatis("Description: UFS build environment common libraries")
 help([[Load UFS Model common libraries]])
 
 local ufs_modules = {
-  {["jasper"]      = "2.0.25"},
-  {["zlib"]        = "1.2.11"},
-  {["libpng"]      = "1.6.37"},
-  {["hdf5"]        = "1.14.0"},
-  {["netcdf"]      = "4.9.2"},
-  {["pio"]         = "2.5.10"},
-  {["esmf"]        = "8.4.2"},
-  {["fms"]         = "2023.01"},
-  {["bacio"]       = "2.4.1"},
-  {["crtm"]        = "2.4.0"},
-  {["g2"]          = "3.4.5"},
-  {["g2tmpl"]      = "1.10.2"},
-  {["ip"]          = "3.3.3"},
-  {["sp"]          = "2.3.3"},
-  {["w3emc"]       = "2.9.2"},
-  {["gftl-shared"] = "v1.5.0"},
-  {["mapl"]        = "2.35.2-esmf-8.4.2"},
+  {["jasper"]          = "2.0.32"},
+  {["zlib"]            = "1.2.13"},
+  {["libpng"]          = "1.6.37"},
+  {["hdf5"]            = "1.14.0"},
+  {["netcdf-c"]        = "4.9.2"},
+  {["netcdf-fortran"]  = "4.6.0"},
+  {["parallelio"]      = "2.5.10"},
+  {["esmf"]            = "8.4.2"},
+  {["fms"]             = "2023.01"},
+  {["bacio"]           = "2.4.1"},
+  {["crtm"]            = "2.4.0"},
+  {["g2"]              = "3.4.5"},
+  {["g2tmpl"]          = "1.10.2"},
+  {["ip"]              = "3.3.3"},
+  {["sp"]              = "2.3.3"},
+  {["w3emc"]           = "2.9.2"},
+  {["gftl-shared"]     = "1.5.0"},
+  {["mapl"]            = "2.35.2-esmf-8.4.2"},
+  {["scotch"]          = "7.0.3"},
 }
 
 for i = 1, #ufs_modules do

diff --git a/modulefiles/ufs_common_spack.lua b/modulefiles/ufs_common_spack.lua
diff --git a/modulefiles/ufs_gaea.intel.lua b/modulefiles/ufs_gaea.intel.lua
@@ -5,23 +5,32 @@ help([[
 
 whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea ]===])
 
-load_any(pathJoin("cmake", os.getenv("cmake_ver") or "3.20.1"),"cmake")
+unload("intel")
+unload("cray-mpich")
+unload("cray-python")
+unload("darshan")
 
-prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2_ncdf492/modulefiles/stack")
-load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0"))
+prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/spack-stack-1.4.1-c4/envs/ufs-pio-2.5.10/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/lustre/f2/pdata/esrl/gsd/spack-stack/modulefiles")
 
-load(pathJoin("intel-classic", os.getenv("intel_classic_ver") or "2022.0.2"))
-load(pathJoin("cray-mpich", os.getenv("cray_mpich_ver") or "7.7.20"))
-load(pathJoin("hpc-intel-classic", os.getenv("hpc_intel_classic_ver") or "2022.0.2"))
-load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "7.7.20"))
-load(pathJoin("libpng", os.getenv("libpng_ver") or "1.6.37"))
+stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2"
+load(pathJoin("stack-intel", stack_intel_ver))
+
+stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "7.7.20"
+load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
+
+stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
+load(pathJoin("stack-python", stack_python_ver))
+
+cmake_ver=os.getenv("cmake_ver") or "3.23.1"
+load(pathJoin("cmake", cmake_ver))
 
 -- Needed at runtime:
 load("alps")
 
 load("ufs_common")
 
 setenv("CC","cc")
-setenv("FC","ftn")
 setenv("CXX","CC")
+setenv("FC","ftn")
 setenv("CMAKE_Platform","gaea.intel")
diff --git a/modulefiles/ufs_hera.gnu.lua b/modulefiles/ufs_hera.gnu.lua
@@ -2,33 +2,25 @@ help([[
 loads UFS Model prerequisites for Hera/GNU
 ]])
 
-prepend_path("MODULEPATH", "/contrib/sutils/modulefiles")
-load("sutils")
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.4.1/envs/ufs-pio-2.5.10/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles")
 
-cmake_ver=os.getenv("cmake_ver") or "3.20.1"
-load(pathJoin("cmake", cmake_ver))
-
-gnu_ver=os.getenv("gnu_ver") or "9.2.0"
-load(pathJoin("gnu", gnu_ver))
-
-prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/gnu-9.2_ncdf492/modulefiles/stack")
+stack_gnu_ver=os.getenv("stack_gnu_ver") or "9.2.0"
+load(pathJoin("stack-gcc", stack_gnu_ver))
 
-hpc_ver=os.getenv("hpc_ver") or "1.2.0"
-load(pathJoin("hpc", hpc_ver))
+stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.5"
+load(pathJoin("stack-openmpi", stack_openmpi_ver))
 
-hpc_gnu_ver=os.getenv("hpc_gnu_ver") or "9.2"
-load(pathJoin("hpc-gnu", hpc_gnu_ver))
+stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
+load(pathJoin("stack-python", stack_python_ver))
 
-hpc_mpich_ver=os.getenv("hpc_mpich_ver") or "3.3.2"
-load(pathJoin("hpc-mpich", hpc_mpich_ver))
-
-scotch_ver=os.getenv("scotch_ver") or "7.0.3"
-load(pathJoin("scotch", scotch_ver))
+cmake_ver=os.getenv("cmake_ver") or "3.23.1"
+load(pathJoin("cmake", cmake_ver))
 
 load("ufs_common")
 
 setenv("CC", "mpicc")
-setenv("CXX", "mpicxx")
+setenv("CXX", "mpic++")
 setenv("FC", "mpif90")
 setenv("CMAKE_Platform", "hera.gnu")
 

diff --git a/modulefiles/ufs_hera.intel.lua b/modulefiles/ufs_hera.intel.lua
@@ -2,31 +2,20 @@ help([[
 loads UFS Model prerequisites for Hera/Intel
 ]])
 
-prepend_path("MODULEPATH", "/contrib/sutils/modulefiles")
-load("sutils")
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.4.1/envs/ufs-pio-2.5.10/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles")
 
-cmake_ver=os.getenv("cmake_ver") or "3.20.1"
-load(pathJoin("cmake", cmake_ver))
-
-intel_ver=os.getenv("intel_ver") or "2022.1.2"
-load(pathJoin("intel", intel_ver))
-
-impi_ver=os.getenv("impi_ver") or "2022.1.2"
-load(pathJoin("impi", impi_ver))
-
-prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2_ncdf492/modulefiles/stack")
+stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
+load(pathJoin("stack-intel", stack_intel_ver))
 
-hpc_ver=os.getenv("hpc_ver") or "1.2.0"
-load(pathJoin("hpc", hpc_ver))
+stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1"
+load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
 
-hpc_intel_ver=os.getenv("hpc_intel_ver") or "2022.1.2"
-load(pathJoin("hpc-intel", hpc_intel_ver))
+stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
+load(pathJoin("stack-python", stack_python_ver))
 
-hpc_impi_ver=os.getenv("hpc_impi_ver") or "2022.1.2"
-load(pathJoin("hpc-impi", hpc_impi_ver))
-
-scotch_ver=os.getenv("scotch_ver") or "7.0.3"
-load(pathJoin("scotch", scotch_ver))
+cmake_ver=os.getenv("cmake_ver") or "3.23.1"
+load(pathJoin("cmake", cmake_ver))
 
 load("ufs_common")
 
@@ -35,7 +24,4 @@ setenv("CXX", "mpiicpc")
 setenv("FC", "mpiifort")
 setenv("CMAKE_Platform", "hera.intel")
 
-prepend_path("MODULEPATH", "/scratch2/BMC/ifi/modulefiles")
-try_load("ifi/20230118-intel-2022.1.2")
-
 whatis("Description: UFS build environment")
+35 −8		.github/workflows/test-cice.yml
+3 −222		cicecore/cicedyn/analysis/ice_diagnostics_bgc.F90
+2 −2		cicecore/cicedyn/analysis/ice_history.F90
+17 −113		cicecore/cicedyn/analysis/ice_history_bgc.F90
+2 −6		cicecore/cicedyn/dynamics/ice_transport_driver.F90
+0 −10		cicecore/cicedyn/general/ice_flux.F90
+0 −6		cicecore/cicedyn/general/ice_flux_bgc.F90
+1 −1		cicecore/cicedyn/general/ice_init.F90
+15 −23		cicecore/cicedyn/general/ice_step_mod.F90
+1 −0		cicecore/cicedyn/infrastructure/ice_read_write.F90
+5 −15		cicecore/cicedyn/infrastructure/io/io_binary/ice_restart.F90
+2 −13		cicecore/cicedyn/infrastructure/io/io_netcdf/ice_restart.F90
+2 −12		cicecore/cicedyn/infrastructure/io/io_pio2/ice_restart.F90
+4 −7		cicecore/drivers/nuopc/cmeps/CICE_InitMod.F90
+23 −16		cicecore/drivers/nuopc/cmeps/CICE_RunMod.F90
+76 −8		cicecore/drivers/nuopc/cmeps/cice_wrapper_mod.F90
+24 −3		cicecore/drivers/nuopc/cmeps/ice_comp_nuopc.F90
+4 −6		cicecore/drivers/standalone/cice/CICE_InitMod.F90
+7 −12		cicecore/drivers/standalone/cice/CICE_RunMod.F90
+59 −16		cicecore/drivers/unittest/gridavgchk/CICE_InitMod.F90
+59 −16		cicecore/drivers/unittest/halochk/CICE_InitMod.F90
+4 −6		cicecore/drivers/unittest/opticep/CICE_InitMod.F90
+7 −12		cicecore/drivers/unittest/opticep/CICE_RunMod.F90
+21 −123		cicecore/drivers/unittest/opticep/ice_init_column.F90
+12 −20		cicecore/drivers/unittest/opticep/ice_step_mod.F90
+59 −17		cicecore/drivers/unittest/sumchk/CICE_InitMod.F90
+17 −58		cicecore/shared/ice_arrays_column.F90
+19 −119		cicecore/shared/ice_init_column.F90
+8 −86		cicecore/shared/ice_restart_column.F90
+4 −2		configuration/scripts/cice.batch.csh
+2 −2		configuration/scripts/cice.launch.csh
+0 −56		configuration/scripts/machines/Macros.badger_intel
+58 −0		configuration/scripts/machines/Macros.chicoma_intel
+6 −4		configuration/scripts/machines/Macros.conda_macos
+0 −47		configuration/scripts/machines/env.badger_intel
+71 −0		configuration/scripts/machines/env.chicoma_intel
+0 −8		configuration/scripts/options/set_nml.zsal
+2 −2		doc/source/user_guide/ug_case_settings.rst
+1 −1		icepack
+6 −5		.github/workflows/srt.yml
+40 −2		cime_config/config_component.xml
+1 −1		cime_config/namelist_definition_drv.xml
+2 −2		mediator/esmFldsExchange_cesm_mod.F90
+18 −2		mediator/esmFldsExchange_nems_mod.F90
+6 −7		mediator/med.F90
+0 −1		mediator/med_internalstate_mod.F90
+3 −3		mediator/med_io_mod.F90
+12 −10		mediator/med_map_mod.F90
+5 −2		mediator/med_methods_mod.F90
+1 −1		mediator/med_phases_history_mod.F90
+119 −50		mediator/med_phases_ocnalb_mod.F90
+6 −86		mediator/med_phases_prep_ocn_mod.F90
+1 −1		atmos_cubed_sphere
+1 −1		atmos_model.F90
+11 −2		ccpp/data/GFS_typedefs.F90
+21 −1		ccpp/data/GFS_typedefs.meta
+13 −0		ccpp/driver/GFS_diagnostics.F90
+1 −1		ccpp/physics
+1 −1		ccpp/suites/suite_FV3_HRRR_c3.xml
+1 −1		ccpp/suites/suite_FV3_HRRR_gf.xml
+37 −4		fv3_cap.F90
+95 −29		io/fv3atm_clm_lake_io.F90
+9 −3		io/fv3atm_restart_io.F90
+16 −7		io/fv3atm_rrfs_sd_io.F90
+29 −13		io/fv3atm_sfc_io.F90
+6 −1		io/module_write_restart_netcdf.F90
+47 −12		io/module_wrt_grid_comp.F90
+19 −9		io/post_fv3.F90
+24 −2		stochastic_physics/stochastic_physics_wrapper.F90
+1 −1		upp