Merge remote-tracking branch 'remotes/upstream/develop' into feature/…

…hafs.v2.0.1

.github/PULL_REQUEST_TEMPLATE

@@ -20,7 +20,11 @@ If there are changes to the build or source code, the tests below must be conduc
 - [ ] Compile branch on Hera using GNU.
 - [ ] Compile branch in 'Debug' mode on WCOSS2.
 - [ ] Run unit tests locally on any Tier 1 machine.
-- [ ] Run relevant consistency tests locally on all Tier 1 machine.
+- [ ] Run relevant consistency tests locally on all Tier 1 machines.
+
+Optional test.
+
+- [ ] Run full set of chgres_cube consistency tests on Hera.
 
 Describe any additional tests performed.
 

CMakeLists.txt

@@ -23,12 +23,16 @@ option(GCYCLE "Enable building global_cycle.fd" ON)
 option(FRENCTOOLS "Enable building fre-nctools.fd" ON)
 option(GRIDTOOLS "Enable building grid_tools.fd" ON)
 option(CHGRES "Enable building chgres_cube.fd" ON)
+option(CHGRES_ALL "Build chgres with all input data options." OFF)
 option(OROG_MASK_TOOLS "Enable building orog_mask_tools.fd" ON)
+# OROG_MASK_TOOLS must be ON for OROG_NETCDF_TOOLS to build.
+option(OROG_NETCDF_TOOLS "Enable building orog_netcdf_tools.fd" OFF)
 option(SFC_CLIMO_GEN "Enable building sfc_climo_gen.fd" ON)
 option(VCOORD_GEN "Enable building vcoord_gen.fd" ON)
 option(FVCOMTOOLS "Enable building fvcom_tools.fd" ON)
 option(GBLEVENTS "Enable building gblevents.fd" ON)
 option(CPLD_GRIDGEN "Enable building cpld_gridgen.fd" ON)
+option(OCNICE_PREP "Enable building ocnice_prep.fd" ON)
 option(WEIGHT_GEN "Enable building weight_gen.fd" ON)
 option(OCEAN_MERGE "Enable building ocean_merge.fd" ON)
 
@@ -38,14 +42,14 @@ option(GFS "Enable building GFS-only utilities" OFF)
 # When building the GFS, the following need not be built
 if(GFS)
  message(STATUS "Building utilities specific to the GFS")
- set(FRENCTOOLS OFF CACHE BOOL "Disable building fre-nctools.fd" FORCE)
  set(GRIDTOOLS OFF CACHE BOOL "Disable building grid_tools.fd" FORCE)
  set(OROG_MASK_TOOLS OFF CACHE BOOL "Disable building orog_mask_tools.fd" FORCE)
  set(SFC_CLIMO_GEN OFF CACHE BOOL "Disable building sfc_climo_gen.fd" FORCE)
  set(VCOORD_GEN OFF CACHE BOOL "Disable building vcoord_gen.fd" FORCE)
  set(FVCOMTOOLS OFF CACHE BOOL "Disable building fvcom_tools.fd" FORCE)
  set(GBLEVENTS OFF CACHE BOOL "Disable building gblevents.fd" FORCE)
  set(CPLD_GRIDGEN OFF CACHE BOOL "Disable building cpld_gridgen.fd" FORCE)
+ set(OCNICE_PREP OFF CACHE BOOL "Disable building ocnice_prep.fd" FORCE)
  set(WEIGHT_GEN OFF CACHE BOOL "Disable building weight_gen.fd" FORCE)
  set(OCEAN_MERGE OFF CACHE BOOL "Disable building ocean_merge.fd" FORCE)
 endif()
@@ -97,15 +101,20 @@ if(OPENMP)
  find_package(OpenMP REQUIRED COMPONENTS Fortran)
 endif()
 
-find_package(sfcio 1.4.0 REQUIRED)
+if(CHGRES_ALL)
+ find_package(sfcio 1.4.0 REQUIRED)
+endif()
 find_package(w3emc 2.9.0 REQUIRED)
 find_package(bacio 2.4.0 REQUIRED)
-find_package(nemsio 2.5.0 REQUIRED)
-find_package(sigio 2.3.0 REQUIRED)
-find_package(sp 2.3.3 REQUIRED)
+if(CHGRES_ALL OR GBLEVENTS)
+ find_package(nemsio 2.5.0 REQUIRED)
+ find_package(sigio 2.3.0 REQUIRED)
+endif()
 find_package(ip 3.3.3 REQUIRED)
+if(ip_VERSION LESS 5.0)
+ find_package(sp 2.3.3 REQUIRED)
+endif()
 find_package(g2 3.4.3 REQUIRED)
-find_package(sigio 2.3.0 REQUIRED)
 
 # If doxygen documentation we enabled, build it. This must come before
 # adding the source code directories; the main documentation build

README.md

@@ -59,7 +59,7 @@ It also uses the following repositories:
 
 ## Installing
 
-On Orion, Hercules, Jet, Hera and WCOSS2 do the following:
+On Orion, Hercules, Jet, Hera, S4, Gaea and WCOSS2 do the following:
 
 1) Set the 'fixed' directories using the `link_fixdirs.sh`
 script in `./fix`. Usage: `./link_fixdirs.sh $RUN_ENVIR $machine`,

VERSION

@@ -1 +1 @@
-1.11.0
+1.13.0

build_all.sh

@@ -15,10 +15,11 @@ else
  readonly DIR_ROOT=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P)
 fi
 
+source "${DIR_ROOT}/sorc/machine-setup.sh"
+
 # User Options
 target=${target:-"NULL"}
 compiler=${compiler:-"intel"}
-PW_CSP=${PW_CSP:-} # TODO: This is an implementation from EPIC and consistent with the UFS WM build system.
 
 if [[ "$target" == "linux.*" || "$target" == "macosx.*" ]]; then
  unset -f module
@@ -27,17 +28,10 @@ if [[ "$target" == "linux.*" || "$target" == "macosx.*" ]]; then
  set -x
 else
  set +x
- source "${DIR_ROOT}/sorc/machine-setup.sh"
- if [[ "${target}" == "noaacloud" ]]; then
- #TODO: This will need to be revisited once the EPIC supported-stacks come online.
- #TODO: This is a hack due to how the spack-stack module files are generated; there may be a better way to do this.
- source /contrib/global-workflow/spack-stack/envs/spack_2021.0.3.env
- else
- module use "${DIR_ROOT}/modulefiles"
- fi
+ module use "${DIR_ROOT}/modulefiles"
  module load "build.$target.$compiler" > /dev/null
  module list
- set -x
+  set -x
 fi
 
 # Ensure the submodules have been initialized.

docs/main.md

@@ -9,6 +9,8 @@ https://github.com/ufs-community/UFS_UTILS.
 
 ## Documentation for Previous Versions of UFS_UTILS
 
+* [UFS_UTILS Version 1.13.0](ver-1.13.0/index.html)
+* [UFS_UTILS Version 1.12.0](ver-1.12.0/index.html)
 * [UFS_UTILS Version 1.11.0](ver-1.11.0/index.html)
 * [UFS_UTILS Version 1.10.0](ver-1.10.0/index.html)
 * [UFS_UTILS Version 1.9.0](ver-1.9.0/index.html)
@@ -24,8 +26,12 @@ https://github.com/ufs-community/UFS_UTILS.
 - <a href="chgres_cube/index.html">chgres_cube</a> - Creates cold
  start initial conditions for FV3 model runs.
 
-- <a href="cpld_gridgen/index.html">cpld_gridgen</a> - Utility to 
- create the Fix and IC files for the S2SW and S2S applications 
+- <a href="cpld_gridgen/index.html">cpld_gridgen</a> - Utility to
+ create the Fix and IC files for the S2SW and S2S applications
+
+- <a href="ocnice_prep/index.html">ocnice_prep</a> - Utility to
+ create the warmstart files for OCN and ICE from component restart
+ files at 1/4 deg tripole resolution
 
 - <a href="emcsfc_ice_blend/index.html">emcsfc_ice_blend</a> - Blends
  National Ice Center sea ice cover and EMC sea ice concentration data
@@ -48,7 +54,7 @@ https://github.com/ufs-community/UFS_UTILS.
 - <a href="gblevents/index.html">gblevents</a> -
  Prepares observational prepbufr reports for subsequent quality
  control and analysis programs.
- 
+
 - <a href="global_cycle/index.html">global_cycle</a> -
  Updates the GFS surface conditions using external snow and sea ice
  analyses. Updates monthly climatological fields such as plant
@@ -60,12 +66,12 @@ https://github.com/ufs-community/UFS_UTILS.
  gnomonic grids, and to compute the equivalent global resolution of a
  regional grid.
 
-- <a href="lsm_routines/index.html">lsm_routines</a> - Land surface 
+- <a href="lsm_routines/index.html">lsm_routines</a> - Land surface
  model-specific routines that are utilised elsewhere within UFS_UTILS.
- Currently, contains the routines required by global_cycle to 
+ Currently, contains the routines required by global_cycle to
  perform data assimilation updates to land model states
 
-- <a href="ocean_merge/index.html">ocean_merge</a> - Utility to 
+- <a href="ocean_merge/index.html">ocean_merge</a> - Utility to
  determine the water mask by merging a lake mask with a mapped
  ocean mask from MOM6.
 
@@ -82,6 +88,6 @@ https://github.com/ufs-community/UFS_UTILS.
  top and the number of vertical levels. Outputs the 'ak' and 'bk'
  parameters used by the forecast model to define the hybrid levels.
 
-- <a href="weight_gen/index.html">weight_gen</a> - Utility to 
+- <a href="weight_gen/index.html">weight_gen</a> - Utility to
  create gaussian grid ESMF 'scrip' files for use in creating
  ESMF interpolation weight files.

docs/source/chgres_cube.rst

@@ -3,7 +3,7 @@
 Introduction
 ------------
 
-The chgres_cube program creates initial condition files to “coldstart” the forecast model. The initial conditions are created from either Finite-Volume Sphere (FV3) Global Forecast System (GFS), North American Mesoscale Forecast System (NAM), Rapid Refresh (RAP), or High Resolution Rapid Refresh (HRRR) gridded binary version 2 (GRIB2) data.
+The chgres_cube program creates initial condition files to coldstart the forecast model. The initial conditions are created from either Finite-Volume Sphere (FV3) Global Forecast System (GFS), North American Mesoscale Forecast System (NAM), Rapid Refresh (RAP), or High Resolution Rapid Refresh (HRRR) gridded binary version 2 (GRIB2) data.
 
 Code structure
 --------------
@@ -55,7 +55,7 @@ Program inputs and outputs for global applications
 
 **Inputs**
 
-Users may create their own global grids, or use the pre-defined files are located `here <https://ftp.emc.ncep.noaa.gov/EIB/UFS/global/fix/fix_fv3_gmted2010.v20191213/>`_.
+Users may create their own global grids, or use the pre-defined files located in the `./CRES directories <https://noaa-nws-global-pds.s3.amazonaws.com/index.html#fix/orog/20231027/>`_. (where CRES is the atmospheric resolution and mxRES is the ocean resolution).
 
  * FV3 mosaic file - (NetCDF format)
  * CRES_mosaic.nc
@@ -69,22 +69,22 @@ Users may create their own global grids, or use the pre-defined files are locate
  * CRES_grid.tile6.nc
 
  * FV3 orography files - (NetCDF format)
- * CRES_oro_data.tile1.nc
- * CRES_oro_data.tile2.nc
- * CRES_oro_data.tile3.nc
- * CRES_oro_data.tile4.nc
- * CRES_oro_data.tile5.nc
- * CRES_oro_data.tile6.nc
-
- * FV3 surface climatological files - Located under the `./fix_sfc <https://ftp.emc.ncep.noaa.gov/EIB/UFS/global/fix/fix_fv3_gmted2010.v20191213/C48/fix_sfc>`_ sub-directory.  One file for each tile. NetCDF format.
- * CRES.facsf.tileX.nc (fractional coverage for strong/weak zenith angle dependent albedo)
- * CRES.maximum_snow_albedo.tileX.nc (maximum snow albedo)
- * CRES.slope_type.tileX.nc (slope type)
- * CRES.snowfree_albedo.tileX.nc (snow-free albedo)
- * CRES.soil_type.tileX.nc (soil type)
- * CRES.subtrate_temperature.tileX.nc (soil substrate temperature)
- * CRES.vegetation_greenness.tileX.nc (vegetation greenness)
- * CRES.vegetation_type.tileX.nc (vegetation type)
+ * CRES.mxRES_oro_data.tile1.nc
+ * CRES.mxRES_oro_data.tile2.nc
+ * CRES.mxRES_oro_data.tile3.nc
+ * CRES.mxRES_oro_data.tile4.nc
+ * CRES.mxRES_oro_data.tile5.nc
+ * CRES.mxRES_oro_data.tile6.nc
+
+ * FV3 surface climatological files - Located under the `./CRES/sfc <https://noaa-nws-global-pds.s3.amazonaws.com/index.html#fix/orog/20231027/>`_ subdirectories. One file for each tile. NetCDF format.
+ * CRES.mxRES.facsf.tileX.nc (fractional coverage for strong/weak zenith angle dependent albedo)
+ * CRES.mxRES.maximum_snow_albedo.tileX.nc (maximum snow albedo)
+ * CRES.mxRES.slope_type.tileX.nc (slope type)
+ * CRES.mxRES.snowfree_albedo.tileX.nc (snow-free albedo)
+ * CRES.mxRES.soil_type.tileX.nc (soil type)
+ * CRES.mxRES.subtrate_temperature.tileX.nc (soil substrate temperature)
+ * CRES.mxRES.vegetation_greenness.tileX.nc (vegetation greenness)
+ * CRES.mxRES.vegetation_type.tileX.nc (vegetation type)
 
  * FV3 vertical coordinate file. Text file. `Located here <https://noaa-ufs-srw-pds.s3.amazonaws.com/index.html#fix/fix_am/>`_.
  * global_hyblev.l$LEVS.txt
@@ -93,15 +93,15 @@ Users may create their own global grids, or use the pre-defined files are locate
 
 **Outputs**
 
- * Atmospheric “coldstart” files. NetCDF.
+ * Atmospheric coldstart files. NetCDF.
  * out.atm.tile1.nc
  * out.atm.tile2.nc
  * out.atm.tile3.nc
  * out.atm.tile4.nc
  * out.atm.tile5.nc
  * out.atm.tile6.nc
 
- * Surface/Near Sea Surface Temperature (NSST) “coldstart” files. NetCDF
+ * Surface/Near Sea Surface Temperature (NSST) coldstart files. NetCDF
  * out.sfc.tile1.nc
  * out.sfc.tile1.nc
  * out.sfc.tile1.nc
@@ -264,10 +264,10 @@ The following four sets of files/directories should all be located in the same d
 
 **Outputs**
 
- * Atmospheric “coldstart” file. NetCDF.
+ * Atmospheric coldstart file. NetCDF.
  * out.atm.tile7.nc
 
- * Surface “coldstart” file. NetCDF.
+ * Surface coldstart file. NetCDF.
  * out.sfc.tile7.nc
 
 Where to find FV3GFS, NAM, HRRR, and RAP GRIB2 data for regional applications
@@ -369,7 +369,7 @@ Namelist variables with “input” in their name refer to data input to chgres_
  * Set to 2 to create a boundary condition file. Use this option for all but the initialization time.
  * halo_blend - Integer number of row/columns to apply halo blending into the domain, where model and lateral boundary tendencies are applied.
  * halo_bndy - Integer number of rows/columns that exist within the halo, where pure lateral boundary conditions are applied.
- * external_model - Name of source model for input data. Valid options: 'GFS', 'NAM', 'RAP', 'HRRR'. (Default: 'GFS')
+ * external_model - Name of source model for input data. Valid options: 'GFS', 'NAM', 'RAP', 'HRRR', 'RRFS'. (Default: 'GFS')
 
 **Optional Entries**
 

docs/source/ufs_utils.rst

@@ -240,11 +240,11 @@ Program inputs and outputs
 
  * The "grid" files (CRES_grid.tile#.nc) containing the geo-reference records for the grid - (NetCDF). Created by the make_hgrid or regional_esg_grid programs.
  * Global 30-arc-second University of Maryland land cover data. Used to create the land-sea mask.
- * landcover30.fixed (unformatted binary). Located here `./fix/fix_orog <https://noaa-ufs-srw-pds.s3.amazonaws.com/index.html#fix/fix_orog/>`_.
+ * landcover.umd.30s.nc (NetCDF). Located here `./fix/fix_orog <https://noaa-ufs-srw-pds.s3.amazonaws.com/index.html#fix/fix_orog/>`_.
  * Global 30-arc-second USGS GMTED2010 orography data.
- * gmted2010.30sec.int (unformatted binary). Located here `./fix/fix_orog <https://noaa-ufs-srw-pds.s3.amazonaws.com/index.html#fix/fix_orog/>`_.
+ * topography.gmted2010.30s.nc (NetCDF). Located here `./fix/fix_orog <https://noaa-ufs-srw-pds.s3.amazonaws.com/index.html#fix/fix_orog/>`_.
  * 30-arc-second RAMP Antarctic terrain data (Radarsat Antarctic Mapping Project)
- * thirty.second.antarctic.new.bin (unformatted binary). Located here `./fix/fix_orog <https://noaa-ufs-srw-pds.s3.amazonaws.com/index.html#fix/fix_orog/>`_.
+ * topography.antarctica.ramp.30s.nc (NetCDF). Located here `./fix/fix_orog <https://noaa-ufs-srw-pds.s3.amazonaws.com/index.html#fix/fix_orog/>`_.
 
 **Output data:** 
 
@@ -665,3 +665,73 @@ Run script
 ----------
 
 To run, use the machine-dependent script under ./util/weight_gen
+
+***************************************************
+UFS_UTILS utilities
+***************************************************
+
+gdas_init
+=========
+
+Introduction
+------------
+
+The gdas_init utility is used to create coldstart initial conditions for global cycled and forecast-only experiments using the chgres_cube program. It has two components: one that pulls the input data required by chgres_cube from HPSS, and one that runs chgres_cube. The utility is only supported on machines with access to HPSS:
+
+ * Hera
+ * Jet
+ * WCOSS2
+ * S4 (Only the chgres_cube step is supported, not the data pull step.)
+
+Location
+--------
+
+Find it here: ./util/gdas_init
+
+Build UFS_UTILS and set 'fixed' directories
+-------------------------------------------
+
+Invoke the build script from the root directory: 
+
+::
+
+ ./build_all.sh
+
+Set the 'fixed' directories using the script in the './fix' subdirectory (where $MACHINE is 'hera', 'jet', 'wcoss2', or 's4'):
+
+::
+
+ ./link_fixdirs.sh emc $MACHINE
+
+Configure for your experiment
+-----------------------------
+
+Edit the variables in the 'config' file for your experiment:
+
+ * **EXTRACT_DIR** - Directory where data extracted from HPSS is stored.
+ * **EXTRACT_DATA** - Set to 'yes' to extract data from HPSS. If data has been extracted and is located in EXTRACT_DIR, set to 'no'. On 's4' this step can't be run. Instead, the data must be pulled from another machine.
+ * **RUN_CHGRES** - To run chgres, set to 'yes'. To extract data only, set to 'no'.
+ * **yy/mm/dd/hh** - The year/month/day/hour of your desired experiment. Use a four digit year and two digits for month/day/hour. **NOTE:** *The standard build of chgres_cube does NOT support experiments prior to June 12, 2019. To coldstart an experiment prior to these dates, contact a repository manager for assistance.*
+ * **LEVS** - Number of hybrid levels plus 1. To run with 127 levels, set LEVS to 128.
+ * **CRES_HIRES** - Resolution of the hires component of your experiment. Example: C768.
+ * **CRES_ENKF** - Resolution of the enkf component of the experiments.
+ * **UFS_DIR** - Location of your cloned UFS_UTILS repository.
+ * **OUTDIR** - Directory where the coldstart data output from chgres is stored.
+ * **CDUMP** - When 'gdas', will process gdas and enkf members. When 'gfs', will process gfs member for running free forecast only.
+ * **use_v16retro** - When 'yes', use v16 retrospective parallel data. The retrospective parallel tarballs can be missing or incomplete. So this option may not always work. Contact a UFS_UTILS repository manager if you encounter problems.
+
+Note: This utility selects the ocean resolution in the set_fixed_files.sh script using a default based on the user-selected CRES value. For example, for a cycled experiment with a CRES_HIRES/CRES_ENKF of C384/C192, the ocean resolution defaults to 0.25/0.50-degree. To choose another ocean resolution, the user will need to manually modify the set_fixed_files.sh script.
+
+
+Kick off the utility
+--------------------
+
+Submit the driver script (where $MACHINE is 'hera', 'jet', 'wcoss2', or 's4')
+
+::
+
+ ./driver.$MACHINE.sh 
+
+The standard output will be placed in log files in the current directory. 
+
+The converted output will be found in $OUTDIR, including the needed abias and radstat initial condition files (if CDUMP=gdas). The files will be in the needed directory structure for the global-workflow system, therefore a user can move the contents of their $OUTDIR directly into their $ROTDIR.

driver_scripts/driver_grid.hercules.sh

@@ -154,8 +154,8 @@ fi
 #-----------------------------------------------------------------------
 
 export home_dir=$SLURM_SUBMIT_DIR/..
-export TEMP_DIR=/work/noaa/stmp/$LOGNAME/fv3_grid.$gtype
-export out_dir=/work/noaa/stmp/$LOGNAME/my_grids
+export TEMP_DIR=/work2/noaa/stmp/$LOGNAME/fv3_grid.$gtype
+export out_dir=/work2/noaa/stmp/$LOGNAME/my_grids
 
 #-----------------------------------------------------------------------
 # Should not need to change anything below here.