diff --git a/docs/index.md b/docs/index.md index 07a587974691..4d484b4e0481 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,12 +1,16 @@ # The Energy Exascale Earth System Model (E3SM) -The documentation for the components of E3SM is found here. +The E3SM documentation is organized into sections for each component model and additional sections for shared tools and general guides. -## Components +## Component Models - [EAM](./EAM/index.md) - [EAMxx](./EAMxx/index.md) - [ELM](./ELM/index.md) - [MOSART](./MOSART/index.md) -Please see [Developing Docs](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/3924787306/Developing+Documentation) to learn about developing documentation for E3SM. +## Tools + +- [generate_domain_files](./generate_domain_files/index.md) + +Please see [Developing Docs](https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/3924787306/Developing+Documentation) to learn about how to contribute to the documentation. diff --git a/mkdocs.yaml b/mkdocs.yaml index aaeefb4c81f0..632091f4ee56 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -4,6 +4,7 @@ nav: - Introduction: 'index.md' - 'Developing Docs': 'https://acme-climate.atlassian.net/wiki/spaces/DOC/pages/3924787306/Developing+Documentation' - Components: '*include ./components/*/mkdocs.yml' + - Tools: '*include ./tools/*/mkdocs.yml' theme: name: material diff --git a/tools/generate_domain_files/docs/index.md b/tools/generate_domain_files/docs/index.md new file mode 100644 index 000000000000..578582051403 --- /dev/null +++ b/tools/generate_domain_files/docs/index.md @@ -0,0 +1,47 @@ +# Generating Domain Files + +Domain files are needed at runtime by the coupler, data models, and land model. The land model uses the mask to determine where to run and the coupler use the land fraction to merge fluxes from multiple surface types to the atmosphere above them. + +Domain files are created from a conservative, monotone mapping file from the ocean grid (where the mask is defined) to the atmosphere grid. + +## Environment + +The new domain generation tool requires a few special packages, such as xarray, numba, and itertools. These are all included in the E3SM unified environment: + + +Alternatively, a simple conda environment can be created with the following command: + +```shell +conda create --name example_env --channel conda-forge xarray numpy numba scikit-learn netcdf4 +``` + +## Map File Generation + +The map file used to generate the domain files can be created a few different ways. For a typical E3SM configuration we recommend using a conservative, monotone map. Here is an example command that can be used to generate one (as of NCO version 5.2.2) + +```shell +ncremap -5 -a traave --src_grd=${OCN_GRID} --dst_grd=${ATM_GRID} --map_file=${MAP_FILE} +``` + +Note that existing ocean grid files can be found in the inputdata repository: `inputdata/ocn/mpas-o//` + +The atmosphere grid file should be on the "pg2" grid. These grid files are easily generated with three TempestRemap commands as follows: + +```shell +NE=30 +GenerateCSMesh --alt --res ${NE} --file ${GRID_FILE_PATH}/ne${NE}.g +GenerateVolumetricMesh --in ${GRID_FILE_PATH}/ne${NE}.g --out ${GRID_FILE_PATH}/ne${NE}pg2.g --np 2 --uniform +ConvertMeshToSCRIP --in ${GRID_FILE_PATH}/ne${NE}pg2.g --out ${GRID_FILE_PATH}/ne${NE}pg2_scrip.nc +``` + +For RRM grids the last two commands would be used on the exodus file produced by [SQuadGen](https://github.com/ClimateGlobalChange/squadgen) (See the [Adding Support for New Grids](https://docs.e3sm.org/user-guides/adding-grid-support-overview.md) tutorial for more information.). + +## Running the Domain Generation Tool + +Below is a typical example of how to invoke the domain generation tool from the command line: + +```shell +NE=30 +MAP_FILE=${MAP_FILE_ROOT}/map_oEC60to30v3_to_ne${NE}pg2_traave.20240313.nc +python generate_domain_files_E3SM.py -m ${MAP_FILE} -o oEC60to30v3 -l ne${NE}pg2 --date-stamp=9999 --output-root=${OUTPUT_ROOT} +``` diff --git a/tools/generate_domain_files/generate_domain_files_E3SM.py b/tools/generate_domain_files/generate_domain_files_E3SM.py new file mode 100644 index 000000000000..efaa7c60c159 --- /dev/null +++ b/tools/generate_domain_files/generate_domain_files_E3SM.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python3 +#--------------------------------------------------------------------------------------------------- +''' +This is a replacement for the legacy gen_domain tool created for CESM. +Most legacy functionality is reproduced, with the notable exception of +the pole point latitude adjustment needed for the CESM FV grid. + +Created April, 2024 by Walter Hannah (LLNL) +''' +#--------------------------------------------------------------------------------------------------- +''' +The map file used to generate the domain files can be created a few different ways. +For a typical E3SM configuration we recommend using a conservative, monotone map. +Here is an example command that can be used to generate one as of NCO version 5.2.2 + + ncremap -5 -a traave --src_grd=${OCN_GRID} --dst_grd=${ATM_GRID} --map_file=${MAP_FILE} + +''' +#--------------------------------------------------------------------------------------------------- +import datetime, os, numpy as np, xarray as xr, numba, itertools +user, host = os.getenv('USER'), os.getenv('HOST') +source_code_meta = 'generate_domain_E3SM.py' +#--------------------------------------------------------------------------------------------------- +class clr:END,RED,GREEN,MAGENTA,CYAN = '\033[0m','\033[31m','\033[32m','\033[35m','\033[36m' +#--------------------------------------------------------------------------------------------------- +usage = ''' +python generate_domain_files_E3SM.py -m + -o + -l + [--output-root ] + [--date-stamp ] + [--fminval ] + [--fmaxval ] + [--set-omask] + +Purpose: + For "bi-grid" configurations of E3SM (land grid is same as atmos): + Given a mapping file from the ocean grid (where the mask is defined) + to the atmosphere grid, this tool creates land and ocean domain files + needed by data model components (ex. datm, dlnd, docn) + + For "tri-grid" configurations of E3SM (land grid is different from atmos/ocn): + In addition to running this tool with the ocn->atm map as above, + a second iteration is needed with a similar ocn->lnd map. + +Environment + + This tool requires a few special packages, such as xarray, numba, and itertools. + These are all included in the E3SM unified environment: + https://e3sm.org/resources/tools/other-tools/e3sm-unified-environment/ + + Otherwise a simple conda environment can be created: + conda create --name example_env --channel conda-forge xarray numpy numba scikit-learn netcdf4 + +The following output domain files are created: + + domain.lnd._..nc + land domain file on the land/atmos grid with a land fraction + corresponding to (1-ocnfrac) mask mapped to the land grid + + domain.ocn._..nc + ocean domain on the land/atmos grid with an ocean fraction based + on the ocean grid mask mapped to the land/atmos grid for when + atm,lnd,ice,ocn are all on the same grid + (not compatible with MPAS sea-ice) + + domain.ocn...nc + ocean domain on the ocean grid +''' +from optparse import OptionParser +parser = OptionParser(usage=usage) +parser.add_option('-m', + dest='map_file', + default=None, + help='Input mapping file from ocean to atmosphere grid - '+\ + 'This is the primary source from which the ocean and'+\ + 'land domains will be determined') +parser.add_option('-l', + dest='lnd_grid', + default=None, + help='Output land grid name') +parser.add_option('-o', + dest='ocn_grid', + default=None, + help='Output ocean grid name') +parser.add_option('--output-root', + dest='output_root', + default='./', + help='Output path for domain files') +parser.add_option('--date-stamp', + dest='date_stamp', + default=None, + help='Creation date stamp for domain files') +parser.add_option('--fminval', + dest='fminval', + default=1e-3, + help='Minimum allowable land fraction (reset to 0 below fminval)') +parser.add_option('--fmaxval', + dest='fmaxval', + default=1, + help='Maximum allowable land fraction (reset to 1 above fmaxval)') +parser.add_option('--set-omask', + dest='set_omask', + default=False, + action='store_true', + help='If True then an ocean mask is not required and will '+\ + 'simply be set to a vector of 1\'s if mask_a is not '+\ + 'present in the input mapping file. If --set-omask is '+\ + 'omitted, then mask_a is required and an error will be '+\ + 'raised if it does not exist in the input mapping file.') +(opts, args) = parser.parse_args() +#--------------------------------------------------------------------------------------------------- +def main(): + global domain_a_grid_file, domain_b_grid_file, ocn_grid_file, atm_grid_file + #------------------------------------------------------------------------------- + # check for valid input arguments + + if opts.map_file is None: + raise ValueError(f'{clr.RED}input map file was not specified{clr.END}') + if opts.lnd_grid is None: + raise ValueError(f'{clr.RED}land grid name was not specified{clr.END}') + if opts.ocn_grid is None: + raise ValueError(f'{clr.RED}ocean grid name was not specified{clr.END}') + if not os.path.exists(opts.output_root) : + raise ValueError(f'{clr.RED}Output root path does not exist{clr.END}') + + #------------------------------------------------------------------------------- + # Set date stamp for file name + + if opts.date_stamp is None: + cdate = datetime.datetime.now().strftime('%Y%m%d') + else: + cdate = opts.date_stamp + + #------------------------------------------------------------------------------- + # specify output file names + + domain_file_ocn_on_ocn = f'{opts.output_root}/domain.ocn.{opts.ocn_grid}.{cdate}.nc' + domain_file_lnd_on_atm = f'{opts.output_root}/domain.lnd.{opts.lnd_grid}_{opts.ocn_grid}.{cdate}.nc' + domain_file_ocn_on_atm = f'{opts.output_root}/domain.ocn.{opts.lnd_grid}_{opts.ocn_grid}.{cdate}.nc' + + # cosmetic clean up of file names + domain_file_ocn_on_ocn = domain_file_ocn_on_ocn.replace('//','/') + domain_file_lnd_on_atm = domain_file_lnd_on_atm.replace('//','/') + domain_file_ocn_on_atm = domain_file_ocn_on_atm.replace('//','/') + + #----------------------------------------------------------------------------- + # print some informative stuff + + print(f''' + Input files and parameter values: + {clr.GREEN}map_file {clr.END}: {opts.map_file} + {clr.GREEN}lnd_grid {clr.END}: {opts.lnd_grid} + {clr.GREEN}ocn_grid {clr.END}: {opts.ocn_grid} + {clr.GREEN}fminval {clr.END}: {opts.fminval} + {clr.GREEN}fmaxval {clr.END}: {opts.fmaxval} + {clr.GREEN}set_omask {clr.END}: {opts.set_omask} + ''') + + #----------------------------------------------------------------------------- + # open map file as dataset + + ds = xr.open_dataset(opts.map_file) + + #----------------------------------------------------------------------------- + # read grid meta-data from map file + + if 'domain_a' in ds.attrs.keys(): domain_a_grid_file = ds.attrs['domain_a'] + if 'domain_b' in ds.attrs.keys(): domain_b_grid_file = ds.attrs['domain_b'] + + if 'grid_file_ocn' in ds.attrs.keys(): + ocn_grid_file = ds.attrs['grid_file_ocn'] + else: + ocn_grid_file = ds.attrs['grid_file_src'] + + if 'grid_file_ocn' in ds.attrs.keys(): + atm_grid_file = ds.attrs['grid_file_atm'] + else: + atm_grid_file = ds.attrs['grid_file_dst'] + + #----------------------------------------------------------------------------- + # print some useful information from the map file + + print(f''' + Grid information from map file: + {clr.CYAN}domain_a file {clr.END}: {domain_a_grid_file} + {clr.CYAN}domain_b file {clr.END}: {domain_b_grid_file} + {clr.CYAN}ocn_grid_file {clr.END}: {ocn_grid_file} + {clr.CYAN}atm_grid_file {clr.END}: {atm_grid_file} + {clr.CYAN}ocn grid size (n_a){clr.END}: {len(ds.n_a)} + {clr.CYAN}atm grid size (n_b){clr.END}: {len(ds.n_b)} + {clr.CYAN}sparse mat size (n_s){clr.END}: {len(ds.n_s)} + ''') + + #----------------------------------------------------------------------------- + # Create ocean domain on ocean grid (domain_file_ocn_on_ocn) + + # Get ocn mask on ocn grid + omask = get_mask(ds,opts,suffix='_a') + ofrac = xr.zeros_like(ds['area_a']) + + ds_out = xr.Dataset() + ds_out['xc'] = ds['xc_a'] .expand_dims(dim='nj').rename({'n_a':'ni'}) + ds_out['yc'] = ds['yc_a'] .expand_dims(dim='nj').rename({'n_a':'ni'}) + ds_out['xv'] = ds['xv_a'] .expand_dims(dim='nj').rename({'n_a':'ni','nv_a':'nv'}) + ds_out['yv'] = ds['yv_a'] .expand_dims(dim='nj').rename({'n_a':'ni','nv_a':'nv'}) + ds_out['area'] = ds['area_a'].expand_dims(dim='nj').rename({'n_a':'ni'}) + ds_out['frac'] = ofrac .expand_dims(dim='nj').rename({'n_a':'ni'}) + ds_out['mask'] = omask .expand_dims(dim='nj').rename({'n_a':'ni'}) + + add_metadata(ds_out) + + ds_out.to_netcdf(path=domain_file_ocn_on_ocn,mode='w') + + print(f'successfully created domain file: {clr.MAGENTA}{domain_file_ocn_on_ocn}{clr.END}') + + #----------------------------------------------------------------------------- + # Create land and ocean domains on atmosphere grid + + xc = ds['xc_b'] + yc = ds['yc_b'] + xv = ds['xv_b'] + yv = ds['yv_b'] + area = ds['area_b'] + + mask_a = get_mask(ds,opts,suffix='_a') + frac_a = xr.where( mask_a!=0, xr.ones_like(ds['area_a']), xr.zeros_like(ds['area_a']) ) + + # compute ocn fraction on atm grid + ofrac = compute_ofrac_on_atm( len(ds['n_s']), np.zeros(ds['area_b'].shape), + frac_a.values, ds['S'].values, + ds['row'].values-1, ds['col'].values-1 ) + ofrac = xr.DataArray(ofrac,dims=['n_b']) + + # lfrac is area frac of mask "_a" on grid "_b" or float(mask) + lfrac = xr.zeros_like(ds['area_b']) + + # convert to land fraction + lfrac_min = opts.fmaxval + lfrac_max = opts.fminval + omask = xr.ones_like(ds['area_b'],dtype=np.int32) + lmask = xr.zeros_like(ds['area_b'],dtype=np.int32) + lfrac = 1 - ofrac + lfrac_min = lfrac.min().values + lfrac_max = lfrac.max().values + lfrac = xr.where( lfrac>opts.fmaxval, 1, lfrac ) + lfrac = xr.where( lfrac