From 2992939cb0e95ecebdd256c98b87896a8049bbd7 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Wed, 10 Feb 2021 12:07:16 -0700 Subject: [PATCH 01/25] moving to the pandas rtd theme, simple layout for now --- docs/_templates/footer.html | 9 + docs/_templates/layout.html | 13 ++ docs/{api.rst => api/index.rst} | 0 docs/conf.py | 45 ++-- docs/getting_started/full_stack_install.rst | 225 ++++++++++++++++++++ docs/getting_started/index.rst | 37 ++++ docs/{ => getting_started}/installation.rst | 6 +- docs/index.rst | 20 +- scripts/install_dev_stack | 178 ++++++++++++++++ setup.py | 8 + 10 files changed, 504 insertions(+), 37 deletions(-) create mode 100644 docs/_templates/footer.html create mode 100644 docs/_templates/layout.html rename docs/{api.rst => api/index.rst} (100%) create mode 100644 docs/getting_started/full_stack_install.rst create mode 100644 docs/getting_started/index.rst rename docs/{ => getting_started}/installation.rst (95%) mode change 100755 => 100644 create mode 100644 scripts/install_dev_stack diff --git a/docs/_templates/footer.html b/docs/_templates/footer.html new file mode 100644 index 0000000..f0df4ac --- /dev/null +++ b/docs/_templates/footer.html @@ -0,0 +1,9 @@ +{% extends "!footer.html" %} + +{% block extrafooter %} +
+ +
+ ARS logo +
+{% endblock %} \ No newline at end of file diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html new file mode 100644 index 0000000..c047960 --- /dev/null +++ b/docs/_templates/layout.html @@ -0,0 +1,13 @@ +{% extends "!layout.html" %} + +{% block header %} + +{% endblock %} + +{% block extrafooter %} +
+ +
+{% endblock %} \ No newline at end of file diff --git a/docs/api.rst b/docs/api/index.rst similarity index 100% rename from docs/api.rst rename to docs/api/index.rst diff --git a/docs/conf.py b/docs/conf.py index 2a780c2..275c510 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,7 +16,12 @@ from mock import Mock as MagicMock import sys import os +from datetime import datetime + from inicheck.tools import config_documentation +from pkg_resources import get_distribution + +import awsm # If extensions (or modules to document with autodoc) are in another # directory, add these directories to sys.path here. If the directory is @@ -27,7 +32,8 @@ # Call auto documentation for our config files config_documentation('./auto_config.rst', - paths=['../awsm/framework/CoreConfig.ini', '../awsm/framework/recipes.ini']) + paths=['../awsm/framework/CoreConfig.ini', + '../awsm/framework/recipes.ini']) # Get the project root dir, which is the parent dir of this @@ -58,14 +64,15 @@ def __getattr__(cls, name): # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', - 'sphinx.ext.viewcode', - 'sphinx.ext.autodoc', - 'sphinx.ext.todo', - 'sphinx.ext.napoleon', - 'sphinx.ext.imgmath' # , - # 'sphinxcontrib.bibtex' - ] +extensions = [ + 'sphinx.ext.doctest', + 'sphinx.ext.autodoc', + 'sphinx.ext.todo', + 'sphinx.ext.viewcode', + 'sphinx.ext.napoleon', + 'sphinx.ext.imgmath', + 'sphinx.ext.intersphinx' +] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -80,19 +87,19 @@ def __getattr__(cls, name): master_doc = 'index' # General information about the project. -project = u'awsm' -copyright = u"2017, Micah Sandusky, USDA Agricultural Research Service" -author = u'Micah Sandusky' +project = 'AWSM' +copyright = f"{datetime.now().year}, USDA Agricultural Research Service, Northwest Watershed Research Center" +author = 'Northwest Watershed Research Center' # The version info for the project you're documenting, acts as replacement # for |version| and |release|, also used in various other places throughout # the built documents. # # The short X.Y version. -version = u'0.1.0' -#version = awsm.__version__ +version = get_distribution('awsm').version + # The full version, including alpha/beta/rc tags. -#release = awsm.__version__ +release = get_distribution('awsm').version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -160,7 +167,7 @@ def __getattr__(cls, name): # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'sphinx_rtd_theme' +html_theme = 'pydata_sphinx_theme' # Theme options are theme-specific and customize the look and feel of a # theme further. For a list of options available for each theme, see the @@ -262,7 +269,7 @@ def __getattr__(cls, name): latex_documents = [ ('index', 'awsm.tex', u'awsm Documentation', - u'Micah Sandusky', 'manual'), + u'USDA ARS NWRC', 'manual'), ] # The name of an image file (relative to this directory) to place at @@ -293,7 +300,7 @@ def __getattr__(cls, name): man_pages = [ ('index', 'awsm', u'awsm Documentation', - [u'Micah Sandusky'], 1) + [u'USDA ARS NWRC'], 1) ] # If true, show URL addresses after external links. @@ -308,7 +315,7 @@ def __getattr__(cls, name): texinfo_documents = [ ('index', 'awsm', u'awsm Documentation', - u'Micah Sandusky', + u'USDA ARS NWRC', 'awsm', 'One line description of project.', 'Miscellaneous'), diff --git a/docs/getting_started/full_stack_install.rst b/docs/getting_started/full_stack_install.rst new file mode 100644 index 0000000..77109fe --- /dev/null +++ b/docs/getting_started/full_stack_install.rst @@ -0,0 +1,225 @@ +======================= +Full Development Installation +======================= + +The full stack installation will allow a user to run AWSM with the following enabled: + +* High Resolution Rapid Refresh (HRRR) as inputs +* WindNinja to downscale HRRR to the iSnobal domain + +To install AWSM, we recommend using the docker image. If that is not feasible, then +install all the necessary dependencies from source. This will install the following packages: + +* Weather Forecast Retrieval +* SMRF +* PySnobal +* Katana (WindNinja) +* AWSM + +Environment Setup +----------------- + +Setup a virtual environment and activate. + +.. code:: bash + + python3 -m virtualenv .venv + source .venv/bin/activate + +System Dependencies +------------------- + +Ubuntu / Debain +~~~~~~~~~~~~~~~ + +.. code:: bash + + sudo apt-get install -y python3-dev \ + python3-pip \ + git \ + gcc \ + g++ \ + cmake \ + make \ + ca-certificates \ + libblas-dev \ + liblapack-dev \ + libatlas-base-dev \ + libffi-dev \ + libssl-dev \ + gfortran \ + libyaml-dev \ + libfreetype6-dev \ + libcurl4-gnutls-dev \ + netcdf-bin \ + libnetcdf-dev \ + libpng-dev \ + python3 \ + python3-pip \ + curl \ + libeccodes-dev \ + libeccodes-tools \ + wget + +wgrib2 +~~~~~~ + +`wgrib2` allows for working with grib2 files and is maintained by NOAA_. + +.. _NOAA: https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/compile_questions.html + +.. code:: bash + + export CC=gcc + export FC=gfortran + curl -L ftp://ftp.cpc.ncep.noaa.gov/wd51we/wgrib2/wgrib2.tgz | tar xz + + cd grib2 + wget ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-c-4.7.3.tar.gz + wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.4/src/hdf5-1.10.4.tar.gz + sed -i "s/USE_NETCDF4=0/USE_NETCDF4=1/" makefile + sed -i "s/USE_NETCDF3=1/USE_NETCDF3=0/" makefile + make + make lib + sudo cp wgrib2/wgrib2 /usr/local/bin/wgrib2 + make deep-clean + +WindNinja +~~~~~~~~~ + +WindNinja has many build dependencies and is well documented on their github wiki. WindNinja relies +on `poppler`, `gdal` and `proj` before it can be installed. + +.. code:: bash + + PREFIX=/usr/local + POPPLER="poppler-0.23.4" + PROJ="proj-4.8.0" + GDAL="gdal-2.2.2" + + # Get and build poppler for PDF support in GDAL + wget http://poppler.freedesktop.org/$POPPLER.tar.xz + tar -xvf $POPPLER.tar.xz + cd $POPPLER/ + ./configure --prefix=$PREFIX --enable-xpdf-headers + make + sudo make install + cd .. + + # Get and build proj + wget http://download.osgeo.org/proj/$PROJ.tar.gz + tar xvfz $PROJ.tar.gz + cd $PROJ + ./configure --prefix=$PREFIX + make clean + make + sudo make install + sudo cp $PREFIX/include/proj_api.h $PREFIX/lib + cd .. + + # Get and build GDAL with poppler support + wget http://download.osgeo.org/gdal/2.2.2/$GDAL.tar.gz + tar -xvf $GDAL.tar.gz + cd $GDAL/ + ./configure --prefix=$PREFIX --with-poppler=$PREFIX + make -j 8 + sudo make install + cd .. + +With the 3 dependencies installed, WindNinja can be installed. + +.. code:: bash + + mkdir -p windninja/build + curl -L https://github.com/firelab/windninja/archive/3.5.0.tar.gz | tar xz + mv windninja-3.5.0 windninja/windninja + cmake -DNINJA_CLI=ON -DNINJAFOAM=OFF -DNINJA_QTGUI=OFF windninja/windninja + make + sudo make install + sudo ldconfig + rm -rf windninja/ + +Model Code +---------- + +Weather Forecast Retrieval +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Weather Forecast Retrieval (WFR) loads gridded datasets like HRRR and formats the data into a +format that SMRF can utilize. + +.. code:: bash + + git clone https://github.com/USDA-ARS-NWRC/weather_forecast_retrieval.git + cd weather_forecast_retrieval + python3 -m pip install -r requirements.txt + python3 -m pip install -e . + +PySnobal +~~~~~~~~ + +PySnobal + +.. code:: bash + + git clone https://github.com/USDA-ARS-NWRC/pysnobal.git + cd pysnobal + python3 -m pip install -e . + +AWSM +~~~~ + +Automated Water Supply Model (AWSM) + +.. code:: bash + + git clone https://github.com/USDA-ARS-NWRC/awsm.git + cd awsm + python3 -m pip install -r requirements.txt + python3 -m pip install -e . + +SMRF +~~~~ + +Spatial Modeling for Resources Framework (SMRF) + +.. code:: bash + + git clone https://github.com/USDA-ARS-NWRC/smrf.git + cd smrf + python3 -m pip install -r requirements.txt + python3 -m pip install -e .[test] + +katana +~~~~~~ + +Katana + +.. code:: bash + + git clone https://github.com/USDA-ARS-NWRC/katana.git + cd awsm + python3 -m pip install -r requirements.txt + python3 -m pip install -e . + +Testing +------- + +After all the dependencies and model code are installed, we recommend testing to +ensure that the code will work as expected during the model simulations. + +The unittest framework is used to run the tests with `python3 -m unittest -v` within +the following repositories: + +- `weather_forecast_retrieval` +- `smrf` +- `awsm` +- `katana` + +If many of the tests provide information that the tests were within a tolerance or +failed because the results were not the same, try to set the following environment variable +to increase the tolerance criteria for passing a test. + +.. code:: bash + + export NOT_ON_GOLD_HOST=YOU_BETCHA \ No newline at end of file diff --git a/docs/getting_started/index.rst b/docs/getting_started/index.rst new file mode 100644 index 0000000..a81ecaf --- /dev/null +++ b/docs/getting_started/index.rst @@ -0,0 +1,37 @@ +=============== +Getting started +=============== + +Installation +------------ + +To install AWSM locally on Linux of MacOSX, first clone the repository and build into a +virtual environment. The general steps are as follows and will +test the AWSM installation by running the tests. + +Clone from the repository + +.. code:: bash + + git clone https://github.com/USDA-ARS-NWRC/awsm.git + + +And install the requirements and run the tests. + +.. code:: bash + + python3 -m pip install -r requirements_dev.txt + python3 -m pip install . + python3 -m unittest -v + + +For in-depth instructions to installing all the development packages and specific requirements +for AWSM, check out the the :doc:`installation page ` + +For Windows, the install method is using Docker. + +.. toctree:: + :maxdepth: 2 + + installation + full_stack_install \ No newline at end of file diff --git a/docs/installation.rst b/docs/getting_started/installation.rst old mode 100755 new mode 100644 similarity index 95% rename from docs/installation.rst rename to docs/getting_started/installation.rst index f1c252a..34965c6 --- a/docs/installation.rst +++ b/docs/getting_started/installation.rst @@ -34,10 +34,6 @@ source your smrfenv instead of step number 1. virtualenv awsmenv source awsmenv/bin/activate -**Tip:** The developers recommend using an alias to quickly turn on -and off your virtual environment. - - 2. Clone AWSM source code from the ARS-NWRC github. .. code:: console @@ -49,7 +45,7 @@ and off your virtual environment. .. code:: console - cd AWSM + cd awsm pip install -r requirements.txt python setup.py install diff --git a/docs/index.rst b/docs/index.rst index a28b41a..845dfa0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,24 +1,18 @@ -Welcome to AWSM's Documentation! +Automated Water Supply Model ====================================== -.. include:: readme.rst +Blurb here .. toctree:: :maxdepth: 2 + :titlesonly: - readme - installation - usage - configuration - contributing - config_ref - api - authors - history + getting_started/index + user_guide/index + api/index Indices and tables ================== * :ref:`genindex` -* :ref:`modindex` -* :ref:`search` +* :ref:`modindex` \ No newline at end of file diff --git a/scripts/install_dev_stack b/scripts/install_dev_stack new file mode 100644 index 0000000..802078c --- /dev/null +++ b/scripts/install_dev_stack @@ -0,0 +1,178 @@ +#!/usr/bin/env bash + +# Install the full development stack of AWSM for all packages +# The code will be installed in the current working directory in the following structure: +# +# code +# |____ weather_forecast_retrieval +# |____ smrf +# |____ awsm +# |____ pysnobal +# |____ katana +# +# NOTE: running this script may or may not work depending on the operating system +# and user privledges. Mainly the use of sudo. + +set -x + +# clean up the directory and start fresh +rm -r code/ +mkdir -p code/build +cd code/build + +############################################ +# install dependancies +############################################ +sudo apt-get install -y python3-dev \ + python3-pip \ + git \ + gcc \ + g++ \ + cmake \ + make \ + ca-certificates \ + libblas-dev \ + liblapack-dev \ + libatlas-base-dev \ + libffi-dev \ + libssl-dev \ + gfortran \ + libyaml-dev \ + libfreetype6-dev \ + libcurl4-gnutls-dev \ + libfontconfig1-dev \ + qt4-dev-tools \ + libqtwebkit-dev \ + libboost-program-options-dev \ + libboost-date-time-dev \ + libgeos-dev \ + libboost-test-dev \ + netcdf-bin \ + libnetcdf-dev \ + libpng-dev \ + python3 \ + python3-pip \ + curl \ + m4 \ + libeccodes-dev \ + libeccodes-tools \ + wget + + +# WGRIB2 +export CC=gcc +export FC=gfortran +curl -L ftp://ftp.cpc.ncep.noaa.gov/wd51we/wgrib2/wgrib2.tgz | tar xz + +cd grib2 +wget ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-c-4.7.3.tar.gz +wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.4/src/hdf5-1.10.4.tar.gz +sed -i "s/USE_NETCDF4=0/USE_NETCDF4=1/" makefile +sed -i "s/USE_NETCDF3=1/USE_NETCDF3=0/" makefile +make +make lib +sudo cp wgrib2/wgrib2 /usr/local/bin/wgrib2 +make deep-clean +cd .. +rm -rf grib2/ + +# WindNinja dependancies +PREFIX=/usr/local +POPPLER="poppler-0.23.4" +PROJ="proj-4.8.0" +GDAL="gdal-2.2.2" + +# Get and build poppler for PDF support in GDAL +wget http://poppler.freedesktop.org/$POPPLER.tar.xz +tar -xvf $POPPLER.tar.xz +cd $POPPLER/ +./configure --prefix=$PREFIX --enable-xpdf-headers +make +sudo make install +cd .. + +# Get and build proj +wget http://download.osgeo.org/proj/$PROJ.tar.gz +tar xvfz $PROJ.tar.gz +cd $PROJ +./configure --prefix=$PREFIX +make clean +make +sudo make install +sudo cp $PREFIX/include/proj_api.h $PREFIX/lib +cd .. + +# Get and build GDAL with poppler support +wget http://download.osgeo.org/gdal/2.2.2/$GDAL.tar.gz +tar -xvf $GDAL.tar.gz +cd $GDAL/ +./configure --prefix=$PREFIX --with-poppler=$PREFIX +make -j 8 +sudo make install +cd .. + +# WindNinja +mkdir -p windninja/build +curl -L https://github.com/firelab/windninja/archive/3.5.0.tar.gz | tar xz +mv windninja-3.5.0 windninja/windninja +cmake -DNINJA_CLI=ON -DNINJAFOAM=OFF -DNINJA_QTGUI=OFF windninja/windninja +make +sudo make install +sudo ldconfig +rm -rf windninja/ +cd .. + +############################################ +# install repositories +############################################ + +# repositories to install +declare -a repositories=( + "https://github.com/USDA-ARS-NWRC/awsm.git" + "https://github.com/USDA-ARS-NWRC/smrf.git" + "https://github.com/USDA-ARS-NWRC/pysnobal.git" + "https://github.com/USDA-ARS-NWRC/weather_forecast_retrieval.git" + "https://github.com/USDA-ARS-NWRC/katana.git" +) + +for repository in "${repositories[@]}" +do + git clone --depth 1 ${repository} +done + +# Weather Forecast Retrevial +cd weather_forecast_retrieval +python3 -m pip install -r requirements.txt +python3 -m pip install -e . + +# AWSM +cd ../awsm +python3 -m pip install -r requirements.txt +python3 -m pip install -e . + +# SMRF +cd ../smrf +python3 -m pip install -r requirements.txt +python3 -m pip install -e .[test] + +# pysnobal +cd ../pysnobal +python3 -m pip install -e . + +# katana +cd ../katana +python3 -m pip install -e . + +# Test all of the repos +export NOT_ON_GOLD_HOST=YOU_BETCHA +cd ../weather_forecast_retrieval +python3 -m unittest -v + +cd ../smrf +python3 -m unittest -v + +cd ../awsm +python3 -m unittest -v + +cd ../katana +python3 -m unittest -v \ No newline at end of file diff --git a/setup.py b/setup.py index c9abb40..d576693 100755 --- a/setup.py +++ b/setup.py @@ -56,4 +56,12 @@ setup_requires=[ 'setuptools_scm', ], + extras_require={ + 'docs': [ + 'Sphinx>=3.0,<=4.0', + 'pydata-sphinx-theme', + 'sphinxcontrib-bibtex>=1.0', + 'sphinxcontrib-websupport>=1.0.1', + ] + }, ) From af904c45ec7567118948b3ab9a48c07f877c58a4 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Fri, 12 Feb 2021 11:30:04 -0700 Subject: [PATCH 02/25] updating the read the docs yml --- readthedocs.yml | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/readthedocs.yml b/readthedocs.yml index 87fa516..293a787 100644 --- a/readthedocs.yml +++ b/readthedocs.yml @@ -1,5 +1,25 @@ -build: - image: latest + +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details +# Required +version: 2 + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + +# Build documentation with MkDocs +#mkdocs: +# configuration: mkdocs.yml + +# Optionally set the version of Python and requirements required to build your docs python: - version: 3.6 + version: 3.8 + install: + - requirements: requirements.txt + - method: pip + path: . + extra_requirements: + - docs \ No newline at end of file From 57cbcd20d2e1da8c2e6f44529cfe655b7f52c4cd Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Fri, 12 Feb 2021 11:41:12 -0700 Subject: [PATCH 03/25] mocking awsm and pysnobal --- docs/conf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 275c510..ca0f051 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -50,8 +50,7 @@ def __getattr__(cls, name): return Mock() -MOCK_MODULES = ['netCDF4', 'pandas', 'smrf', - 'smrf.utils', 'smrf.envphys'] +MOCK_MODULES = ['netCDF4', 'pandas', 'awsm', 'pysnobal'] sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) From a330a01eb5a1b0fda468c08de2608b66f4a364e6 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Fri, 12 Feb 2021 11:48:46 -0700 Subject: [PATCH 04/25] updating mock --- docs/conf.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index ca0f051..286a157 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,10 +13,10 @@ # All configuration values have a default; values that are commented out # serve to show the default. -from mock import Mock as MagicMock import sys import os from datetime import datetime +from unittest.mock import Mock from inicheck.tools import config_documentation from pkg_resources import get_distribution @@ -44,13 +44,13 @@ # -- Have to do a mock install of some modules that RTD doesn't have -------- -class Mock(MagicMock): - @classmethod - def __getattr__(cls, name): - return Mock() +# class Mock(MagicMock): +# @classmethod +# def __getattr__(cls, name): +# return Mock() -MOCK_MODULES = ['netCDF4', 'pandas', 'awsm', 'pysnobal'] +MOCK_MODULES = ['netCDF4', 'pandas', 'pysnobal'] sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) From 565025b7ee8440b5cad34ef547c6fd87955588fd Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Fri, 12 Feb 2021 12:01:39 -0700 Subject: [PATCH 05/25] mock more? --- docs/conf.py | 9 +-------- docs/index.rst | 2 +- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 286a157..dd714ca 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -43,14 +43,7 @@ sys.path.insert(0, os.path.abspath('../')) # -- Have to do a mock install of some modules that RTD doesn't have -------- - -# class Mock(MagicMock): -# @classmethod -# def __getattr__(cls, name): -# return Mock() - - -MOCK_MODULES = ['netCDF4', 'pandas', 'pysnobal'] +MOCK_MODULES = ['netCDF4', 'pandas', 'pysnobal', 'pysnobal.c_snobal'] sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) diff --git a/docs/index.rst b/docs/index.rst index 845dfa0..609b80d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,7 +1,7 @@ Automated Water Supply Model ====================================== -Blurb here +Automated Water Supply Model (AWSM) was developed at the USDA Agricultural Research Service (ARS) in Boise, ID. AWSM was designed to streamline the work flow used by the ARS to forecast the water supply of multiple water basins. AWSM standardizes the steps needed to distribute weather station data with SMRF, run an energy and mass balance with iSnobal, and process the results, while maintaining the flexibility of each program. .. toctree:: :maxdepth: 2 From 928e6fe447858295e70e6483bc186a75c00f55e0 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Fri, 12 Feb 2021 13:05:03 -0700 Subject: [PATCH 06/25] add some deps to the docs that are needed before pysnobal can be installed --- docs/index.rst | 6 +++++- readthedocs.yml | 4 ++-- setup.py | 2 ++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 609b80d..e21b277 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,7 +1,11 @@ Automated Water Supply Model ====================================== -Automated Water Supply Model (AWSM) was developed at the USDA Agricultural Research Service (ARS) in Boise, ID. AWSM was designed to streamline the work flow used by the ARS to forecast the water supply of multiple water basins. AWSM standardizes the steps needed to distribute weather station data with SMRF, run an energy and mass balance with iSnobal, and process the results, while maintaining the flexibility of each program. +Automated Water Supply Model (AWSM) was developed at the USDA Agricultural Research Service +(ARS) in Boise, ID. AWSM was designed to streamline the work flow used by the ARS to forecast +the water supply of multiple water basins. AWSM standardizes the steps needed to distribute +weather station data with SMRF, run an energy and mass balance with iSnobal, and process the +results, while maintaining the flexibility of each program. .. toctree:: :maxdepth: 2 diff --git a/readthedocs.yml b/readthedocs.yml index 293a787..8ee75d7 100644 --- a/readthedocs.yml +++ b/readthedocs.yml @@ -18,8 +18,8 @@ sphinx: python: version: 3.8 install: - - requirements: requirements.txt - method: pip path: . extra_requirements: - - docs \ No newline at end of file + - docs + - requirements: requirements_dev.txt \ No newline at end of file diff --git a/setup.py b/setup.py index d576693..da3886c 100755 --- a/setup.py +++ b/setup.py @@ -58,6 +58,8 @@ ], extras_require={ 'docs': [ + 'Cython', + 'numpy<1.19', 'Sphinx>=3.0,<=4.0', 'pydata-sphinx-theme', 'sphinxcontrib-bibtex>=1.0', From a405ecf28d9a3eaabd0d0be46e0cc8bb633438e7 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Thu, 6 May 2021 09:40:59 -0600 Subject: [PATCH 07/25] added folder structure to dev stack --- docs/getting_started/full_stack_install.rst | 28 +++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/docs/getting_started/full_stack_install.rst b/docs/getting_started/full_stack_install.rst index 77109fe..4dfa2a6 100644 --- a/docs/getting_started/full_stack_install.rst +++ b/docs/getting_started/full_stack_install.rst @@ -1,6 +1,6 @@ -======================= +============================= Full Development Installation -======================= +============================= The full stack installation will allow a user to run AWSM with the following enabled: @@ -26,6 +26,27 @@ Setup a virtual environment and activate. python3 -m virtualenv .venv source .venv/bin/activate +Build Folder Structure +---------------------- + +The basic folder structure for the full stack will be as follows. Everything +under `windninja` will build the dependencies and windninja code. + +- awsm_project + - windninja (can be removed after install) + - grib2 + - poppler-0.23.4 + - proj-4.8.0 + - gdal-2.2.2 + - windninja + - build + - awsm + - smrf + - pysnobal + - katana + - weather_forecast_retrieval (if needed) + + System Dependencies ------------------- @@ -148,6 +169,9 @@ Weather Forecast Retrieval Weather Forecast Retrieval (WFR) loads gridded datasets like HRRR and formats the data into a format that SMRF can utilize. +NOTE: Weather forecast retrieval is a dependency of SMRF and should not to be installed from source +unless modifying the weather forecast retrieval code. + .. code:: bash git clone https://github.com/USDA-ARS-NWRC/weather_forecast_retrieval.git From 72c3c6ef16723ea61a73cbfe4eeb5fe6426fed12 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Thu, 6 May 2021 12:46:12 -0600 Subject: [PATCH 08/25] updated full stack docs --- docs/getting_started/full_stack_install.rst | 150 ++++++++++---------- 1 file changed, 73 insertions(+), 77 deletions(-) diff --git a/docs/getting_started/full_stack_install.rst b/docs/getting_started/full_stack_install.rst index 4dfa2a6..9a977f0 100644 --- a/docs/getting_started/full_stack_install.rst +++ b/docs/getting_started/full_stack_install.rst @@ -16,30 +16,20 @@ install all the necessary dependencies from source. This will install the follow * Katana (WindNinja) * AWSM -Environment Setup ------------------ - -Setup a virtual environment and activate. - -.. code:: bash - python3 -m virtualenv .venv - source .venv/bin/activate - -Build Folder Structure +Basic Folder Structure ---------------------- The basic folder structure for the full stack will be as follows. Everything under `windninja` will build the dependencies and windninja code. - awsm_project + - .venv - windninja (can be removed after install) - - grib2 - - poppler-0.23.4 - - proj-4.8.0 - - gdal-2.2.2 - windninja - build + - + - grib2 (can be removed after install) - awsm - smrf - pysnobal @@ -53,11 +43,18 @@ System Dependencies Ubuntu / Debain ~~~~~~~~~~~~~~~ -.. code:: bash +Python dependencies - sudo apt-get install -y python3-dev \ +..code:: bash + sudo apt-get install -y python3 \ + python3-dev \ python3-pip \ - git \ + +System dependencies + +.. code:: bash + + sudo apt-get install -y git \ gcc \ g++ \ cmake \ @@ -71,31 +68,83 @@ Ubuntu / Debain gfortran \ libyaml-dev \ libfreetype6-dev \ - libcurl4-gnutls-dev \ netcdf-bin \ - libnetcdf-dev \ libpng-dev \ - python3 \ - python3-pip \ + m4 \ curl \ libeccodes-dev \ libeccodes-tools \ wget +Environment Setup +----------------- + +Setup a virtual environment and activate. + +.. code:: bash + + python3 -m virtualenv .venv + source .venv/bin/activate + + +WindNinja +~~~~~~~~~ + +WindNinja has many build dependencies and is well documented on their github `wiki`_. WindNinja relies +on `poppler`, `gdal` and `proj` before it can be installed. WindNinja provides a handy `build_deps.sh` +script that aids in the building of the dependencies. + +.. _wiki: https://github.com/firelab/windninja/wiki/Building-WindNinja-on-Linux + +Download WindNinja and move to the correct location. + +.. code:: bash + + cd awsm_project + mkdir -p windninja/build + curl -L https://github.com/firelab/windninja/archive/3.5.0.tar.gz | tar xz + mv windninja-3.5.0 windninja/windninja + +Now build the dependencies for WindNinja with their `build_deps.sh` script. This +will take a long time. + +.. code:: bash + + cd awsm_project/windninja + sh windninja/scripts/build_deps.sh + +Build WindNinja + +..code:: bash + + cd awsm_project/windninja + cmake -DNINJA_CLI=ON -DNINJAFOAM=OFF -DNINJA_QTGUI=OFF windninja + make + sudo make install + sudo ldconfig + +Clean up the WindNinja build folder. + +..code:: bash + + rm -rf awsm_project/windninja/ + wgrib2 ~~~~~~ -`wgrib2` allows for working with grib2 files and is maintained by NOAA_. +`wgrib2` allows for working with grib2 files and is maintained by NOAA_. The install +will take time as `wgrib2` will perform tests during installation. .. _NOAA: https://www.cpc.ncep.noaa.gov/products/wesley/wgrib2/compile_questions.html .. code:: bash + cd awsm_project export CC=gcc export FC=gfortran curl -L ftp://ftp.cpc.ncep.noaa.gov/wd51we/wgrib2/wgrib2.tgz | tar xz - cd grib2 + cd awsm_project/grib2 wget ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-c-4.7.3.tar.gz wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.4/src/hdf5-1.10.4.tar.gz sed -i "s/USE_NETCDF4=0/USE_NETCDF4=1/" makefile @@ -105,60 +154,7 @@ wgrib2 sudo cp wgrib2/wgrib2 /usr/local/bin/wgrib2 make deep-clean -WindNinja -~~~~~~~~~ - -WindNinja has many build dependencies and is well documented on their github wiki. WindNinja relies -on `poppler`, `gdal` and `proj` before it can be installed. - -.. code:: bash - - PREFIX=/usr/local - POPPLER="poppler-0.23.4" - PROJ="proj-4.8.0" - GDAL="gdal-2.2.2" - - # Get and build poppler for PDF support in GDAL - wget http://poppler.freedesktop.org/$POPPLER.tar.xz - tar -xvf $POPPLER.tar.xz - cd $POPPLER/ - ./configure --prefix=$PREFIX --enable-xpdf-headers - make - sudo make install - cd .. - - # Get and build proj - wget http://download.osgeo.org/proj/$PROJ.tar.gz - tar xvfz $PROJ.tar.gz - cd $PROJ - ./configure --prefix=$PREFIX - make clean - make - sudo make install - sudo cp $PREFIX/include/proj_api.h $PREFIX/lib - cd .. - - # Get and build GDAL with poppler support - wget http://download.osgeo.org/gdal/2.2.2/$GDAL.tar.gz - tar -xvf $GDAL.tar.gz - cd $GDAL/ - ./configure --prefix=$PREFIX --with-poppler=$PREFIX - make -j 8 - sudo make install - cd .. - -With the 3 dependencies installed, WindNinja can be installed. - -.. code:: bash - - mkdir -p windninja/build - curl -L https://github.com/firelab/windninja/archive/3.5.0.tar.gz | tar xz - mv windninja-3.5.0 windninja/windninja - cmake -DNINJA_CLI=ON -DNINJAFOAM=OFF -DNINJA_QTGUI=OFF windninja/windninja - make - sudo make install - sudo ldconfig - rm -rf windninja/ + rm -rf awsm_project/grib2 Model Code ---------- From 67115d61755e9329c64bcee497905cffa033232d Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Fri, 7 May 2021 13:54:29 -0600 Subject: [PATCH 09/25] page placeholders for getting started, awsm basin setup filled out --- docs/conf.py | 6 +- docs/getting_started/basin_setup.rst | 239 ++++++++++++++++++++ docs/getting_started/create_config.rst | 101 +++++++++ docs/getting_started/docker.rst | 29 +++ docs/getting_started/full_stack_install.rst | 11 +- docs/getting_started/index.rst | 6 +- docs/getting_started/run_awsm.rst | 92 ++++++++ docs/user_guide/index.rst | 3 + 8 files changed, 478 insertions(+), 9 deletions(-) create mode 100644 docs/getting_started/basin_setup.rst create mode 100644 docs/getting_started/create_config.rst create mode 100644 docs/getting_started/docker.rst create mode 100644 docs/getting_started/run_awsm.rst create mode 100644 docs/user_guide/index.rst diff --git a/docs/conf.py b/docs/conf.py index dd714ca..76a3150 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -202,9 +202,9 @@ #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} -html_sidebars = {'**': ['globaltoc.html', - 'relations.html', 'sourcelink.html', 'searchbox.html']} +html_sidebars = {} +# html_sidebars = {'**': ['globaltoc.html', +# 'relations.html', 'sourcelink.html', 'searchbox.html']} # Additional templates that should be rendered to pages, maps page names # to template names. diff --git a/docs/getting_started/basin_setup.rst b/docs/getting_started/basin_setup.rst new file mode 100644 index 0000000..21834c7 --- /dev/null +++ b/docs/getting_started/basin_setup.rst @@ -0,0 +1,239 @@ +AWSM Basin Setup +================ + +All components will be described using docker containers such that the generation +of the necessary files is reproducible. Note that exact docker versions will be +specified throughout the document but can be updated to different versions. + +The initial set up is performed using the package `basin_setup`_. `basin_setup`_ requires a DEM and it +the `LANDFIRE`_ dataset to fill in the vegetation layers and parameters required to run iSnobal. + +.. _basin_setup: https://github.com/USDA-ARS-NWRC/basin_setup +.. _LANDFIRE: https://www.landfire.gov + + +``topo.nc`` file +---------------- + +The basin setup procdure creates a `topo.nc` that contains the following static layers: + +1. Digital elevation model +2. Vegetation type +3. Vegetation height +4. Vegetation extinction coefficient +5. Vegetation optical transmissivity +6. Basin mask (optional) + +All these layers are stored in a netCDF file, typically referred to the ``topo.nc`` file. More +information about the ``topo.nc`` file are in the SMRF documentation_. + +.. note:: + + The ``topo.nc`` **must** have projection information. It's just good practice. Using + ``basin_setup`` will ensure the projection is in the ``topo.nc``. + + +.. _documentation: https://smrf.readthedocs.io/en/latest/getting_started/create_topo.html + + +Creating the DEM +---------------- + +The DEM was obtained from the USGS National Elevation `dataset`_ at ~10-m (9.26-m) resolution. +The original tiles are provided in EPSG:4269 - NAD83 – Geographic projection. Download the necessary +tiles to cover the modeling domain. + +.. _dataset: https://catalog.data.gov/dataset/usgs-national-elevation-dataset-ned + +Join the tiles were in the native projection and resolution (EPSG 4269). Subsequently, crop the joined DEM +and resample to 50-m resolution and reproject to UTM coordinates (for example EPSG 32613, UTM 13N - WGS 84), +all of which is accomplished using a single ``gdalwarp`` instruction (also available in QGIS or similar ArcGIS +command). + +.. note:: + Beware of using Nearest Neightbour interpolation as it can create artifacts in the resulting DEM. Use average, + bilinear or cubic interpolation. + +A sample ``gdalwarp`` command for the Gunnison in Colorado. This will resample to 50-m grid resolution and +reproject from EPSG:4269 to EPSG:32613: + +.. code:: bash + + gdalwarp \ + -s_srs EPSG:4269 \ + -t_srs EPSG:32613 \ + -tr 50.0 50.0 \ + -r bilinear \ + -te 302500.0 4263100.0 388350.0 4328700.0 \ + -te_srs EPSG:32613 \ + -of GTiff \ + in_dem_10_meters.tif out_dem_50_meters.tif + + +Pour Point File +--------------- + +Once the final DEM is generated, the basin is delineated using one or multiple pour points to define the +basin and any sub-basins. The pour points are contained within a ``.bna`` file that indicates the pour +point locations. + +The format for the ``.bna`` file: + +.. code:: sh + + "Point code", "Another point code", 1 + , + +An example for the Gunnison that will delineate multiple sub-basins with the basin outlet at "TRAC2": + +.. code:: sh + + "TRAC2","TRAC2H_F",1 + 364060.6510282378,4302483.0984167568 + "TPIC2","TPIC2L_F",1 + 360137.0765313815,4297634.3167137895 + "ALTC2","ALTC2L_F",1 + 339502.3149144283,4281186.4048871947 + "ALEC2","ALEC2H_F",1 + 339065.7024648183,4281297.3988735778 + "OHOC2","OHOC2H_F",1 + 331764.4055900126,4272796.4096229207 + "GUSC2","GUSC2L_F",1 + 330122.1191776489,4267747.993258521 + + +Delineate Basin, Generate Topo +------------------------------ + +The basin delineation and creation of the ``topo.nc`` file is done with the docker version +of ``basin_setup``. A ``docker-compose.yml`` file aids the composition of the docker commands +and simplifies mounting data volumes to the docker image. The following ``docker-compose.yml`` +file contains two services, `delineate` runs the delineation routine and ``basin_setup`` creates +the ``topo.nc``. + +.. code:: yml + + version: '3' + + services: + delineate: + image: usdaarsnwrc/basin_setup:0.14 + volumes: + - ./topo:/data + - ./veg_data:/Downloads + entrypoint: delineate + + basin_setup: + image: usdaarsnwrc/basin_setup:0.14 + volumes: + - ./topo:/data + - ./veg_data:/Downloads + entrypoint: basin_setup + + +With the DEM tiff file ``out_dem_50_meters.tif`` and the ``.bna`` file, run ``delineate`` +with the ``basin_setup`` docker image. This will delineate the basin with `TauDEM`_ and +create sub-basins for each pour point. + +.. _TauDEM: https://hydrology.usu.edu/taudem/taudem5/index.html + +.. code:: bash + + docker-compose run delineate \ + -p pour_points.bna \ + -d out_dem_50_meters.tif \ + -t 100000 \ + -n 2 \ + --debug \ + --streamflow + +The ``delineate`` command will create a file in ``./topo/delineation/basin_outline.shp`` which +will contain the delineated basin. Open the shape file and ensure that the basin +delineation performed as expected. Next, create the topo with ``basin_setup``. The LANDFIRE +dataset is quite large (~1.5GB) and can be downloaded prior and/or reused. Ensure that the +LANDFIRE dataset is in the ``./veg_data`` folder and the download will be skipped. + +.. code:: bash + + docker-compose run basin_setup \ + -f delineation/basin_outline.shp \ + -bn BasinName \ + -dm out_dem_50_meters.tif \ + -d /Downloads # will download LANDFIRE here if not present + + +View ``topo.nc`` +---------------- + +Open the ``topo.nc`` in your favorite netcdf viewer or GIS program. Ensure all the layers +have been generated. The example below shows the Lakes basin in the AWSM tests. + +.. note:: + + The global attributes for the ``topo.nc`` include the version of ``basin_setup`` + and the command used to create the file for reproducibility. + + +.. code:: bash + + $ ncdump -h awsm/tests/basins/Lakes/topo/topo.nc + + $ netcdf topo { + dimensions: + y = 168 ; + x = 156 ; + variables: + float y(y) ; + y:least_significant_digit = 3LL ; + y:description = "UTM, north south" ; + y:long_name = "y coordinate" ; + y:units = "meters" ; + y:standard_name = "projection_y_coordinate" ; + float x(x) ; + x:least_significant_digit = 3LL ; + x:description = "UTM, east west" ; + x:long_name = "x coordinate" ; + x:units = "meters" ; + x:standard_name = "projection_x_coordinate" ; + float dem(y, x) ; + dem:least_significant_digit = 3LL ; + dem:long_name = "dem" ; + dem:grid_mapping = "projection" ; + ubyte mask(y, x) ; + mask:least_significant_digit = 3LL ; + mask:long_name = "Lakes Basin" ; + mask:grid_mapping = "projection" ; + ushort veg_type(y, x) ; + veg_type:least_significant_digit = 3LL ; + veg_type:long_name = "vegetation type" ; + veg_type:grid_mapping = "projection" ; + float veg_height(y, x) ; + veg_height:least_significant_digit = 3LL ; + veg_height:long_name = "vegetation height" ; + veg_height:grid_mapping = "projection" ; + float veg_k(y, x) ; + veg_k:least_significant_digit = 4LL ; + veg_k:long_name = "vegetation k" ; + veg_k:grid_mapping = "projection" ; + float veg_tau(y, x) ; + veg_tau:least_significant_digit = 4LL ; + veg_tau:long_name = "vegetation tau" ; + veg_tau:grid_mapping = "projection" ; + char projection ; + projection:grid_mapping_name = "universal_transverse_mercator" ; + projection:utm_zone_number = 11. ; + projection:semi_major_axis = 6378137. ; + projection:inverse_flattening = 298.257223563 ; + projection:spatial_ref = "PROJCS[\"WGS84/UTMzone11N\",\nGEOGCS[\"WGS84\",\nDATUM[\"WGS_1984\",\nSPHEROID[\"WGS84\",6378137,298.257223563,\nAUTHORITY[\"EPSG\",\"7030\"]],\nAUTHORITY[\"EPSG\",\"6326\"]],\nPRIMEM[\"Greenwich\",0,\nAUTHORITY[\"EPSG\",\"8901\"]],\nUNIT[\"degree\",0.01745329251994328,\nAUTHORITY[\"EPSG\",\"9122\"]],\nAUTHORITY[\"EPSG\",\"4326\"]],\nUNIT[\"metre\",1,\nAUTHORITY[\"EPSG\",\"9001\"]],\nPROJECTION[\"Transverse_Mercator\"],\nPARAMETER[\"latitude_of_origin\",0],\nPARAMETER[\"central_meridian\",-117],\nPARAMETER[\"scale_factor\",0.9996],\nPARAMETER[\"false_easting\",500000],\nPARAMETER[\"false_northing\",0],\nAUTHORITY[\"EPSG\",\"32611\"],\nAXIS[\"Easting\",EAST],\nAXIS[\"Northing\",NORTH]]" ; + projection:_CoordinateTransformType = "Projection" ; + projection:_CoordinateAxisTypes = "GeoX GeoY" ; + + // global attributes: + :last_modified = "[2019-12-31 21:33:38] Data added or updated" ; + string :Conventions = "CF-1.6" ; + string :dateCreated = "2019-12-31 21:33:38" ; + string :Title = "Topographic Images for SMRF/AWSM" ; + string :history = "[2019-12-31 21:33:38] Create netCDF4 file using Basin Setup v0.13.0" ; + string :institution = "USDA Agricultural Research Service, Northwest Watershed Research Center" ; + string :generation_command = "/usr/local/bin/basin_setup -f delineation/basin_outline.shp -bn Lakes Basin -dm lakes_dem_UTM11_WGS84.tif -d /Downloads -ex 319975 4158253 327755 4166675" ; + } \ No newline at end of file diff --git a/docs/getting_started/create_config.rst b/docs/getting_started/create_config.rst new file mode 100644 index 0000000..adb21b4 --- /dev/null +++ b/docs/getting_started/create_config.rst @@ -0,0 +1,101 @@ +Create a config file +==================== + +After the topo file has been created, build the SMRF configuration file. For in depth documentation +see how to :doc:`use a configuration file <../user_guide/configuration>` and the +:doc:`core configuration <../user_guide/core_config>` for all SMRF options. + +.. note:: + + Configuration file paths are relative to the configuration file location. + +At a minimum to get started, the following configuration file will apply all the defaults. +The required changes are specifying the path to the ``topo.nc`` file, dates to run the model +and the location of the csv input data. + + +.. code:: + + ################################################################################ + # Files for DEM and vegetation + ################################################################################ + [topo] + filename: ./topo/topo.nc + + ################################################################################ + # Dates to run model + ################################################################################ + [time] + time_step: 60 + start_date: 1998-01-14 15:00:00 + end_date: 1998-01-14 19:00:00 + time_zone: utc + + ################################################################################ + # CSV section configurations + ################################################################################ + [csv] + wind_speed: ./station_data/wind_speed.csv + air_temp: ./station_data/air_temp.csv + cloud_factor: ./station_data/cloud_factor.csv + wind_direction: ./station_data/wind_direction.csv + precip: ./station_data/precip.csv + vapor_pressure: ./station_data/vapor_pressure.csv + metadata: ./station_data/metadata.csv + + ################################################################################ + # Air temperature distribution + ################################################################################ + [air_temp] + + ################################################################################ + # Vapor pressure distribution + ################################################################################ + [vapor_pressure] + + ################################################################################ + # Wind speed and wind direction distribution + ################################################################################ + [wind] + maxus_netcdf: ./topo/maxus.nc + + ################################################################################ + # Precipitation distribution + ################################################################################ + [precip] + + ################################################################################ + # Albedo distribution + ################################################################################ + [albedo] + + ################################################################################ + # Cloud Factor - Fraction used to limit solar radiation Cloudy (0) - Sunny (1) + ################################################################################ + [cloud_factor] + + ################################################################################ + # Solar radiation + ################################################################################ + [solar] + + ################################################################################ + # Incoming thermal radiation + ################################################################################ + [thermal] + + ################################################################################ + # Soil temperature + ################################################################################ + [soil_temp] + + ################################################################################ + # Output variables + ################################################################################ + [output] + out_location: ./output + + ################################################################################ + # System variables and Logging + ################################################################################ + [system] diff --git a/docs/getting_started/docker.rst b/docs/getting_started/docker.rst new file mode 100644 index 0000000..ab3b859 --- /dev/null +++ b/docs/getting_started/docker.rst @@ -0,0 +1,29 @@ +AWSM and Docker +=============== + +SMRF is also built into a docker image to make it easy to install on any operating system. +The docker images are built automatically from the Github repository and include the latest +code base or stable release images. + +The SMRF docker image has a folder meant to mount data inside the docker image at ``/data``. + +.. code:: console + + docker run -v :/data usdaarsnwrc/smrf run_smrf + +The ```` should be the path to where the configuration file, data and topo are on +the host machine. This will also be the location to where the SMRF output will go. + +.. note:: + + The paths in the configuration file must be adjusted for being inside the docker image. For example, + in the command above the path to the config will be inside the docker image. This would be + ``/data/config.ini`` and not the path on the host machine. + +In a way that ARS uses this, we keep the config, topo and data on one location as the files are fairly +small. The output then is put in another location as it file size can be much larger. To facilitate +this, mount the input and output data separately and modify the configuration paths. + +.. code:: console + + docker run -v :/data/input -v :/data/output usdaarsnwrc/smrf run_smrf \ No newline at end of file diff --git a/docs/getting_started/full_stack_install.rst b/docs/getting_started/full_stack_install.rst index 9a977f0..e97f673 100644 --- a/docs/getting_started/full_stack_install.rst +++ b/docs/getting_started/full_stack_install.rst @@ -1,4 +1,3 @@ -============================= Full Development Installation ============================= @@ -115,7 +114,7 @@ will take a long time. Build WindNinja -..code:: bash +.. code:: bash cd awsm_project/windninja cmake -DNINJA_CLI=ON -DNINJAFOAM=OFF -DNINJA_QTGUI=OFF windninja @@ -125,7 +124,7 @@ Build WindNinja Clean up the WindNinja build folder. -..code:: bash +.. code:: bash rm -rf awsm_project/windninja/ @@ -165,8 +164,10 @@ Weather Forecast Retrieval Weather Forecast Retrieval (WFR) loads gridded datasets like HRRR and formats the data into a format that SMRF can utilize. -NOTE: Weather forecast retrieval is a dependency of SMRF and should not to be installed from source -unless modifying the weather forecast retrieval code. +.. note:: + + Weather forecast retrieval is a dependency of SMRF and should not to be installed from source + unless modifying the weather forecast retrieval code. .. code:: bash diff --git a/docs/getting_started/index.rst b/docs/getting_started/index.rst index a81ecaf..10d63b2 100644 --- a/docs/getting_started/index.rst +++ b/docs/getting_started/index.rst @@ -34,4 +34,8 @@ For Windows, the install method is using Docker. :maxdepth: 2 installation - full_stack_install \ No newline at end of file + full_stack_install + basin_setup + create_config + run_awsm + docker \ No newline at end of file diff --git a/docs/getting_started/run_awsm.rst b/docs/getting_started/run_awsm.rst new file mode 100644 index 0000000..0dafdb8 --- /dev/null +++ b/docs/getting_started/run_awsm.rst @@ -0,0 +1,92 @@ +Run AWSM +======== + +After installing SMRF, generating a topo and creating a configuration file, SMRF can be ran. There are +two ways to run SMRF, first is through the ``run_smrf`` command or through the SMRF API. If SMRF is being +used as input to a snow or hydrology model, we recommend to use ``run_smrf`` as it will generate all +the input required. + +``run_smrf`` command +-------------------- + +To run a full simulation simply run (barring any errors): + +.. code:: bash + + run_smrf + + +SMRF API +-------- + +The ``smrf`` package can also be used as an API, typically to focus on a single variable. There are steps that +SMRF uses to load the data then distribute and usage of the API should follow the same pattern. For example, +below is the function :mod:`run_smrf `. + +.. code:: python + + with SMRF(config) as s: + # load topo data + s.loadTopo() + + # initialize the distribution + s.create_distribution() + + # initialize the outputs if desired + s.initializeOutput() + + # load weather data and station metadata + s.loadData() + + # distribute + s.distributeData() + + +The next example below builds on above and will distribute air temperature and vapor pressure. They can be used to +get the distributed dew point or web bulb temperature. + +.. code:: python + + configFile = 'config.ini' + + with smrf.framework.SMRF(configFile) as s: + + # =================================================================== + # Model setup and initialize + # =================================================================== + + # These are steps that will load the necessary data and initialize + # the framework. Once loaded, this shouldn't need to be re-ran except + # if something major changes + + # load topo data + s.loadTopo() + + # Create the distribution class + s.distribute['air_temp'] = smrf.distribute.air_temp.ta( + s.config['air_temp']) + s.distribute['vapor_pressure'] = smrf.distribute.vapor_pressure.vp( + s.config['vapor_pressure']) + + # load weather data and station metadata + s.loadData() + + # Initialize the distribution + for v in s.distribute: + s.distribute[v].initialize(s.topo, s.data) + + # initialize the outputs if desired + s.initializeOutput() + + # Distribute the data and output + for output_count, t in enumerate(s.date_time): + + s.distribute['air_temp'].distribute(s.data.air_temp.ix[t]) + s.distribute['vapor_pressure'].distribute( + s.data.vapor_pressure.ix[t], + s.distribute['air_temp'].air_temp) + + # output at the frequency and the last time step + if (output_count % s.config['output']['frequency'] == 0) or \ + (output_count == len(s.date_time)): + s.output(t) \ No newline at end of file diff --git a/docs/user_guide/index.rst b/docs/user_guide/index.rst new file mode 100644 index 0000000..829391e --- /dev/null +++ b/docs/user_guide/index.rst @@ -0,0 +1,3 @@ +========== +User guide +========== \ No newline at end of file From 81cd29d6ba82b2c0fee5cb44b0b42d6ccd8ce296 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Fri, 28 May 2021 10:35:17 -0600 Subject: [PATCH 10/25] getting started --- docs/getting_started/full_stack_install.rst | 14 ++++++- docs/getting_started/index.rst | 41 ++++++++++++--------- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/docs/getting_started/full_stack_install.rst b/docs/getting_started/full_stack_install.rst index e97f673..58399db 100644 --- a/docs/getting_started/full_stack_install.rst +++ b/docs/getting_started/full_stack_install.rst @@ -1,7 +1,7 @@ Full Development Installation ============================= -The full stack installation will allow a user to run AWSM with the following enabled: +The full stack installation will allow a user to run AWSM from source with the following enabled: * High Resolution Rapid Refresh (HRRR) as inputs * WindNinja to downscale HRRR to the iSnobal domain @@ -15,6 +15,15 @@ install all the necessary dependencies from source. This will install the follow * Katana (WindNinja) * AWSM +.. note:: + + **Use at your own risk!** While this contains the latest code, it is not guaranteed + to work with the whole modeling framework. + + The latest code on contains all the latest developments. However, this must be used with caution + as it can be under active development, may change at any time and is not guaranteed to work with + the rest of the modeling framework at that moment. Once the code has been fully tested within the + modeling framework, a new release will be created to signal a move to a stable version. Basic Folder Structure ---------------------- @@ -44,7 +53,8 @@ Ubuntu / Debain Python dependencies -..code:: bash +.. code:: bash + sudo apt-get install -y python3 \ python3-dev \ python3-pip \ diff --git a/docs/getting_started/index.rst b/docs/getting_started/index.rst index 10d63b2..5d4917b 100644 --- a/docs/getting_started/index.rst +++ b/docs/getting_started/index.rst @@ -2,33 +2,40 @@ Getting started =============== -Installation ------------- +Which version? +-------------- -To install AWSM locally on Linux of MacOSX, first clone the repository and build into a -virtual environment. The general steps are as follows and will -test the AWSM installation by running the tests. +Stable +~~~~~~ -Clone from the repository +The stable version of AWSM is currently `v0.10`. The code can be downloaded from the `releases`_ +or can be found on the `release-0.10` `branch`_. -.. code:: bash +.. _releases: https://github.com/USDA-ARS-NWRC/awsm/releases +.. _branch: https://github.com/USDA-ARS-NWRC/awsm/tree/release-0.10 - git clone https://github.com/USDA-ARS-NWRC/awsm.git +Best for: +- Applying the model in near real time +- Researchers wanting a ready to use model +- Those wanting the most stable and tested code -And install the requirements and run the tests. +Experimental +~~~~~~~~~~~~ -.. code:: bash +.. note:: + ⚠️ Use at your own risk! While this contains the latest code, it is not guaranteed to work with the whole modeling framework. - python3 -m pip install -r requirements_dev.txt - python3 -m pip install . - python3 -m unittest -v +The latest code on `main` contains all the latest development to AWSM. However, this must be used with caution as it +can be under active development, may change at any time and is not guaranteed to work with the rest of the modeling framework +at that moment. Once the code has been fully tested within the modeling framework, a new release will be created to signal +a move to a stable version. +Best for: -For in-depth instructions to installing all the development packages and specific requirements -for AWSM, check out the the :doc:`installation page ` - -For Windows, the install method is using Docker. +- Those planning on developing with AWSM +- Model simulations require features only found in the latest code +- Okay with the possibility that AWSM doesn't work with the rest of the modeling system .. toctree:: :maxdepth: 2 From 0417bd353b9a73fa41e54af474d0df7cc98cfabb Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Fri, 28 May 2021 13:46:59 -0600 Subject: [PATCH 11/25] getting started docs --- docs/getting_started/create_config.rst | 187 +++++++++++++++++++- docs/getting_started/docker.rst | 29 --- docs/getting_started/full_stack_install.rst | 32 ++-- docs/getting_started/index.rst | 20 ++- docs/getting_started/stable_install.rst | 77 ++++++++ 5 files changed, 292 insertions(+), 53 deletions(-) delete mode 100644 docs/getting_started/docker.rst create mode 100644 docs/getting_started/stable_install.rst diff --git a/docs/getting_started/create_config.rst b/docs/getting_started/create_config.rst index adb21b4..28ccf57 100644 --- a/docs/getting_started/create_config.rst +++ b/docs/getting_started/create_config.rst @@ -1,9 +1,9 @@ Create a config file ==================== -After the topo file has been created, build the SMRF configuration file. For in depth documentation +After the topo file has been created, build the AWSM configuration file. For in depth documentation see how to :doc:`use a configuration file <../user_guide/configuration>` and the -:doc:`core configuration <../user_guide/core_config>` for all SMRF options. +:doc:`core configuration <../user_guide/core_config>` for all AWSM options. .. note:: @@ -13,6 +13,25 @@ At a minimum to get started, the following configuration file will apply all the The required changes are specifying the path to the ``topo.nc`` file, dates to run the model and the location of the csv input data. +Stable vs. Experimental +----------------------- + +The configuration file is not compatible between the stable and experimental model versions. SMRF ``v0.10`` +made a large overhaul to the configuration file to become more verbose and clean up options. If using +the stable version, the configuration file will be based on the previous `core config`_. + +.. _core config: https://github.com/USDA-ARS-NWRC/smrf/blob/release-0.9/smrf/framework/CoreConfig.ini + +.. note:: + + Migrating from the stable SMRF can be achieved through ``inicheck`` `changelog`_. + +.. _changelog: https://inicheck.readthedocs.io/en/latest/changelogs.html + +Station Data Example +-------------------- + +The example below shows the minimum needed for using weather stations data with AWSM. .. code:: @@ -99,3 +118,167 @@ and the location of the csv input data. # System variables and Logging ################################################################################ [system] + + ################################################################################ + # Configurations for AWSM Master section + ################################################################################ + [awsm master] + run_smrf: True + model_type: ipysnobal + + ################################################################################ + # Configurations for PATHS section for rigid directory work + ################################################################################ + [paths] + path_dr: ./output + basin: rme + project_name: rme_test + project_description: fast rme test run + + ################################################################################ + # System parameters + ################################################################################ + [awsm system] + + ################################################################################ + # Parameters for restarting from crash + ################################################################################ + [isnobal restart] + + ################################################################################ + # Running Python wrapped iSnobal + ################################################################################ + [ipysnobal] + + +Atmospheric Model Example +------------------------- + +Atmospheric models require a few changes to the config file to tell SMRF that the input +data is gridded model output and how to interpolate to the AWSM model domain. For another +example, see the Lakes test within AWSM. + + +.. code:: + + ################################################################################ + # Files for DEM and vegetation + ################################################################################ + [topo] + filename: ./topo/topo.nc + + ################################################################################ + # Dates to run model + ################################################################################ + [time] + time_step: 60 + start_date: 2019-10-01 15:00 + end_date: 2019-10-01 17:00 + + ################################################################################ + # Gridded dataset i.e. wrf_out + ################################################################################ + [gridded] + hrrr_directory: ./input + data_type: hrrr_grib + + ################################################################################ + # Air temperature distribution + ################################################################################ + [air_temp] + distribution: grid + grid_local: True + + ################################################################################ + # Vapor pressure distribution + ################################################################################ + [vapor_pressure] + distribution: grid + grid_local: True + + ################################################################################ + # Wind speed and wind direction distribution + ################################################################################ + [wind] + wind_model: wind_ninja + distribution: grid + wind_ninja_dir: ./input + wind_ninja_dxdy: 200 + wind_ninja_pref: topo_windninja_topo + wind_ninja_tz: UTC + + ################################################################################ + # Precipitation distribution + ################################################################################ + [precip] + distribution: grid + grid_local: True + + ################################################################################ + # Albedo distribution + ################################################################################ + [albedo] + + ################################################################################ + # Solar radiation distribution + ################################################################################ + [solar] + + ################################################################################ + # Cloud Factor - Fraction used to limit solar radiation Cloudy (0) - Sunny (1) + ################################################################################ + [cloud_factor] + + ################################################################################ + # Thermal radiation distribution + ################################################################################ + [thermal] + + ################################################################################ + # Soil temperature + ################################################################################ + [soil_temp] + + ################################################################################ + # Output variables + ################################################################################ + [output] + out_location: ./output + + ################################################################################ + # System variables + ################################################################################ + [system] + + ################################################################################ + # Configurations for AWSM Master section + ################################################################################ + [awsm master] + run_smrf: True + model_type: ipysnobal + + ################################################################################ + # Configurations for PATHS section for rigid directory work + ################################################################################ + [paths] + path_dr: ./output + basin: lakes + project_name: lakes_gold + project_description: Lakes gold HRRR simulation + + ################################################################################ + # System parameters + ################################################################################ + [awsm system] + log_to_file: True + + ################################################################################ + # Parameters for restarting from crash + ################################################################################ + [isnobal restart] + + ################################################################################ + # Running Python wrapped iSnobal + ################################################################################ + [ipysnobal] + diff --git a/docs/getting_started/docker.rst b/docs/getting_started/docker.rst deleted file mode 100644 index ab3b859..0000000 --- a/docs/getting_started/docker.rst +++ /dev/null @@ -1,29 +0,0 @@ -AWSM and Docker -=============== - -SMRF is also built into a docker image to make it easy to install on any operating system. -The docker images are built automatically from the Github repository and include the latest -code base or stable release images. - -The SMRF docker image has a folder meant to mount data inside the docker image at ``/data``. - -.. code:: console - - docker run -v :/data usdaarsnwrc/smrf run_smrf - -The ```` should be the path to where the configuration file, data and topo are on -the host machine. This will also be the location to where the SMRF output will go. - -.. note:: - - The paths in the configuration file must be adjusted for being inside the docker image. For example, - in the command above the path to the config will be inside the docker image. This would be - ``/data/config.ini`` and not the path on the host machine. - -In a way that ARS uses this, we keep the config, topo and data on one location as the files are fairly -small. The output then is put in another location as it file size can be much larger. To facilitate -this, mount the input and output data separately and modify the configuration paths. - -.. code:: console - - docker run -v :/data/input -v :/data/output usdaarsnwrc/smrf run_smrf \ No newline at end of file diff --git a/docs/getting_started/full_stack_install.rst b/docs/getting_started/full_stack_install.rst index 58399db..bba81af 100644 --- a/docs/getting_started/full_stack_install.rst +++ b/docs/getting_started/full_stack_install.rst @@ -1,5 +1,5 @@ -Full Development Installation -============================= +EXPERIMENTAL: Full Development Installation +=========================================== The full stack installation will allow a user to run AWSM from source with the following enabled: @@ -17,7 +17,7 @@ install all the necessary dependencies from source. This will install the follow .. note:: - **Use at your own risk!** While this contains the latest code, it is not guaranteed + ⚠️ **Use at your own risk!** While this contains the latest code, it is not guaranteed to work with the whole modeling framework. The latest code on contains all the latest developments. However, this must be used with caution @@ -31,18 +31,20 @@ Basic Folder Structure The basic folder structure for the full stack will be as follows. Everything under `windninja` will build the dependencies and windninja code. -- awsm_project - - .venv - - windninja (can be removed after install) - - windninja - - build - - - - grib2 (can be removed after install) - - awsm - - smrf - - pysnobal - - katana - - weather_forecast_retrieval (if needed) +:: + + awsm_project + ├── .venv + ├── windninja (can be removed after install) + │ ├── windninja + │ ├── build + │ ├── + ├── grib2 (can be removed after install) + ├── awsm + ├── smrf + ├── pysnobal + ├── katana + ├── weather_forecast_retrieval (if needed) System Dependencies diff --git a/docs/getting_started/index.rst b/docs/getting_started/index.rst index 5d4917b..5195445 100644 --- a/docs/getting_started/index.rst +++ b/docs/getting_started/index.rst @@ -2,18 +2,19 @@ Getting started =============== -Which version? --------------- +Which version to use? +--------------------- Stable ~~~~~~ -The stable version of AWSM is currently `v0.10`. The code can be downloaded from the `releases`_ -or can be found on the `release-0.10` `branch`_. +The stable version of AWSM is currently :code:`v0.10`. The code can be downloaded from the `releases`_, +can be found on the :code:`release-0.10` `branch`_ or the stable :doc:`docker image <./stable_install>`. .. _releases: https://github.com/USDA-ARS-NWRC/awsm/releases .. _branch: https://github.com/USDA-ARS-NWRC/awsm/tree/release-0.10 + Best for: - Applying the model in near real time @@ -26,7 +27,7 @@ Experimental .. note:: ⚠️ Use at your own risk! While this contains the latest code, it is not guaranteed to work with the whole modeling framework. -The latest code on `main` contains all the latest development to AWSM. However, this must be used with caution as it +The latest code on :code:`main` contains all the latest development to AWSM. However, this must be used with caution as it can be under active development, may change at any time and is not guaranteed to work with the rest of the modeling framework at that moment. Once the code has been fully tested within the modeling framework, a new release will be created to signal a move to a stable version. @@ -37,12 +38,17 @@ Best for: - Model simulations require features only found in the latest code - Okay with the possibility that AWSM doesn't work with the rest of the modeling system +Content +~~~~~~~ + +The following will guide the installation and setup of AWSM. + .. toctree:: :maxdepth: 2 - installation + stable_install full_stack_install basin_setup create_config run_awsm - docker \ No newline at end of file + \ No newline at end of file diff --git a/docs/getting_started/stable_install.rst b/docs/getting_started/stable_install.rst new file mode 100644 index 0000000..66368b7 --- /dev/null +++ b/docs/getting_started/stable_install.rst @@ -0,0 +1,77 @@ +STABLE: Installation with Docker +================================ + +The AWSM docker version for ``release-0.10`` is built on an Ubuntu 18.04 image and +has all the necessary dependencies installed. To get the stable version of AWSM: + +.. code:: console + + docker pull usdaarsnwrc/awsm:0.10 + +Basic Project Folder Structure +------------------------------ + +We recommend the following or similar folder structure for the AWSM simulations. By keeping +to this folder structure, all the inputs and model configuration are in one place and can +be easily backed up, shared with others or packaged for a reproducible journal submission. + +:: + + awsm_project + ├── config + │ ├── wy2021_awsm_config.ini + | |── wy2021_awsm_config_v2.ini + ├── topo + | |── topo.nc + | |── lidar + | |── wy2020_lidar_snow_depths.nc + ├── station_data + | |── air_temp.csv + + +- The ``config`` folder contains all the configuration files for the model simulations. +- The ``topo`` folder contains the ``topo.nc`` file and lidar snow depths if available +- The ``station_data`` folder contains csv files if using weather stations for input data +- If using atmospheric model for AWSM inputs, these do not have to be located in the ``awsm_project`` folder. +- The AWSM outputs can be large and also do not have to be located in the ``awsm_project`` folder. + +Docker Examples +--------------- + +The AWSM docker image has a folder meant to mount data inside the docker image at ``/data``. We +recommend having a separate folder for the config, topo, inputs and outputs to make the file paths +verbose. The following docker examples could also be put into a ``docker-compose.yml`` to simplify +the command line. + +.. note:: + + The paths in the configuration file must be adjusted for being inside the docker image. For example, + in the command below, the path to the config will be inside the docker image. This would be + ``/data/config/config.ini`` and not the path on the host machine. + + +Station Data +~~~~~~~~~~~~ + +.. code:: console + + docker run \ + -v ./awsm_project/station_data:/data/input \ + -v ./awsm_project/config:/data/config \ + -v ./awsm_project/topo:/data/topo \ + -v /path/to/output:/data/output \ + usdaarsnwrc/awsm:0.10 /data/config/.ini + + +Atmospheric Data +~~~~~~~~~~~~ + +.. code:: console + + docker run \ + -v /path/to/atmospheric/data:/data/input \ + -v ./awsm_project/config:/data/config \ + -v ./awsm_project/topo:/data/topo \ + -v /path/to/output:/data/output \ + usdaarsnwrc/awsm:0.10 /data/config/.ini + From d78911770e8cd66564900621caf4a044cac5d746 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Tue, 1 Jun 2021 16:08:44 -0600 Subject: [PATCH 12/25] install doc structure --- docs/getting_started/index.rst | 3 +- .../{ => install}/full_stack_install.rst | 34 ++++++++++++------- docs/getting_started/install/index.rst | 8 +++++ .../{ => install}/stable_install.rst | 24 +++++++++++-- 4 files changed, 53 insertions(+), 16 deletions(-) rename docs/getting_started/{ => install}/full_stack_install.rst (88%) create mode 100644 docs/getting_started/install/index.rst rename docs/getting_started/{ => install}/stable_install.rst (81%) diff --git a/docs/getting_started/index.rst b/docs/getting_started/index.rst index 5195445..5ef1b30 100644 --- a/docs/getting_started/index.rst +++ b/docs/getting_started/index.rst @@ -46,8 +46,7 @@ The following will guide the installation and setup of AWSM. .. toctree:: :maxdepth: 2 - stable_install - full_stack_install + install/index basin_setup create_config run_awsm diff --git a/docs/getting_started/full_stack_install.rst b/docs/getting_started/install/full_stack_install.rst similarity index 88% rename from docs/getting_started/full_stack_install.rst rename to docs/getting_started/install/full_stack_install.rst index bba81af..2e69b7f 100644 --- a/docs/getting_started/full_stack_install.rst +++ b/docs/getting_started/install/full_stack_install.rst @@ -1,5 +1,5 @@ -EXPERIMENTAL: Full Development Installation -=========================================== +Full Development Installation +============================= The full stack installation will allow a user to run AWSM from source with the following enabled: @@ -35,16 +35,26 @@ under `windninja` will build the dependencies and windninja code. awsm_project ├── .venv - ├── windninja (can be removed after install) - │ ├── windninja - │ ├── build - │ ├── - ├── grib2 (can be removed after install) - ├── awsm - ├── smrf - ├── pysnobal - ├── katana - ├── weather_forecast_retrieval (if needed) + ├── config + │ ├── wy2021_awsm_config.ini + | |── wy2021_awsm_config_v2.ini + ├── topo + | |── topo.nc + | |── lidar + | |── wy2020_lidar_snow_depths.nc + ├── station_data + | |── air_temp.csv + ├── code + | ├── windninja (can be removed after install) + | │ ├── windninja + | │ ├── build + | │ ├── + | ├── grib2 (can be removed after install) + | ├── awsm + | ├── smrf + | ├── pysnobal + | ├── katana + | ├── weather_forecast_retrieval (if needed) System Dependencies diff --git a/docs/getting_started/install/index.rst b/docs/getting_started/install/index.rst new file mode 100644 index 0000000..93951ba --- /dev/null +++ b/docs/getting_started/install/index.rst @@ -0,0 +1,8 @@ +Installation +============ + +.. toctree:: + :maxdepth: 1 + + stable_install + full_stack_install \ No newline at end of file diff --git a/docs/getting_started/stable_install.rst b/docs/getting_started/install/stable_install.rst similarity index 81% rename from docs/getting_started/stable_install.rst rename to docs/getting_started/install/stable_install.rst index 66368b7..123318f 100644 --- a/docs/getting_started/stable_install.rst +++ b/docs/getting_started/install/stable_install.rst @@ -1,5 +1,12 @@ -STABLE: Installation with Docker -================================ +Installation with Docker +======================== + +Both the stable and experimental version of AWSM are built into Docker images. While +the code and config files may be different, the way to run AWSM with Docker is the same +between versions. + +Stable +------ The AWSM docker version for ``release-0.10`` is built on an Ubuntu 18.04 image and has all the necessary dependencies installed. To get the stable version of AWSM: @@ -8,6 +15,19 @@ has all the necessary dependencies installed. To get the stable version of AWSM: docker pull usdaarsnwrc/awsm:0.10 +Experimental +------------ + +The experimental AWSM docker is built on a Python 3.6 image and +has all the necessary dependencies installed. To get the latest version of AWSM +or specify a specific release with the release version: + +.. code:: console + + docker pull usdaarsnwrc/awsm:latest + docker pull usdaarsnwrc/awsm:0.11.3 + + Basic Project Folder Structure ------------------------------ From 935d4824f23530e1395fc562f407c775c1414484 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Wed, 9 Jun 2021 10:50:43 -0600 Subject: [PATCH 13/25] getting started docs --- docs/getting_started/create_config.rst | 5 +- .../install/full_stack_install.rst | 23 ++--- docs/getting_started/run_awsm.rst | 86 ++----------------- docs/user_guide/index.rst | 6 +- scripts/awsm | 2 + 5 files changed, 31 insertions(+), 91 deletions(-) diff --git a/docs/getting_started/create_config.rst b/docs/getting_started/create_config.rst index 28ccf57..225a60b 100644 --- a/docs/getting_started/create_config.rst +++ b/docs/getting_started/create_config.rst @@ -13,11 +13,14 @@ At a minimum to get started, the following configuration file will apply all the The required changes are specifying the path to the ``topo.nc`` file, dates to run the model and the location of the csv input data. +See the :doc:`../user_guide/index` for more in depth usage for different configuration scenarios. + + Stable vs. Experimental ----------------------- The configuration file is not compatible between the stable and experimental model versions. SMRF ``v0.10`` -made a large overhaul to the configuration file to become more verbose and clean up options. If using +made a large overhaul to the configuration file to become more verbose and cleaned up options. If using the stable version, the configuration file will be based on the previous `core config`_. .. _core config: https://github.com/USDA-ARS-NWRC/smrf/blob/release-0.9/smrf/framework/CoreConfig.ini diff --git a/docs/getting_started/install/full_stack_install.rst b/docs/getting_started/install/full_stack_install.rst index 2e69b7f..843b242 100644 --- a/docs/getting_started/install/full_stack_install.rst +++ b/docs/getting_started/install/full_stack_install.rst @@ -97,16 +97,6 @@ System dependencies libeccodes-tools \ wget -Environment Setup ------------------ - -Setup a virtual environment and activate. - -.. code:: bash - - python3 -m virtualenv .venv - source .venv/bin/activate - WindNinja ~~~~~~~~~ @@ -180,6 +170,17 @@ will take time as `wgrib2` will perform tests during installation. Model Code ---------- +Environment Setup +~~~~~~~~~~~~~~~~~ + +Setup a virtual environment and activate. + +.. code:: bash + + python3 -m virtualenv .venv + source .venv/bin/activate + + Weather Forecast Retrieval ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -188,7 +189,7 @@ format that SMRF can utilize. .. note:: - Weather forecast retrieval is a dependency of SMRF and should not to be installed from source + Weather forecast retrieval is a dependency of SMRF and does not to be installed from source unless modifying the weather forecast retrieval code. .. code:: bash diff --git a/docs/getting_started/run_awsm.rst b/docs/getting_started/run_awsm.rst index 0dafdb8..b212e12 100644 --- a/docs/getting_started/run_awsm.rst +++ b/docs/getting_started/run_awsm.rst @@ -1,92 +1,22 @@ Run AWSM ======== -After installing SMRF, generating a topo and creating a configuration file, SMRF can be ran. There are -two ways to run SMRF, first is through the ``run_smrf`` command or through the SMRF API. If SMRF is being -used as input to a snow or hydrology model, we recommend to use ``run_smrf`` as it will generate all -the input required. +After installing AWSM, generating a topo and creating a configuration file, AWSM can be ran. There are +two ways to run AWSM, first is through the ``awsm`` command or through the AWSM API. -``run_smrf`` command +``awsm`` command -------------------- To run a full simulation simply run (barring any errors): .. code:: bash - run_smrf + awsm -SMRF API +AWSM API -------- -The ``smrf`` package can also be used as an API, typically to focus on a single variable. There are steps that -SMRF uses to load the data then distribute and usage of the API should follow the same pattern. For example, -below is the function :mod:`run_smrf `. - -.. code:: python - - with SMRF(config) as s: - # load topo data - s.loadTopo() - - # initialize the distribution - s.create_distribution() - - # initialize the outputs if desired - s.initializeOutput() - - # load weather data and station metadata - s.loadData() - - # distribute - s.distributeData() - - -The next example below builds on above and will distribute air temperature and vapor pressure. They can be used to -get the distributed dew point or web bulb temperature. - -.. code:: python - - configFile = 'config.ini' - - with smrf.framework.SMRF(configFile) as s: - - # =================================================================== - # Model setup and initialize - # =================================================================== - - # These are steps that will load the necessary data and initialize - # the framework. Once loaded, this shouldn't need to be re-ran except - # if something major changes - - # load topo data - s.loadTopo() - - # Create the distribution class - s.distribute['air_temp'] = smrf.distribute.air_temp.ta( - s.config['air_temp']) - s.distribute['vapor_pressure'] = smrf.distribute.vapor_pressure.vp( - s.config['vapor_pressure']) - - # load weather data and station metadata - s.loadData() - - # Initialize the distribution - for v in s.distribute: - s.distribute[v].initialize(s.topo, s.data) - - # initialize the outputs if desired - s.initializeOutput() - - # Distribute the data and output - for output_count, t in enumerate(s.date_time): - - s.distribute['air_temp'].distribute(s.data.air_temp.ix[t]) - s.distribute['vapor_pressure'].distribute( - s.data.vapor_pressure.ix[t], - s.distribute['air_temp'].air_temp) - - # output at the frequency and the last time step - if (output_count % s.config['output']['frequency'] == 0) or \ - (output_count == len(s.date_time)): - s.output(t) \ No newline at end of file +The ``awsm`` package can also be used as an API, but is not common practice. The main class +:mod:`AWSM ` has many methods that can be used to tailor a simulation +that deviates from the standard workflow in AWSM. \ No newline at end of file diff --git a/docs/user_guide/index.rst b/docs/user_guide/index.rst index 829391e..b8f595f 100644 --- a/docs/user_guide/index.rst +++ b/docs/user_guide/index.rst @@ -1,3 +1,7 @@ ========== User guide -========== \ No newline at end of file +========== + +.. toctree:: + :maxdepth: 2 + \ No newline at end of file diff --git a/scripts/awsm b/scripts/awsm index 26c64b4..8342a60 100755 --- a/scripts/awsm +++ b/scripts/awsm @@ -6,6 +6,7 @@ from datetime import datetime import sys import os + def run(): ''' awsm is a command line program meant to take a single @@ -23,5 +24,6 @@ def run(): run_awsm(configFile) + if __name__ == '__main__': run() From 24c43ca4198218746633edfaec0e1b2d206d32aa Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Wed, 9 Jun 2021 13:37:35 -0600 Subject: [PATCH 14/25] tk < 0 doc --- docs/user_guide/index.rst | 20 +++++++++++++- docs/user_guide/tk_lt_zero.rst | 48 ++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 docs/user_guide/tk_lt_zero.rst diff --git a/docs/user_guide/index.rst b/docs/user_guide/index.rst index b8f595f..0cffe47 100644 --- a/docs/user_guide/index.rst +++ b/docs/user_guide/index.rst @@ -2,6 +2,24 @@ User guide ========== +The User Guide covers different topic areas in AWSM and will show +how to run specific types of simulations using different inputs, outputs +and/or models. + +- Docker volume mounting and what needs to be set + - permission problems +- Using WindNinja and Katana + - making sure the data is in the right directory +- Input data and removing config sections + - What does ``time_step`` mean + - checking for grid cells in the model domain +- Custom vegetation in the ``topo.nc`` +- References +- Debugging errors with ``debug`` log level +- Lidar assimilation +- restarting + .. toctree:: :maxdepth: 2 - \ No newline at end of file + + tk_lt_zero \ No newline at end of file diff --git a/docs/user_guide/tk_lt_zero.rst b/docs/user_guide/tk_lt_zero.rst new file mode 100644 index 0000000..ecd4e44 --- /dev/null +++ b/docs/user_guide/tk_lt_zero.rst @@ -0,0 +1,48 @@ +iSnobal error: ``tk < 0`` +========================= + +One of the most common errors when running iSnobal is the +``tk < 0`` error. While this error is not very informative, the solution +is quite straightforward. + +Why the error? +-------------- + +The error is occurring because the surface layer's temperature is below 0 degrees +Kelvin. This can happen when the surface energy budget for the timestep is negative +and the snowpack is loosing a large amount of energy (negative energy balance). +The negative energy balance can create a very cold snowpack and drive the temperature +to absolute zero. + +This error typically occurs during cold and very windy conditions over a shallow snowpack, +making the turbulent transfer the largest component of the energy balance. + +How to fix +---------- + +Step 1 +~~~~~~ + +The best method to fix this error is to increase the iSnobal mass thresholds to refine +and stabilize the turbulent transfer calculations. This can be achieved by using the +``thresh_small``, ``thresh_medium`` or ``thresh_normal`` in the ``[ipysnobal]`` section. + + .. note:: + + We recommend to start changing the ``thresh_medium`` to start becasue this will target + the shallow snowpack pixels. + + The default value is ``thresh_medium: 10`` and a good start is to change to + ``thresh_medium: 20``. + +Step 2 +~~~~~~ + +After changing the configuration file, the model will need to be reran. Start the model +a day or few hours before the time step that crashed. Run for a few timesteps or until +the cold and windy condition is complete. + +Step 3 +~~~~~~ + +After the model is through the period, change the thresholds back and continue running. From d4b7703339c8e3c67caca884da8995eadea970c2 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Wed, 9 Jun 2021 15:00:08 -0600 Subject: [PATCH 15/25] user guide for configuration files, added the smrf config file as well --- .gitignore | 1 - docs/conf.py | 15 +- .../install/stable_install.rst | 2 +- docs/getting_started/installation.rst | 1 + docs/user_guide/config/awsm.rst | 263 ++++ docs/user_guide/config/awsm_core_config.rst | 8 + docs/user_guide/config/index.rst | 13 + docs/user_guide/config/smrf.rst | 1397 +++++++++++++++++ docs/user_guide/config/smrf_core_config.rst | 8 + docs/{ => user_guide}/configuration.rst | 32 +- docs/user_guide/debugging.rst | 15 + docs/user_guide/index.rst | 3 + docs/user_guide/tk_lt_zero.rst | 18 + 13 files changed, 1752 insertions(+), 24 deletions(-) create mode 100644 docs/user_guide/config/awsm.rst create mode 100644 docs/user_guide/config/awsm_core_config.rst create mode 100644 docs/user_guide/config/index.rst create mode 100644 docs/user_guide/config/smrf.rst create mode 100644 docs/user_guide/config/smrf_core_config.rst rename docs/{ => user_guide}/configuration.rst (80%) create mode 100644 docs/user_guide/debugging.rst diff --git a/.gitignore b/.gitignore index a823e53..05068ab 100644 --- a/.gitignore +++ b/.gitignore @@ -74,7 +74,6 @@ coverage.xml # Sphinx documentation docs/_build/ -docs/auto_config.rst # PyBuilder target/ diff --git a/docs/conf.py b/docs/conf.py index 76a3150..0ca548f 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,9 +31,18 @@ # Call auto documentation for our config files -config_documentation('./auto_config.rst', - paths=['../awsm/framework/CoreConfig.ini', - '../awsm/framework/recipes.ini']) +config_documentation( + './user_guide/config/awsm.rst', + paths=[ + '../awsm/framework/CoreConfig.ini', + '../awsm/framework/recipes.ini' + ] +) + +config_documentation( + './user_guide/config/smrf.rst', + modules='smrf' +) # Get the project root dir, which is the parent dir of this diff --git a/docs/getting_started/install/stable_install.rst b/docs/getting_started/install/stable_install.rst index 123318f..14d1f0f 100644 --- a/docs/getting_started/install/stable_install.rst +++ b/docs/getting_started/install/stable_install.rst @@ -84,7 +84,7 @@ Station Data Atmospheric Data -~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~ .. code:: console diff --git a/docs/getting_started/installation.rst b/docs/getting_started/installation.rst index 34965c6..17af66a 100644 --- a/docs/getting_started/installation.rst +++ b/docs/getting_started/installation.rst @@ -11,6 +11,7 @@ AWSM was designed to run simulations with SMRF_ and PySnobal_. These are the two main dependencies for AWSM. To install the dependencies: .. code:: bash + python3 -m pip install -r requirements.txt .. _SMRF: https://github.com/USDA-ARS-NWRC/smrf diff --git a/docs/user_guide/config/awsm.rst b/docs/user_guide/config/awsm.rst new file mode 100644 index 0000000..189d12e --- /dev/null +++ b/docs/user_guide/config/awsm.rst @@ -0,0 +1,263 @@ + +For configuration file syntax information please visit http://inicheck.readthedocs.io/en/latest/ + + +awsm master +----------- + +| **model_type** +| Model used to simulate snowpack for given time period. Choose None if not running model +| *Default: None* +| *Type: string* +| *Options:* + *none smrf_ipysnobal ipysnobal* +| + +| **run_smrf** +| Specifies whether or not to run SMRF to distribute forcing data. +| *Default: False* +| *Type: bool* +| + + +paths +----- + +| **basin** +| name of basin to run. i.e. tuolumne or brb. No spaces please +| *Default: None* +| *Type: string* +| + +| **folder_date_style** +| style of date that gets appended to generated folders for each date range that is simulated +| *Default: start_end* +| *Type: string* +| *Options:* + *day start_end* +| + +| **path_dr** +| path to starting drive for AWSM directory structure. This path MUST exist +| *Default: None* +| *Type: criticaldirectory* +| + +| **project_description** +| project description that will be stored in text file in project directory. You cannot use punctuation here because config readers cannot deal with that +| *Default: None* +| *Type: string* +| + +| **project_name** +| name of project that will be used in directory structure. No spaces please. +| *Default: None* +| *Type: string* +| + + +awsm system +----------- + +| **daily_folders** +| seperate daily output folders. Used mainly for shortterm forecasts +| *Default: False* +| *Type: bool* +| + +| **log_level** +| level of information to be logged +| *Default: debug* +| *Type: string* +| *Options:* + *debug info error* +| + +| **log_to_file** +| log to auto generated file or print to screen +| *Default: True* +| *Type: bool* +| + +| **netcdf_output_precision** +| NetCDF variable output precision for float (32-bit) or double (64-bit) +| *Default: float* +| *Type: string* +| *Options:* + *float double* +| + +| **output_frequency** +| frequency of snow model outputs in hours. This is sepreate from SMRF +| *Default: 24* +| *Type: int* +| + +| **run_for_nsteps** +| number of timesteps to run iSnobal. This is optional and mainly used in model crash scenarios +| *Default: None* +| *Type: int* +| + + +update depth +------------ + +| **buffer** +| number of buffer cells for update interpolation of variables +| *Default: 400* +| *Type: int* +| + +| **flight_numbers** +| list of flight number integers to use. Integers start at 1. Default uses all within date range +| *Default: None* +| *Type: int* +| + +| **update** +| should we update depth with LiDAR at given intervals +| *Default: False* +| *Type: bool* +| + +| **update_change_file** +| optional file to save the changes in swe depth and density resulting from a depth update +| *Default: None* +| *Type: filename* +| + +| **update_file** +| netCDF containing depth images and dates for updating +| *Default: None* +| *Type: discretionarycriticalfilename* +| + + +isnobal restart +--------------- + +| **depth_thresh** +| threshold in meters for depth to be removed upon restart. This can help with shallow snowpack that causes crashes +| *Default: 0.05* +| *Type: float* +| + +| **output_folders** +| let the restart procedure know where to look for the previous outputs. If standard then it looks in the same directory but it may look in the previous day output if daily +| *Default: standard* +| *Type: string* +| *Options:* + *standard daily* +| + +| **restart_crash** +| whether or not to restart iSnobal from crashed run +| *Default: False* +| *Type: bool* +| + +| **wyh_restart_output** +| last iSnobal output wyhr. Program will look at output files to find output state at this wyhr for restart +| *Default: None* +| *Type: int* +| + + +ipysnobal +--------- + +| **active_layer** +| height of iSnobal active layer in meters +| *Default: 0.25* +| *Type: float* +| + +| **init_file** +| init file containing model state to initialize snow model +| *Default: None* +| *Type: filename* +| + +| **init_type** +| type of file for initializing model +| *Default: None* +| *Type: string* +| *Options:* + *none netcdf netcdf_out* +| + +| **ithreads** +| numbers threads for running snow model +| *Default: 1* +| *Type: int* +| + +| **mask_isnobal** +| Mask snowpack model output. +| *Default: False* +| *Type: bool* +| + +| **max_h2o** +| maximum volumetric content of liquid water in the snowpack +| *Default: 0.01* +| *Type: float* +| + +| **output_file_name** +| name of the output file +| *Default: ipysnobal* +| *Type: string* +| + +| **restart_date_time** +| Restart iPysnobal at this date time which will keep the output in the same project and run directory. This will be the first time step that iPysnobal will perform. +| *Default: None* +| *Type: datetime* +| + +| **thresh_medium** +| medium mass threshold for timestep refinement within iSnobal +| *Default: 10* +| *Type: int* +| + +| **thresh_normal** +| normal mass threshold for timestep refinement within iSnobal +| *Default: 60* +| *Type: int* +| + +| **thresh_small** +| small mass threshold for timestep refinement within iSnobal +| *Default: 1* +| *Type: int* +| + +| **variables** +| Output variables for Pysnobal +| *Default: thickness snow_density specific_mass liquid_water temperature_surface temperature_lower temperature_snowcover thickness_lower water_saturation net_radiation sensible_heat latent_heat snow_soil precip_advected sum_energy_balance evaporation snowmelt surface_water_input cold_content* +| *Type: string* +| *Options:* + *thickness snow_density specific_mass liquid_water temperature_surface temperature_lower temperature_snowcover thickness_lower water_saturation net_radiation sensible_heat latent_heat snow_soil precip_advected sum_energy_balance evaporation snowmelt surface_water_input cold_content* +| + +| **z_g** +| depth of soil temperature data in meters +| *Default: 0.5* +| *Type: float* +| + +| **z_t** +| height of temperature data in meters +| *Default: 5.0* +| *Type: float* +| + +| **z_u** +| height of wind speed data in meters +| *Default: 5.0* +| *Type: float* +| + diff --git a/docs/user_guide/config/awsm_core_config.rst b/docs/user_guide/config/awsm_core_config.rst new file mode 100644 index 0000000..d095e20 --- /dev/null +++ b/docs/user_guide/config/awsm_core_config.rst @@ -0,0 +1,8 @@ + +AWSM +==== + +The AWSM configuration file is described in detail below. This information is +all based on the CoreConfig file stored under ``awsm.framework.CoreConfig.ini``. + +.. include:: ./awsm.rst \ No newline at end of file diff --git a/docs/user_guide/config/index.rst b/docs/user_guide/config/index.rst new file mode 100644 index 0000000..1037659 --- /dev/null +++ b/docs/user_guide/config/index.rst @@ -0,0 +1,13 @@ +Configuration File Reference +============================ + +The configuration file to run AWSM is an extension of the SMRF configuration. A configuration +file used with AWSM will have sections specific to SMRF and sections specific to AWSM. The +configuration sections get combined into one file to make it easier to keep track of configuration +options. See :doc:`../../getting_started/create_config` for an example of a simple configuration file. + +.. toctree:: + :maxdepth: 2 + + smrf_core_config + awsm_core_config \ No newline at end of file diff --git a/docs/user_guide/config/smrf.rst b/docs/user_guide/config/smrf.rst new file mode 100644 index 0000000..802aee4 --- /dev/null +++ b/docs/user_guide/config/smrf.rst @@ -0,0 +1,1397 @@ + +For configuration file syntax information please visit http://inicheck.readthedocs.io/en/latest/ + + +topo +---- + +| **filename** +| A netCDF file containing all veg info and dem. +| *Default: None* +| *Type: criticalfilename* +| + +| **gradient_method** +| Method to use for calculating the slope and aspect. gradient_d8 uses 3 by 3 finite difference window and gradient_d4 uses a two cell finite difference for x and y +| *Default: gradient_d8* +| *Type: string* +| *Options:* + *gradient_d8 gradient_d4* +| + +| **northern_hemisphere** +| Boolean describing whether the model domain is in the northern hemisphere or not +| *Default: True* +| *Type: bool* +| + +| **sky_view_factor_angles** +| Number of directions to estimate the horizon. Divides 360 degrees into evenly spaced directions. +| *Default: 72* +| *Type: int* +| + + +time +---- + +| **end_date** +| Date and time to end the data distribution that can be parsed by pandas.to_datetime +| *Default: None* +| *Type: datetimeorderedpair* +| + +| **start_date** +| Date and time to start the data distribution that can be parsed by pandas.to_datetime +| *Default: None* +| *Type: datetimeorderedpair* +| + +| **time_step** +| Time interval that SMRF distributes data at in minutes +| *Default: 60* +| *Type: int* +| + +| **time_zone** +| Case sensitive time zone for all times provided and how the model will be run. See pytz docs for information on what is accepted. Full list can be found en.wikipedia.org/wiki/List_of_tz_database_time_zones +| *Default: UTC* +| *Type: rawstring* +| + + +csv +--- + +| **air_temp** +| Path to CSV containing the station measured air temperature +| *Default: None* +| *Type: criticalfilename* +| + +| **cloud_factor** +| Path to CSV containing the station measured cloud factor +| *Default: None* +| *Type: criticalfilename* +| + +| **metadata** +| Path to CSV containing the station metadata +| *Default: None* +| *Type: criticalfilename* +| + +| **precip** +| Path to CSV containing the station measured precipitation +| *Default: None* +| *Type: criticalfilename* +| + +| **stations** +| List of station IDs to use for distributing any of the variables +| *Default: None* +| *Type: station* +| + +| **vapor_pressure** +| Path to CSV containing the station measured vapor pressure +| *Default: None* +| *Type: criticalfilename* +| + +| **wind_direction** +| Path to CSV containing the station measured wind direction +| *Default: None* +| *Type: criticalfilename* +| + +| **wind_speed** +| Path to CSV containing the station measured wind speed +| *Default: None* +| *Type: criticalfilename* +| + + +gridded +------- + +| **data_type** +| Type of gridded input data +| *Default: hrrr_grib* +| *Type: string* +| *Options:* + *wrf hrrr_grib netcdf* +| + +| **hrrr_directory** +| Path to the top level directory where multiple HRRR gridded dataset are located +| *Default: None* +| *Type: criticaldirectory* +| + +| **hrrr_load_method** +| Method to load the HRRR data either load all data first or for each timestep +| *Default: first* +| *Type: string* +| *Options:* + *first timestep* +| + +| **netcdf_file** +| Path to the netCDF file containing weather data +| *Default: None* +| *Type: criticalfilename* +| + +| **wrf_file** +| Path to the netCDF file containing WRF data +| *Default: None* +| *Type: criticalfilename* +| + + +air_temp +-------- + +| **detrend** +| Whether to elevationally detrend prior to distributing +| *Default: true* +| *Type: bool* +| + +| **detrend_slope** +| If detrend is true constrain the detrend_slope to positive (1) or negative (-1) or no constraint (0) +| *Default: -1* +| *Type: int* +| *Options:* + *-1 0 1* +| + +| **distribution** +| Distribution method to use for . Stations use dk idw or kriging. Gridded data use grid. Stations use dk idw or kriging. Gridded data use grid. +| *Default: idw* +| *Type: string* +| *Options:* + *dk idw grid kriging* +| + +| **dk_ncores** +| Number of threads or processors to use in the dk calculation +| *Default: 1* +| *Type: int* +| + +| **grid_local** +| Use local elevation gradients in gridded interpolation +| *Default: False* +| *Type: bool* +| + +| **grid_local_n** +| number of closest grid cells to use for calculating elevation gradient +| *Default: 25* +| *Type: int* +| + +| **grid_mask** +| Mask the distribution calculations +| *Default: True* +| *Type: bool* +| + +| **grid_method** +| Gridded interpolation method to use for air temperature +| *Default: cubic* +| *Type: string* +| *Options:* + *nearest linear cubic* +| + +| **idw_power** +| Power for decay of a stations influence in inverse distance weighting. +| *Default: 2.0* +| *Type: float* +| + +| **krig_anisotropy_angle** +| CCW angle (in degrees) by which to rotate coordinate system in order to take into account anisotropy. +| *Default: 0.0* +| *Type: float* +| + +| **krig_anisotropy_scaling** +| Scalar stretching value for kriging to take into account anisotropy. +| *Default: 1.0* +| *Type: float* +| + +| **krig_coordinates_type** +| Determines if the x and y coordinates are interpreted as on a plane (euclidean) or as coordinates on a sphere (geographic). +| *Default: euclidean* +| *Type: string* +| *Options:* + *euclidean geographic* +| + +| **krig_nlags** +| Number of averaging bins for the kriging semivariogram +| *Default: 6* +| *Type: int* +| + +| **krig_variogram_model** +| Specifies which kriging variogram model to use +| *Default: linear* +| *Type: string* +| *Options:* + *linear power gaussian spherical exponential hole-effect* +| + +| **krig_weight** +| Flag that specifies if the kriging semivariance at smaller lags should be weighted more heavily when automatically calculating variogram model. +| *Default: False* +| *Type: bool* +| + +| **max** +| Maximum possible value for air temperature in Celsius +| *Default: 47.0* +| *Type: float* +| + +| **min** +| Minimum possible value for air temperature in Celsius +| *Default: -73.0* +| *Type: float* +| + +| **stations** +| Stations to use for distributing air temperature +| *Default: None* +| *Type: station* +| + + +vapor_pressure +-------------- + +| **detrend** +| Whether to elevationally detrend prior to distributing +| *Default: true* +| *Type: bool* +| + +| **detrend_slope** +| If detrend is true constrain the slope to positive (1) or negative (-1) or no constraint (0) +| *Default: -1* +| *Type: int* +| *Options:* + *-1 0 1* +| + +| **dew_point_nthreads** +| Number of threads to use in the dew point calculation +| *Default: 2* +| *Type: int* +| + +| **dew_point_tolerance** +| Solving criteria for the dew point calculation +| *Default: 0.01* +| *Type: float* +| + +| **distribution** +| Distribution method to use for vapor pressure. Stations use dk idw or kriging. Gridded data use grid. +| *Default: idw* +| *Type: string* +| *Options:* + *dk idw grid kriging* +| + +| **dk_ncores** +| Number of threads to use in the dk calculation +| *Default: 1* +| *Type: int* +| + +| **grid_local** +| Use local elevation gradients in gridded interpolation +| *Default: False* +| *Type: bool* +| + +| **grid_local_n** +| number of closest grid cells to use for calculating elevation gradient +| *Default: 25* +| *Type: int* +| + +| **grid_mask** +| Mask the distribution calculations +| *Default: True* +| *Type: bool* +| + +| **grid_method** +| interpolation method to use for this variable +| *Default: cubic* +| *Type: string* +| *Options:* + *nearest linear cubic* +| + +| **idw_power** +| Power for decay of a stations influence in inverse distance weighting +| *Default: 2.0* +| *Type: float* +| + +| **krig_anisotropy_angle** +| CCW angle (in degrees) by which to rotate coordinate system in order to take into account anisotropy. +| *Default: 0.0* +| *Type: float* +| + +| **krig_anisotropy_scaling** +| Scalar stretching value for kriging to take into account anisotropy. +| *Default: 1.0* +| *Type: float* +| + +| **krig_coordinates_type** +| Determines if the x and y coordinates are interpreted as on a plane (euclidean) or as coordinates on a sphere (geographic). +| *Default: euclidean* +| *Type: string* +| *Options:* + *euclidean geographic* +| + +| **krig_nlags** +| Number of averaging bins for the kriging semivariogram +| *Default: 6* +| *Type: int* +| + +| **krig_variogram_model** +| Specifies which kriging variogram model to use +| *Default: linear* +| *Type: string* +| *Options:* + *linear power gaussian spherical exponential hole-effect* +| + +| **krig_weight** +| Flag that specifies if the kriging semivariance at smaller lags should be weighted more heavily when automatically calculating variogram model. +| *Default: False* +| *Type: bool* +| + +| **max** +| Maximum possible vapor pressure in Pascals +| *Default: 5000.0* +| *Type: float* +| + +| **min** +| Minimum possible vapor pressure in Pascals +| *Default: 20.0* +| *Type: float* +| + +| **stations** +| Stations to use for distributing vapor pressure in Pascals +| *Default: None* +| *Type: station* +| + + +wind +---- + +| **detrend** +| Whether to elevationally detrend prior to distributing +| *Default: False* +| *Type: bool* +| + +| **detrend_slope** +| if detrend is true constrain the detrend_slope to positive (1) or negative (-1) or no constraint (0) +| *Default: 1* +| *Type: int* +| *Options:* + *-1 0 1* +| + +| **distribution** +| Distribution method to use for wind. Stations use dk idw or kriging. Gridded data use grid. +| *Default: idw* +| *Type: string* +| *Options:* + *dk idw grid kriging* +| + +| **dk_ncores** +| Number of threads to use in the dk calculation +| *Default: 2* +| *Type: int* +| + +| **grid_local** +| Use local elevation gradients in gridded interpolation +| *Default: False* +| *Type: bool* +| + +| **grid_local_n** +| Number of closest grid cells to use for calculating elevation gradient +| *Default: 25* +| *Type: int* +| + +| **grid_mask** +| Mask the distribution calculations +| *Default: True* +| *Type: bool* +| + +| **grid_method** +| interpolation method to use for wind +| *Default: linear* +| *Type: string* +| *Options:* + *nearest linear cubic* +| + +| **idw_power** +| Power for decay of a stations influence in inverse distance weighting +| *Default: 2.0* +| *Type: float* +| + +| **krig_anisotropy_angle** +| CCW angle (in degrees) by which to rotate coordinate system in order to take into account anisotropy. +| *Default: 0.0* +| *Type: float* +| + +| **krig_anisotropy_scaling** +| Scalar stretching value for kriging to take into account anisotropy. +| *Default: 1.0* +| *Type: float* +| + +| **krig_coordinates_type** +| Determines if the x and y coordinates are interpreted as on a plane (euclidean) or as coordinates on a sphere (geographic). +| *Default: euclidean* +| *Type: string* +| *Options:* + *euclidean geographic* +| + +| **krig_nlags** +| Number of averaging bins for the kriging semivariogram +| *Default: 6* +| *Type: int* +| + +| **krig_variogram_model** +| Specifies which kriging variogram model to use +| *Default: linear* +| *Type: string* +| *Options:* + *linear power gaussian spherical exponential hole-effect* +| + +| **krig_weight** +| Flag that specifies if the kriging semivariance at smaller lags should be weighted more heavily when automatically calculating variogram model. +| *Default: False* +| *Type: bool* +| + +| **max** +| Maximum possible wind in M/s +| *Default: 35.0* +| *Type: float* +| + +| **maxus_netcdf** +| NetCDF file containing the maxus values for wind +| *Default: None* +| *Type: criticalfilename* +| + +| **min** +| Minimum possible for wind in M/s +| *Default: 0.447* +| *Type: float* +| + +| **reduction_factor** +| If wind speeds are still off here is a scaling factor +| *Default: 1.0* +| *Type: float* +| + +| **station_default** +| Account for sheltered station wind measurements for example 11.4 equates to a small forest opening and 0 equates to unsheltered measurements. +| *Default: 11.4* +| *Type: float* +| + +| **station_peak** +| Name of stations that lie on a peak or a high point +| *Default: None* +| *Type: station* +| + +| **stations** +| Stations to use for distributing wind in M/s +| *Default: None* +| *Type: station* +| + +| **veg_3011** +| Applies the value where vegetation equals 3011(Rocky Mountain aspen) +| *Default: 3.3* +| *Type: float* +| + +| **veg_3061** +| Applies the value where vegetation equals 3061(mixed aspen) +| *Default: 3.3* +| *Type: float* +| + +| **veg_41** +| Applies the value where vegetation type equals NLCD class 41 +| *Default: 3.3* +| *Type: float* +| + +| **veg_42** +| Applies the value where vegetation type equals NLCD class 42 +| *Default: 3.3* +| *Type: float* +| + +| **veg_43** +| Applies the value where vegetation type equals NLCD class 43 +| *Default: 11.4* +| *Type: float* +| + +| **veg_default** +| Applies the value to all vegetation not specified +| *Default: 0.0* +| *Type: float* +| + +| **wind_model** +| Wind model to interpolate wind measurements to the model domain +| *Default: winstral* +| *Type: string* +| *Options:* + *winstral wind_ninja interp* +| + +| **wind_ninja_dir** +| Location in which the ascii files are output from the WindNinja simulation. This serves as a trigger for checking for WindNinja files. +| *Default: None* +| *Type: criticaldirectory* +| + +| **wind_ninja_dxdy** +| grid spacing at which the WindNinja ascii files are output. +| *Default: 100* +| *Type: int* +| + +| **wind_ninja_height** +| The output height of wind fields from WindNinja in meters. +| *Default: 5.0* +| *Type: string* +| + +| **wind_ninja_pref** +| Prefix of all outputs from WindNinja that matches the topo input to WindNinja. +| *Default: None* +| *Type: string* +| + +| **wind_ninja_roughness** +| The surface roughness used in WindNinja generally grass. +| *Default: 0.01* +| *Type: string* +| + +| **wind_ninja_tz** +| Time zone that from the WindNinja config. +| *Default: UTC* +| *Type: string* +| + + +precip +------ + +| **detrend** +| Whether to elevationally detrend prior to distributing +| *Default: true* +| *Type: bool* +| + +| **detrend_slope** +| if detrend is true constrain the detrend_slope to positive (1) or negative (-1) or no constraint (0) +| *Default: 1* +| *Type: int* +| *Options:* + *-1 0 1* +| + +| **distribution** +| Distribution method to use for precipitation. Stations use dk idw or kriging. Gridded data use grid. +| *Default: dk* +| *Type: string* +| *Options:* + *dk idw grid kriging* +| + +| **dk_ncores** +| Number of threads to use in the dk calculation +| *Default: 2* +| *Type: int* +| + +| **grid_local** +| Use local elevation gradients in gridded interpolation +| *Default: False* +| *Type: bool* +| + +| **grid_local_n** +| number of closest grid cells to use for calculating elevation gradient +| *Default: 25* +| *Type: int* +| + +| **grid_mask** +| Mask the distribution calculations +| *Default: True* +| *Type: bool* +| + +| **grid_method** +| interpolation method to use for precipitation +| *Default: cubic* +| *Type: string* +| *Options:* + *nearest linear cubic* +| + +| **idw_power** +| Power for decay of a stations influence in inverse distance weighting +| *Default: 2.0* +| *Type: float* +| + +| **krig_anisotropy_angle** +| CCW angle (in degrees) by which to rotate coordinate system in order to take into account anisotropy. +| *Default: 0.0* +| *Type: float* +| + +| **krig_anisotropy_scaling** +| Scalar stretching value for kriging to take into account anisotropy. +| *Default: 1.0* +| *Type: float* +| + +| **krig_coordinates_type** +| Determines if the x and y coordinates are interpreted as on a plane (euclidean) or as coordinates on a sphere (geographic). +| *Default: euclidean* +| *Type: string* +| *Options:* + *euclidean geographic* +| + +| **krig_nlags** +| Number of averaging bins for the kriging semivariogram +| *Default: 6* +| *Type: int* +| + +| **krig_variogram_model** +| Specifies which kriging variogram model to use +| *Default: linear* +| *Type: string* +| *Options:* + *linear power gaussian spherical exponential hole-effect* +| + +| **krig_weight** +| Flag that specifies if the kriging semivariance at smaller lags should be weighted more heavily when automatically calculating variogram model. +| *Default: False* +| *Type: bool* +| + +| **marks2017_timesteps_to_end_storms** +| number of timesteps to elapse with precip under start criteria before ending a storm. +| *Default: 6* +| *Type: int* +| + +| **max** +| Maximum possible precipitation in millimeters +| *Default: None* +| *Type: float* +| + +| **min** +| Minimum possible for precipitation in millimeters +| *Default: 0.0* +| *Type: float* +| + +| **new_snow_density_model** +| Method to use for calculating the new snow density +| *Default: susong1999* +| *Type: string* +| *Options:* + *marks2017 susong1999 piecewise_susong1999* +| + +| **precip_rescaling_model** +| Method to use for redistributing precipitation. Winstrals method focuses forming drifts from wind +| *Default: None* +| *Type: string* +| *Options:* + *winstral* +| + +| **precip_temp_method** +| which variable to use for precip temperature +| *Default: dew_point* +| *Type: string* +| *Options:* + *dew_point wet_bulb* +| + +| **station_adjust_for_undercatch** +| Apply undercatch relationships to precip gauges +| *Default: true* +| *Type: bool* +| + +| **station_undercatch_model_default** +| WMO model used to adjust for undercatch of precipitaiton +| *Default: us_nws_8_shielded* +| *Type: string* +| *Options:* + *us_nws_8_shielded us_nws_8_unshielded* +| + +| **stations** +| Stations to use for distributing this precipitation +| *Default: None* +| *Type: station* +| + +| **storm_days_restart** +| Path to netcdf representing the last storm days so a run can continue in between stops +| *Default: None* +| *Type: discretionarycriticalfilename* +| + +| **storm_mass_threshold** +| Start criteria for a storm in mm of measured precipitation in millimeters in any pixel over the domain. +| *Default: 1.0* +| *Type: float* +| + +| **susong1999_timesteps_to_end_storms** +| number of timesteps to elapse with precip under start criteria before ending a storm. +| *Default: 6* +| *Type: int* +| + +| **winstral_max_drift** +| max multiplier for precip redistribution in a drift cell +| *Default: 3.5* +| *Type: float* +| + +| **winstral_max_scour** +| max multiplier for precip redistribution to account for wind scour. +| *Default: 1.0* +| *Type: float* +| + +| **winstral_min_drift** +| min multiplier for precip redistribution in a drift cell +| *Default: 1.0* +| *Type: float* +| + +| **winstral_min_scour** +| minimum multiplier for precip redistribution to account for wind scour. +| *Default: 0.55* +| *Type: float* +| + +| **winstral_tbreak_netcdf** +| NetCDF file containing the tbreak values for wind +| *Default: None* +| *Type: filename* +| + +| **winstral_tbreak_threshold** +| Threshold for drift cells measured in degrees from tbreak file. +| *Default: 7.0* +| *Type: float* +| + +| **winstral_veg_3011** +| Interference inverse factor for precip redistribution where vegetation equals 3011(Rocky Mountain Aspen). +| *Default: 0.7* +| *Type: float* +| + +| **winstral_veg_3061** +| Interference inverse factor for precip redistribution where vegetation equals 3061(Mixed Aspen). +| *Default: 0.7* +| *Type: float* +| + +| **winstral_veg_41** +| Interference inverse factor for precip redistribution where vegetation equals 41. +| *Default: 0.7* +| *Type: float* +| + +| **winstral_veg_42** +| Interference inverse factor for precip redistribution where vegetation equals 42. +| *Default: 0.7* +| *Type: float* +| + +| **winstral_veg_43** +| Interference inverse factor for precip redistribution where vegetation equals 43. +| *Default: 0.7* +| *Type: float* +| + +| **winstral_veg_default** +| Applies the value to all vegetation not specified +| *Default: 1.0* +| *Type: float* +| + + +albedo +------ + +| **date_method_decay_power** +| Exponent value of the decay rate equation prescribed by the method. +| *Default: 0.714* +| *Type: float* +| + +| **date_method_end_decay** +| Starting date for applying the decay method described by date_method +| *Default: None* +| *Type: datetimeorderedpair* +| + +| **date_method_start_decay** +| Starting date for applying the decay method described by date_method +| *Default: None* +| *Type: datetimeorderedpair* +| + +| **date_method_veg_41** +| Applies the value where vegetation equals 41 +| *Default: 0.36* +| *Type: float* +| + +| **date_method_veg_42** +| Applies the value where vegetation equals 42 +| *Default: 0.36* +| *Type: float* +| + +| **date_method_veg_43** +| Applies the value where vegetation equals 43 +| *Default: 0.25* +| *Type: float* +| + +| **date_method_veg_default** +| Applies the value to all vegetation not specified +| *Default: 0.25* +| *Type: float* +| + +| **decay_method** +| Describe how the albedo decays in the late season +| *Default: None* +| *Type: string* +| *Options:* + * hardy2000 date_method none* +| + +| **dirt** +| Effective contamination for adjustment to visible albedo (usually between 1.5-3.0) +| *Default: 2.0* +| *Type: float* +| + +| **grain_size** +| Effective optical grain radius of snow after last storm in micro-meters +| *Default: 100.0* +| *Type: float* +| + +| **grid_mask** +| Mask the distribution calculations +| *Default: True* +| *Type: bool* +| + +| **hardy2000_litter_albedo** +| Albedo of the litter on the snow using the hard method +| *Default: 0.2* +| *Type: float* +| + +| **hardy2000_litter_default** +| Litter rate for places where vegetation not specified for Hardy et al. 2000 decay method +| *Default: 0.003* +| *Type: float* +| + +| **hardy2000_litter_veg_41** +| Litter rate for places where vegetation not specified for Hardy et al. 2000 decay method for vegetation classes NLCD 41 +| *Default: 0.006* +| *Type: float* +| + +| **hardy2000_litter_veg_42** +| Litter rate for places where vegetation not specified for Hardy et al. 2000 decay method for vegetation classes NLCD 42 +| *Default: 0.006* +| *Type: float* +| + +| **hardy2000_litter_veg_43** +| Litter rate for places where vegetation not specified for Hardy et al. 2000 decay method for vegetation classes NLCD 43 +| *Default: 0.003* +| *Type: float* +| + +| **max** +| Maximum possible for albedo +| *Default: 1.0* +| *Type: float* +| + +| **max_grain** +| Max optical grain radius of snow possible in micro-meters +| *Default: 700.0* +| *Type: float* +| + +| **min** +| Minimum possible for albedo +| *Default: 0.0* +| *Type: float* +| + + +cloud_factor +------------ + +| **detrend** +| Whether to elevationally detrend prior to distributing +| *Default: false* +| *Type: bool* +| + +| **detrend_slope** +| If detrend is true constrain the detrend_slope to positive (1) or negative (-1) or no constraint (0) +| *Default: 0* +| *Type: int* +| *Options:* + *-1 0 1* +| + +| **distribution** +| Distribution method to use for cloud factor. Stations use dk idw or kriging. Gridded data use grid. Stations use dk idw or kriging. Gridded data use grid. +| *Default: idw* +| *Type: string* +| *Options:* + *dk idw grid kriging* +| + +| **dk_ncores** +| Number of threads or processors to use in the dk calculation +| *Default: 1* +| *Type: int* +| + +| **grid_local** +| Use local elevation gradients in gridded interpolation +| *Default: False* +| *Type: bool* +| + +| **grid_local_n** +| number of closest grid cells to use for calculating elevation gradient +| *Default: 25* +| *Type: int* +| + +| **grid_mask** +| Mask the distribution calculations +| *Default: True* +| *Type: bool* +| + +| **grid_method** +| Gridded interpolation method to use for cloud factor +| *Default: cubic* +| *Type: string* +| *Options:* + *nearest linear cubic* +| + +| **idw_power** +| Power for decay of a stations influence in inverse distance weighting. +| *Default: 2.0* +| *Type: float* +| + +| **krig_anisotropy_angle** +| CCW angle (in degrees) by which to rotate coordinate system in order to take into account anisotropy. +| *Default: 0.0* +| *Type: float* +| + +| **krig_anisotropy_scaling** +| Scalar stretching value for kriging to take into account anisotropy. +| *Default: 1.0* +| *Type: float* +| + +| **krig_coordinates_type** +| Determines if the x and y coordinates are interpreted as on a plane (euclidean) or as coordinates on a sphere (geographic). +| *Default: euclidean* +| *Type: string* +| *Options:* + *euclidean geographic* +| + +| **krig_nlags** +| Number of averaging bins for the kriging semivariogram +| *Default: 6* +| *Type: int* +| + +| **krig_variogram_model** +| Specifies which kriging variogram model to use +| *Default: linear* +| *Type: string* +| *Options:* + *linear power gaussian spherical exponential hole-effect* +| + +| **krig_weight** +| Flag that specifies if the kriging semivariance at smaller lags should be weighted more heavily when automatically calculating variogram model. +| *Default: False* +| *Type: bool* +| + +| **max** +| Max prossible cloud factor as a decimal representing full clouds (0) to full sun (1). +| *Default: 1.0* +| *Type: float* +| + +| **min** +| Minimum possible cloud factor as a decimal representing full clouds (0) to full sun (1). +| *Default: 0.0* +| *Type: float* +| + +| **stations** +| Stations to use for distributing cloud factor as a decimal representing full clouds (0) to full sun (1). +| *Default: None* +| *Type: station* +| + + +solar +----- + +| **clear_gamma** +| Scattering asymmetry parameter +| *Default: 0.3* +| *Type: float* +| + +| **clear_omega** +| Single-scattering albedo +| *Default: 0.85* +| *Type: float* +| + +| **clear_opt_depth** +| Elevation of optical depth measurement +| *Default: 100.0* +| *Type: float* +| + +| **clear_tau** +| Optical depth at z +| *Default: 0.2* +| *Type: float* +| + +| **correct_albedo** +| Multiply the solar radiation by 1-snow_albedo. +| *Default: true* +| *Type: bool* +| + +| **correct_cloud** +| Multiply the solar radiation by the cloud factor derived by station data. +| *Default: true* +| *Type: bool* +| + +| **correct_veg** +| Apply solar radiation corrections according to veg_type +| *Default: true* +| *Type: bool* +| + +| **max** +| Maximum possible solar radiation in W/m^2 +| *Default: 800.0* +| *Type: float* +| + +| **min** +| Minimum possible solar radiation in W/m^2 +| *Default: 0.0* +| *Type: float* +| + + +thermal +------- + +| **clear_sky_method** +| Method for calculating the clear sky thermal radiation +| *Default: marks1979* +| *Type: string* +| *Options:* + *marks1979 dilley1998 prata1996 angstrom1918* +| + +| **cloud_method** +| Method for adjusting thermal radiation due to cloud effects +| *Default: garen2005* +| *Type: string* +| *Options:* + *garen2005 unsworth1975 kimball1982 crawford1999* +| + +| **correct_cloud** +| Specify whether to use the cloud adjustments in thermal calculation +| *Default: true* +| *Type: bool* +| + +| **correct_terrain** +| Specify whether to account for vegetation in the thermal calculations +| *Default: true* +| *Type: bool* +| + +| **correct_veg** +| Specify whether to account for vegetation in the thermal calculations +| *Default: true* +| *Type: bool* +| + +| **detrend** +| Whether to elevationally the detrend prior to distributing +| *Default: False* +| *Type: bool* +| + +| **detrend_slope** +| if detrend is true constrain the detrend_slope to positive (1) or negative (-1) or no constraint (0) +| *Default: 0* +| *Type: int* +| *Options:* + *-1 0 1* +| + +| **distribution** +| Distribution method to use for incoming thermal when using HRRR input data. +| *Default: grid* +| *Type: string* +| *Options:* + *grid* +| + +| **grid_local** +| Use local elevation gradients in gridded interpolation +| *Default: False* +| *Type: bool* +| + +| **grid_local_n** +| number of closest grid cells to use for calculating elevation gradient +| *Default: 25* +| *Type: int* +| + +| **grid_mask** +| Mask the thermal radiation calculations +| *Default: True* +| *Type: bool* +| + +| **grid_method** +| interpolation method to use for this variable +| *Default: cubic* +| *Type: string* +| *Options:* + *nearest linear cubic* +| + +| **marks1979_nthreads** +| Number of threads to use thermal radiation calcs when using Marks1979 +| *Default: 2* +| *Type: int* +| + +| **max** +| Maximum possible incoming thermal radiation in W/m^2 +| *Default: 600.0* +| *Type: float* +| + +| **min** +| Minimum possible incoming thermal radiation in W/m^2 +| *Default: 0.0* +| *Type: float* +| + + +soil_temp +--------- + +| **temp** +| Constant value to use for the soil temperature. +| *Default: -2.5* +| *Type: float* +| + + +output +------ + +| **file_type** +| Format to use for outputting data. +| *Default: netcdf* +| *Type: string* +| *Options:* + *netcdf* +| + +| **frequency** +| Number of timesteps between output values. 1 is every timestep. +| *Default: 1* +| *Type: int* +| + +| **input_backup** +| Specify whether to backup the input data and create config file to run the smrf run from that backup +| *Default: true* +| *Type: bool* +| + +| **mask_output** +| Mask the final NetCDF output. +| *Default: False* +| *Type: bool* +| + +| **netcdf_output_precision** +| NetCDF variable output precision for float (32-bit) or double (64-bit) +| *Default: float* +| *Type: string* +| *Options:* + *float double* +| + +| **out_location** +| Directory to output results +| *Default: None* +| *Type: directory* +| + +| **variables** +| Variables to output after being calculated. +| *Default: thermal air_temp vapor_pressure wind_speed wind_direction net_solar precip percent_snow snow_density precip_temp* +| *Type: string* +| *Options:* + *all air_temp albedo_vis albedo_ir precip percent_snow snow_density storm_days precip_temp clear_ir_beam clear_ir_diffuse clear_vis_beam clear_vis_diffuse cloud_factor cloud_ir_beam cloud_ir_diffuse cloud_vis_beam cloud_vis_diffuse net_solar veg_ir_beam veg_ir_diffuse veg_vis_beam veg_vis_diffuse thermal vapor_pressure dew_point flatwind wind_speed wind_direction thermal_clear thermal_veg thermal_cloud* +| + + +system +------ + +| **log_file** +| File path to a txt file for the log info to be outputted +| *Default: None* +| *Type: filename* +| + +| **log_level** +| level of information to be logged +| *Default: debug* +| *Type: string* +| *Options:* + *debug info error* +| + +| **qotw** +| +| *Default: false* +| *Type: bool* +| + +| **queue_max_values** +| How many timesteps that a calculation can get ahead while threading if it is independent of other variables. +| *Default: 2* +| *Type: int* +| + +| **threading** +| Specify whether to use python threading in calculations. +| *Default: true* +| *Type: bool* +| + +| **time_out** +| Amount of time to wait for a thread before timing out +| *Default: None* +| *Type: float* +| + diff --git a/docs/user_guide/config/smrf_core_config.rst b/docs/user_guide/config/smrf_core_config.rst new file mode 100644 index 0000000..06ca3a4 --- /dev/null +++ b/docs/user_guide/config/smrf_core_config.rst @@ -0,0 +1,8 @@ + +SMRF +==== + +The SMRF configuration file is described in detail below. This information is +all based on the CoreConfig file stored under ``smrf.framework.CoreConfig.ini``. + +.. include:: ./smrf.rst \ No newline at end of file diff --git a/docs/configuration.rst b/docs/user_guide/configuration.rst similarity index 80% rename from docs/configuration.rst rename to docs/user_guide/configuration.rst index 65e782b..332fe6f 100644 --- a/docs/configuration.rst +++ b/docs/user_guide/configuration.rst @@ -1,6 +1,3 @@ -.. _using-configs: - -========================= Using Configuration Files ========================= @@ -26,16 +23,18 @@ Understanding Configuration Files ---------------------------------- The easiest way to get started is to look at one of the config files -in the repo already. A simple case to use is our reynolds mountain east test -which can be view easily here_. - -.. _here: https://github.com/USDA-ARS-NWRC/awsm/blob/devel/tests/RME/config.ini +in the repo already (:doc:`../getting_started/create_config`). Take a look at the "topo" section from the config file show below -.. literalinclude:: ../tests/RME/config.ini - :lines: 20-29 - :language: none +.. code:: + + ################################################################################ + # Files for DEM and vegetation + ################################################################################ + + [topo] + filename: ./topo/topo.nc This section describes all the topographic information required for AWSM to run. At the top of the section there is comment that describes the section. @@ -45,13 +44,10 @@ underneath are assigned values by using the colon. Editing/Checking Configuration Files ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Use any text editor to make changes to a config file. We like to use atom_ with -the .ini syntax package installed. - -.. _atom: https://atom.io/ +Use any text editor to make changes to a config file. If you are unsure of what to use various entries in your config file refer to -the :ref:`config-file-reference` or use the inicheck command for command line help. +the :doc:`config/index` or use the inicheck command for command line help. Below is an example of how to use the inicheck details option to figure out what options are available for the topo section type item. @@ -87,14 +83,12 @@ populated list of options. This will create a config file using the same name but call "config_full.ini" at the end. -.. _core-config: - Core Configuration File ----------------------- Each configuration file is checked against the core configuration file stored -./awsm/framework/core_config.ini and various scenarios are guided by the a recipes -file that is stored in ./awsm/framework/recipes.ini. These files work together +``./awsm/framework/core_config.ini`` and various scenarios are guided by the a recipes +file that is stored in ``./awsm/framework/recipes.ini``. These files work together to guide the outcomes of the configuration file. To learn more about syntax and how to contribute to a Core or Master configuration diff --git a/docs/user_guide/debugging.rst b/docs/user_guide/debugging.rst new file mode 100644 index 0000000..5274780 --- /dev/null +++ b/docs/user_guide/debugging.rst @@ -0,0 +1,15 @@ +AWSM Debugging +============== + +While many errors can be handled with AWSM, sometimes not everything is caught. The +following are a few suggestions on debugging an AWSM simulation. + +Configuration file issues +------------------------- + +A common issue are problems with the configuration file. + +Logfile verbosity +----------------- + +Logfile \ No newline at end of file diff --git a/docs/user_guide/index.rst b/docs/user_guide/index.rst index 0cffe47..58f8e6d 100644 --- a/docs/user_guide/index.rst +++ b/docs/user_guide/index.rst @@ -22,4 +22,7 @@ and/or models. .. toctree:: :maxdepth: 2 + configuration + config/index + debugging tk_lt_zero \ No newline at end of file diff --git a/docs/user_guide/tk_lt_zero.rst b/docs/user_guide/tk_lt_zero.rst index ecd4e44..93a758c 100644 --- a/docs/user_guide/tk_lt_zero.rst +++ b/docs/user_guide/tk_lt_zero.rst @@ -35,6 +35,13 @@ and stabilize the turbulent transfer calculations. This can be achieved by using The default value is ``thresh_medium: 10`` and a good start is to change to ``thresh_medium: 20``. +.. code:: + + [ipysnobal] + thresh_normal: 60 + thresh_medium: 20 + thresh_small: 1 + Step 2 ~~~~~~ @@ -42,7 +49,18 @@ After changing the configuration file, the model will need to be reran. Start th a day or few hours before the time step that crashed. Run for a few timesteps or until the cold and windy condition is complete. +.. code:: + + awsm config.ini + Step 3 ~~~~~~ After the model is through the period, change the thresholds back and continue running. + +.. code:: + + [ipysnobal] + thresh_normal: 60 + thresh_medium: 10 + thresh_small: 1 \ No newline at end of file From b709b20cd24f21d8eac7b16e050e2d470efe659e Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Thu, 10 Jun 2021 10:01:26 -0600 Subject: [PATCH 16/25] debugging docs --- docs/getting_started/create_config.rst | 2 +- docs/user_guide/debugging.rst | 51 +++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/docs/getting_started/create_config.rst b/docs/getting_started/create_config.rst index 225a60b..ff3a533 100644 --- a/docs/getting_started/create_config.rst +++ b/docs/getting_started/create_config.rst @@ -3,7 +3,7 @@ Create a config file After the topo file has been created, build the AWSM configuration file. For in depth documentation see how to :doc:`use a configuration file <../user_guide/configuration>` and the -:doc:`core configuration <../user_guide/core_config>` for all AWSM options. +:doc:`core configuration <../user_guide/config/index>` for all AWSM options. .. note:: diff --git a/docs/user_guide/debugging.rst b/docs/user_guide/debugging.rst index 5274780..0b2a016 100644 --- a/docs/user_guide/debugging.rst +++ b/docs/user_guide/debugging.rst @@ -7,9 +7,56 @@ following are a few suggestions on debugging an AWSM simulation. Configuration file issues ------------------------- -A common issue are problems with the configuration file. +A common issue are problems with the configuration file. There are many options that +can be changed/set and various combinations changing what is required. At the beginning of each +simulation, AWSM will check the configuration file for any issues. If there are any, it will +display the warnings or errors in the console. + +An example for a warning, the AWSM simulation will continue but ensure that the warnings +are not impacting the simulation. The below warning is a great example where AWSM is warning +that the log file doesn't exist, which is okay because AWSM will create the log file automatically. + +.. code:: bash + + Configuration File Status Report: + ========================================================================================== + WARNINGS: + + Section Item Message + ------------------------------------------------------------------------------------------ + system log_file File does not exist. + + +When an error in the configuration file occurs, ``inicheck`` will not let the simulation continue. +Look at the error message and consult the documentation for either the :doc:`config/smrf_core_config` or +:doc:`config/awsm_core_config` configuration. + +In the example below, the topo file name does not exist and the simulation will exit. Fix by +providing the correct file path. + +.. code:: bash + + Configuration File Status Report: + ========================================================================================== + ERRORS: + + Section Item Message + ------------------------------------------------------------------------------------------ + topo filename File does not exist. + + Logfile verbosity ----------------- -Logfile \ No newline at end of file +AWSM and SMRF provide a log of information about what each are doing. The logging capabilties +allow for AWSM to output various levels of information based on Python's ``logging`` module +(`list of levels`_). If issues occur with AWSM, a good first step is to turn up the logging. +This can be accomplished in the configuration file by setting ``log_level`` to ``DEBUG``. + +.. _`list of levels`: https://docs.python.org/3/library/logging.html#levels + +.. code:: bash + + [awsm system] + log_level: debug \ No newline at end of file From 964d201adc32b2e53e8f22cc2bcc9991610e84c8 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Thu, 10 Jun 2021 10:27:00 -0600 Subject: [PATCH 17/25] input data user guide --- docs/user_guide/index.rst | 5 +--- docs/user_guide/input_data.rst | 54 ++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 4 deletions(-) create mode 100644 docs/user_guide/input_data.rst diff --git a/docs/user_guide/index.rst b/docs/user_guide/index.rst index 58f8e6d..c9099cf 100644 --- a/docs/user_guide/index.rst +++ b/docs/user_guide/index.rst @@ -10,12 +10,8 @@ and/or models. - permission problems - Using WindNinja and Katana - making sure the data is in the right directory -- Input data and removing config sections - - What does ``time_step`` mean - - checking for grid cells in the model domain - Custom vegetation in the ``topo.nc`` - References -- Debugging errors with ``debug`` log level - Lidar assimilation - restarting @@ -25,4 +21,5 @@ and/or models. configuration config/index debugging + input_data tk_lt_zero \ No newline at end of file diff --git a/docs/user_guide/input_data.rst b/docs/user_guide/input_data.rst new file mode 100644 index 0000000..4076841 --- /dev/null +++ b/docs/user_guide/input_data.rst @@ -0,0 +1,54 @@ +Choosing Input Data +=================== + +To generate all the input forcing data required to run iSnobal, the following +measured or derived variables are needed with a full description of the input +data found in the `SMRF documentation`_. + +.. _`SMRF documentation`: https://smrf.readthedocs.io/en/latest/user_guide/input_data.html + +Variable Descriptions +--------------------- + +Air temperature [Celcius] + Measured or modeled air temperature at the surface + +Vapor pressure [Pascals] + Derived from the air temperature and measured relative humidity. Can be calculated + using the IPW utility ``sat2vp`` or the SMRF function ``rh2vp``. + +Precipitation [mm] + Instantaneous precipitation with no negative values. If using a weighing precipitation + gauge that outputs accumulated precipitation, the value must be converted. + +Wind speed [meters per second] + The measured wind speed at the surface. Typically an average value over the measurement + interval. + +Wind direction [degrees] + The measured wind direction at the surface. Typically an average value over the measurement + interval. + +Cloud factor [None] + The percentage between 0 and 1 of the incoming solar radiation that is obstructed by clouds. + 0 equates to no light and 1 equates to no clouds. The cloud factor is derived from the + measured solar radiation and the modeled clear sky solar radiation. The modeled clear sky + solar radiation can be calculated using the IPW utility ``twostream`` or the SMRF + function ``model_solar``. + + +Input data in the configuration file +------------------------------------ + +There are two main sections in the configuration file that dictate which dataset is loaded. The +``[csv]`` section contains file paths to csv data and the ``[gridded]`` section contains +the information for gridded datasets like HRRR and WRF. + +.. note:: + + Only one section can be used at a time. If the simulation is using HRRR, then add the ``[gridded]`` + section and remove the ``[csv]`` section. + +The data sections are closely tied to the ``[time]`` section as the ``start_date`` amd ``end_date`` must be +within the files provided. The ``time_step`` represents the time step to distribute the data (typically every hour) +and should match the input data time step. From 78aecc504f162f55bbaf82ff854428b5c5828b93 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Thu, 10 Jun 2021 15:14:45 -0600 Subject: [PATCH 18/25] laying out the advanced topo guide --- docs/getting_started/basin_setup.rst | 3 ++- docs/user_guide/index.rst | 1 + docs/user_guide/topo_veg.rst | 37 ++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 docs/user_guide/topo_veg.rst diff --git a/docs/getting_started/basin_setup.rst b/docs/getting_started/basin_setup.rst index 21834c7..3a868e1 100644 --- a/docs/getting_started/basin_setup.rst +++ b/docs/getting_started/basin_setup.rst @@ -25,7 +25,8 @@ The basin setup procdure creates a `topo.nc` that contains the following static 6. Basin mask (optional) All these layers are stored in a netCDF file, typically referred to the ``topo.nc`` file. More -information about the ``topo.nc`` file are in the SMRF documentation_. +information about the ``topo.nc`` file are in the SMRF documentation_ with advanced setup in +:doc:`../user_guide/topo_veg`. .. note:: diff --git a/docs/user_guide/index.rst b/docs/user_guide/index.rst index c9099cf..f3c709b 100644 --- a/docs/user_guide/index.rst +++ b/docs/user_guide/index.rst @@ -20,6 +20,7 @@ and/or models. configuration config/index + topo_veg debugging input_data tk_lt_zero \ No newline at end of file diff --git a/docs/user_guide/topo_veg.rst b/docs/user_guide/topo_veg.rst new file mode 100644 index 0000000..1a9c9bc --- /dev/null +++ b/docs/user_guide/topo_veg.rst @@ -0,0 +1,37 @@ +Advanced ``topo.nc`` Setup +========================== + +Creating the ``topo.nc`` following the :doc:`../getting_started/basin_setup` will +result in a useable topo file that can be used to run AWSM. However, sometimes more +customization is needed to fit a particular study or watershed. This guide will go +through the layers within the ``topo.nc`` and how add/change these fields. + +The following layers are required in the ``topo.nc``: + +1. Digital elevation model +2. Vegetation type +3. Vegetation height +4. Vegetation extinction coefficient +5. Vegetation optical transmissivity +6. Basin mask (optional) + + +Digitial Elevation Model +------------------------ + +DEM + +Vegetation Type +--------------- + +Vegetation Height +----------------- + +Vegetation Extinction Coefficient +--------------------------------- + +Vegetation Optical Transmissivity +--------------------------------- + +Basin Mask (optional) +--------------------- \ No newline at end of file From 9389862e838835acc6f242dbbe01eb92f6ac9808 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Fri, 11 Jun 2021 12:24:23 -0600 Subject: [PATCH 19/25] docs requirements --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index da3886c..262fe37 100755 --- a/setup.py +++ b/setup.py @@ -59,8 +59,8 @@ extras_require={ 'docs': [ 'Cython', - 'numpy<1.19', - 'Sphinx>=3.0,<=4.0', + 'numpy', + 'Sphinx>=4.0,<5.0', 'pydata-sphinx-theme', 'sphinxcontrib-bibtex>=1.0', 'sphinxcontrib-websupport>=1.0.1', From 7d29eab19d4b84236b5b3647561bf0af33ba6e7f Mon Sep 17 00:00:00 2001 From: Andrew Hedrick Date: Wed, 16 Jun 2021 16:17:21 -0600 Subject: [PATCH 20/25] Added info to basin_setup description --- docs/getting_started/basin_setup.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/getting_started/basin_setup.rst b/docs/getting_started/basin_setup.rst index 3a868e1..c6ddcab 100644 --- a/docs/getting_started/basin_setup.rst +++ b/docs/getting_started/basin_setup.rst @@ -152,13 +152,16 @@ The ``delineate`` command will create a file in ``./topo/delineation/basin_outli will contain the delineated basin. Open the shape file and ensure that the basin delineation performed as expected. Next, create the topo with ``basin_setup``. The LANDFIRE dataset is quite large (~1.5GB) and can be downloaded prior and/or reused. Ensure that the -LANDFIRE dataset is in the ``./veg_data`` folder and the download will be skipped. +LANDFIRE dataset is in the ``./veg_data`` folder and the download will be skipped. The domain +default grid cell size is 50 m, but if another size is desired it can be set with the -c option. +Finally, the extents of the domain can be defined using the -ex option. .. code:: bash docker-compose run basin_setup \ -f delineation/basin_outline.shp \ -bn BasinName \ + -c 50 \ -dm out_dem_50_meters.tif \ -d /Downloads # will download LANDFIRE here if not present @@ -237,4 +240,4 @@ have been generated. The example below shows the Lakes basin in the AWSM tests. string :history = "[2019-12-31 21:33:38] Create netCDF4 file using Basin Setup v0.13.0" ; string :institution = "USDA Agricultural Research Service, Northwest Watershed Research Center" ; string :generation_command = "/usr/local/bin/basin_setup -f delineation/basin_outline.shp -bn Lakes Basin -dm lakes_dem_UTM11_WGS84.tif -d /Downloads -ex 319975 4158253 327755 4166675" ; - } \ No newline at end of file + } From 324058a30baf7e69c49bb91de900c496a02e2043 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Thu, 22 Jul 2021 09:31:07 -0600 Subject: [PATCH 21/25] refs --- docs/getting_started/basin_setup.rst | 4 ++-- docs/user_guide/index.rst | 16 +++------------- docs/user_guide/references.rst | 13 +++++++++++++ docs/user_guide/tk_lt_zero.rst | 4 ++-- 4 files changed, 20 insertions(+), 17 deletions(-) create mode 100644 docs/user_guide/references.rst diff --git a/docs/getting_started/basin_setup.rst b/docs/getting_started/basin_setup.rst index 3a868e1..f93f27f 100644 --- a/docs/getting_started/basin_setup.rst +++ b/docs/getting_started/basin_setup.rst @@ -5,8 +5,8 @@ All components will be described using docker containers such that the generatio of the necessary files is reproducible. Note that exact docker versions will be specified throughout the document but can be updated to different versions. -The initial set up is performed using the package `basin_setup`_. `basin_setup`_ requires a DEM and it -the `LANDFIRE`_ dataset to fill in the vegetation layers and parameters required to run iSnobal. +The initial set up is performed using the package `basin_setup`_. `generate_topo`_ requires a DEM and it +the `LANDFIRE`_ dataset (version 1.4.0) to fill in the vegetation layers and parameters required to run iSnobal. .. _basin_setup: https://github.com/USDA-ARS-NWRC/basin_setup .. _LANDFIRE: https://www.landfire.gov diff --git a/docs/user_guide/index.rst b/docs/user_guide/index.rst index f3c709b..0b263b4 100644 --- a/docs/user_guide/index.rst +++ b/docs/user_guide/index.rst @@ -2,18 +2,7 @@ User guide ========== -The User Guide covers different topic areas in AWSM and will show -how to run specific types of simulations using different inputs, outputs -and/or models. - -- Docker volume mounting and what needs to be set - - permission problems -- Using WindNinja and Katana - - making sure the data is in the right directory -- Custom vegetation in the ``topo.nc`` -- References -- Lidar assimilation -- restarting +The User Guide covers different topic areas in AWSM. .. toctree:: :maxdepth: 2 @@ -23,4 +12,5 @@ and/or models. topo_veg debugging input_data - tk_lt_zero \ No newline at end of file + tk_lt_zero + references \ No newline at end of file diff --git a/docs/user_guide/references.rst b/docs/user_guide/references.rst new file mode 100644 index 0000000..bb00b76 --- /dev/null +++ b/docs/user_guide/references.rst @@ -0,0 +1,13 @@ +References +---------- + +iSnobal +======= + + +AWSM +==== + + +SMRF +==== \ No newline at end of file diff --git a/docs/user_guide/tk_lt_zero.rst b/docs/user_guide/tk_lt_zero.rst index 93a758c..a04f579 100644 --- a/docs/user_guide/tk_lt_zero.rst +++ b/docs/user_guide/tk_lt_zero.rst @@ -39,7 +39,7 @@ and stabilize the turbulent transfer calculations. This can be achieved by using [ipysnobal] thresh_normal: 60 - thresh_medium: 20 + thresh_medium: 20 <--- thresh_small: 1 Step 2 @@ -62,5 +62,5 @@ After the model is through the period, change the thresholds back and continue r [ipysnobal] thresh_normal: 60 - thresh_medium: 10 + thresh_medium: 10 <--- thresh_small: 1 \ No newline at end of file From 1a19e4b2d2b9ff768346a6ccd2179a7dfced8731 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Thu, 22 Jul 2021 09:46:08 -0600 Subject: [PATCH 22/25] updated basin setup docs --- docs/getting_started/basin_setup.rst | 50 ++++++++++++++-------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/docs/getting_started/basin_setup.rst b/docs/getting_started/basin_setup.rst index 251b22a..cfec6a5 100644 --- a/docs/getting_started/basin_setup.rst +++ b/docs/getting_started/basin_setup.rst @@ -5,7 +5,7 @@ All components will be described using docker containers such that the generatio of the necessary files is reproducible. Note that exact docker versions will be specified throughout the document but can be updated to different versions. -The initial set up is performed using the package `basin_setup`_. `generate_topo`_ requires a DEM and it +The initial set up is performed using the package `basin_setup`_. `basin_setup`_ requires a DEM and it the `LANDFIRE`_ dataset (version 1.4.0) to fill in the vegetation layers and parameters required to run iSnobal. .. _basin_setup: https://github.com/USDA-ARS-NWRC/basin_setup @@ -31,7 +31,8 @@ information about the ``topo.nc`` file are in the SMRF documentation_ with advan .. note:: The ``topo.nc`` **must** have projection information. It's just good practice. Using - ``basin_setup`` will ensure the projection is in the ``topo.nc``. + ``basin_setup`` will ensure the projection is in the ``topo.nc`` which will come from + the basin outline shapefile. .. _documentation: https://smrf.readthedocs.io/en/latest/getting_started/create_topo.html @@ -42,17 +43,17 @@ Creating the DEM The DEM was obtained from the USGS National Elevation `dataset`_ at ~10-m (9.26-m) resolution. The original tiles are provided in EPSG:4269 - NAD83 – Geographic projection. Download the necessary -tiles to cover the modeling domain. +tiles to cover the modeling domain. Join the tiles were in the native projection and resolution (EPSG 4269). .. _dataset: https://catalog.data.gov/dataset/usgs-national-elevation-dataset-ned -Join the tiles were in the native projection and resolution (EPSG 4269). Subsequently, crop the joined DEM +Optionally, crop the joined DEM and resample to 50-m resolution and reproject to UTM coordinates (for example EPSG 32613, UTM 13N - WGS 84), all of which is accomplished using a single ``gdalwarp`` instruction (also available in QGIS or similar ArcGIS -command). +command). This will also be performed within the ``generate_topo`` command. .. note:: - Beware of using Nearest Neightbour interpolation as it can create artifacts in the resulting DEM. Use average, + Beware of using Nearest Neighbor interpolation as it can create artifacts in the resulting DEM. Use average, bilinear or cubic interpolation. A sample ``gdalwarp`` command for the Gunnison in Colorado. This will resample to 50-m grid resolution and @@ -109,7 +110,7 @@ Delineate Basin, Generate Topo The basin delineation and creation of the ``topo.nc`` file is done with the docker version of ``basin_setup``. A ``docker-compose.yml`` file aids the composition of the docker commands and simplifies mounting data volumes to the docker image. The following ``docker-compose.yml`` -file contains two services, `delineate` runs the delineation routine and ``basin_setup`` creates +file contains two services, ``delineate`` runs the delineation routine and ``generate_topo`` creates the ``topo.nc``. .. code:: yml @@ -118,18 +119,17 @@ the ``topo.nc``. services: delineate: - image: usdaarsnwrc/basin_setup:0.14 + image: usdaarsnwrc/basin_setup:0.15 volumes: - ./topo:/data - - ./veg_data:/Downloads entrypoint: delineate - basin_setup: - image: usdaarsnwrc/basin_setup:0.14 + generate_topo: + image: usdaarsnwrc/basin_setup:0.15 volumes: - - ./topo:/data - - ./veg_data:/Downloads - entrypoint: basin_setup + - ./topo:/data/topo + - ./veg_data:/data/veg_data + entrypoint: generate_topo With the DEM tiff file ``out_dem_50_meters.tif`` and the ``.bna`` file, run ``delineate`` @@ -150,20 +150,20 @@ create sub-basins for each pour point. The ``delineate`` command will create a file in ``./topo/delineation/basin_outline.shp`` which will contain the delineated basin. Open the shape file and ensure that the basin -delineation performed as expected. Next, create the topo with ``basin_setup``. The LANDFIRE -dataset is quite large (~1.5GB) and can be downloaded prior and/or reused. Ensure that the -LANDFIRE dataset is in the ``./veg_data`` folder and the download will be skipped. The domain -default grid cell size is 50 m, but if another size is desired it can be set with the -c option. -Finally, the extents of the domain can be defined using the -ex option. +delineation performed as expected. + +Next, create the topo with ``generate_topo``. The LANDFIRE version 1.4.0 +dataset is quite large (~3GB) and must be downloaded prior. Ensure that the +LANDFIRE dataset is in the ``./veg_data`` folder and unzipped. ``generate_topo`` uses a +configuration file to specify all the required parameters to run. See the +`CoreConfig`_ for options and the `sample configuration files`_. + +.. _CoreConfig: https://github.com/USDA-ARS-NWRC/basin_setup/blob/main/basin_setup/CoreConfig.ini +.. _sample configuration files: https://github.com/USDA-ARS-NWRC/basin_setup/blob/main/tests/Lakes/config.ini .. code:: bash - docker-compose run basin_setup \ - -f delineation/basin_outline.shp \ - -bn BasinName \ - -c 50 \ - -dm out_dem_50_meters.tif \ - -d /Downloads # will download LANDFIRE here if not present + docker-compose run generate_topo /data/topo/config.ini View ``topo.nc`` From 589d2605857547b3e1f2a8bbb417da01bcd1a831 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Thu, 22 Jul 2021 10:01:17 -0600 Subject: [PATCH 23/25] awsm api docs --- Makefile | 2 +- docs/api/awsm.framework.rst | 29 +++++++++++ docs/api/awsm.interface.rst | 21 ++++++++ docs/api/awsm.models.pysnobal.rst | 37 +++++++++++++ docs/api/awsm.models.rst | 29 +++++++++++ docs/api/awsm.rst | 21 ++++++++ docs/api/awsm.tests.output.rst | 21 ++++++++ docs/api/awsm.tests.restart.rst | 29 +++++++++++ docs/api/awsm.tests.rst | 86 +++++++++++++++++++++++++++++++ docs/api/index.rst | 6 ++- docs/api/modules.rst | 7 +++ docs/authors.rst | 1 - docs/getting_started/index.rst | 2 +- docs/user_guide/index.rst | 3 +- 14 files changed, 288 insertions(+), 6 deletions(-) create mode 100644 docs/api/awsm.framework.rst create mode 100644 docs/api/awsm.interface.rst create mode 100644 docs/api/awsm.models.pysnobal.rst create mode 100644 docs/api/awsm.models.rst create mode 100644 docs/api/awsm.rst create mode 100644 docs/api/awsm.tests.output.rst create mode 100644 docs/api/awsm.tests.restart.rst create mode 100644 docs/api/awsm.tests.rst create mode 100644 docs/api/modules.rst delete mode 100644 docs/authors.rst diff --git a/Makefile b/Makefile index 49ae3cd..d85fb6c 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,7 @@ coverage-html: coverage ## check code coverage quickly with the default Python docs: ## generate Sphinx HTML documentation, including API docs rm -f docs/awsm.rst rm -f docs/modules.rst - sphinx-apidoc -o docs/ awsm + sphinx-apidoc -o docs/api awsm $(MAKE) -C docs clean $(MAKE) -C docs html $(BROWSER) docs/_build/html/index.html diff --git a/docs/api/awsm.framework.rst b/docs/api/awsm.framework.rst new file mode 100644 index 0000000..a93e1b6 --- /dev/null +++ b/docs/api/awsm.framework.rst @@ -0,0 +1,29 @@ +awsm.framework package +====================== + +Submodules +---------- + +awsm.framework.ascii\_art module +-------------------------------- + +.. automodule:: awsm.framework.ascii_art + :members: + :undoc-members: + :show-inheritance: + +awsm.framework.framework module +------------------------------- + +.. automodule:: awsm.framework.framework + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: awsm.framework + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/awsm.interface.rst b/docs/api/awsm.interface.rst new file mode 100644 index 0000000..f7354e4 --- /dev/null +++ b/docs/api/awsm.interface.rst @@ -0,0 +1,21 @@ +awsm.interface package +====================== + +Submodules +---------- + +awsm.interface.ingest\_data module +---------------------------------- + +.. automodule:: awsm.interface.ingest_data + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: awsm.interface + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/awsm.models.pysnobal.rst b/docs/api/awsm.models.pysnobal.rst new file mode 100644 index 0000000..60672d6 --- /dev/null +++ b/docs/api/awsm.models.pysnobal.rst @@ -0,0 +1,37 @@ +awsm.models.pysnobal package +============================ + +Submodules +---------- + +awsm.models.pysnobal.init\_model module +--------------------------------------- + +.. automodule:: awsm.models.pysnobal.init_model + :members: + :undoc-members: + :show-inheritance: + +awsm.models.pysnobal.ipysnobal module +------------------------------------- + +.. automodule:: awsm.models.pysnobal.ipysnobal + :members: + :undoc-members: + :show-inheritance: + +awsm.models.pysnobal.pysnobal\_io module +---------------------------------------- + +.. automodule:: awsm.models.pysnobal.pysnobal_io + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: awsm.models.pysnobal + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/awsm.models.rst b/docs/api/awsm.models.rst new file mode 100644 index 0000000..da3bf1c --- /dev/null +++ b/docs/api/awsm.models.rst @@ -0,0 +1,29 @@ +awsm.models package +=================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + awsm.models.pysnobal + +Submodules +---------- + +awsm.models.smrf\_connector module +---------------------------------- + +.. automodule:: awsm.models.smrf_connector + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: awsm.models + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/awsm.rst b/docs/api/awsm.rst new file mode 100644 index 0000000..696cc08 --- /dev/null +++ b/docs/api/awsm.rst @@ -0,0 +1,21 @@ +awsm package +============ + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + awsm.framework + awsm.interface + awsm.models + awsm.tests + +Module contents +--------------- + +.. automodule:: awsm + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/awsm.tests.output.rst b/docs/api/awsm.tests.output.rst new file mode 100644 index 0000000..d7e0761 --- /dev/null +++ b/docs/api/awsm.tests.output.rst @@ -0,0 +1,21 @@ +awsm.tests.output package +========================= + +Submodules +---------- + +awsm.tests.output.test\_file\_output module +------------------------------------------- + +.. automodule:: awsm.tests.output.test_file_output + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: awsm.tests.output + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/awsm.tests.restart.rst b/docs/api/awsm.tests.restart.rst new file mode 100644 index 0000000..544a118 --- /dev/null +++ b/docs/api/awsm.tests.restart.rst @@ -0,0 +1,29 @@ +awsm.tests.restart package +========================== + +Submodules +---------- + +awsm.tests.restart.test\_lakes module +------------------------------------- + +.. automodule:: awsm.tests.restart.test_lakes + :members: + :undoc-members: + :show-inheritance: + +awsm.tests.restart.test\_rme module +----------------------------------- + +.. automodule:: awsm.tests.restart.test_rme + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: awsm.tests.restart + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/awsm.tests.rst b/docs/api/awsm.tests.rst new file mode 100644 index 0000000..5604173 --- /dev/null +++ b/docs/api/awsm.tests.rst @@ -0,0 +1,86 @@ +awsm.tests package +================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + awsm.tests.output + awsm.tests.restart + +Submodules +---------- + +awsm.tests.awsm\_test\_case module +---------------------------------- + +.. automodule:: awsm.tests.awsm_test_case + :members: + :undoc-members: + :show-inheritance: + +awsm.tests.awsm\_test\_case\_lakes module +----------------------------------------- + +.. automodule:: awsm.tests.awsm_test_case_lakes + :members: + :undoc-members: + :show-inheritance: + +awsm.tests.check\_mixin module +------------------------------ + +.. automodule:: awsm.tests.check_mixin + :members: + :undoc-members: + :show-inheritance: + +awsm.tests.test\_awsm\_directory module +--------------------------------------- + +.. automodule:: awsm.tests.test_awsm_directory + :members: + :undoc-members: + :show-inheritance: + +awsm.tests.test\_awsm\_lakes module +----------------------------------- + +.. automodule:: awsm.tests.test_awsm_lakes + :members: + :undoc-members: + :show-inheritance: + +awsm.tests.test\_awsm\_lakes\_init module +----------------------------------------- + +.. automodule:: awsm.tests.test_awsm_lakes_init + :members: + :undoc-members: + :show-inheritance: + +awsm.tests.test\_awsm\_lakes\_lidar\_update module +-------------------------------------------------- + +.. automodule:: awsm.tests.test_awsm_lakes_lidar_update + :members: + :undoc-members: + :show-inheritance: + +awsm.tests.test\_awsm\_rme module +--------------------------------- + +.. automodule:: awsm.tests.test_awsm_rme + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: awsm.tests + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/index.rst b/docs/api/index.rst index 76be2dc..4013ab6 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -8,4 +8,8 @@ The API here describes all the classes and functions used in AWSM. -.. include:: modules.rst +.. toctree:: + + awsm.framework + awsm.interface + awsm.models diff --git a/docs/api/modules.rst b/docs/api/modules.rst new file mode 100644 index 0000000..9de51af --- /dev/null +++ b/docs/api/modules.rst @@ -0,0 +1,7 @@ +awsm +==== + +.. toctree:: + :maxdepth: 4 + + awsm diff --git a/docs/authors.rst b/docs/authors.rst deleted file mode 100644 index e122f91..0000000 --- a/docs/authors.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../AUTHORS.rst diff --git a/docs/getting_started/index.rst b/docs/getting_started/index.rst index 5ef1b30..8401891 100644 --- a/docs/getting_started/index.rst +++ b/docs/getting_started/index.rst @@ -9,7 +9,7 @@ Stable ~~~~~~ The stable version of AWSM is currently :code:`v0.10`. The code can be downloaded from the `releases`_, -can be found on the :code:`release-0.10` `branch`_ or the stable :doc:`docker image <./stable_install>`. +can be found on the :code:`release-0.10` `branch`_ or the stable :doc:`docker image <./install/stable_install>`. .. _releases: https://github.com/USDA-ARS-NWRC/awsm/releases .. _branch: https://github.com/USDA-ARS-NWRC/awsm/tree/release-0.10 diff --git a/docs/user_guide/index.rst b/docs/user_guide/index.rst index 0b263b4..c5e6b45 100644 --- a/docs/user_guide/index.rst +++ b/docs/user_guide/index.rst @@ -12,5 +12,4 @@ The User Guide covers different topic areas in AWSM. topo_veg debugging input_data - tk_lt_zero - references \ No newline at end of file + tk_lt_zero \ No newline at end of file From 8aeab60270bdad7bddfac4573a88f0d2e66dc16f Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Thu, 22 Jul 2021 10:04:14 -0600 Subject: [PATCH 24/25] cleaning up unused doc files --- docs/config_ref.rst | 30 ------- docs/contributing.rst | 1 - docs/getting_started/installation.rst | 92 ------------------- docs/history.rst | 1 - docs/readme.rst | 125 -------------------------- docs/usage.rst | 24 ----- 6 files changed, 273 deletions(-) delete mode 100644 docs/config_ref.rst delete mode 100644 docs/contributing.rst delete mode 100644 docs/getting_started/installation.rst delete mode 100644 docs/history.rst delete mode 100644 docs/readme.rst delete mode 100644 docs/usage.rst diff --git a/docs/config_ref.rst b/docs/config_ref.rst deleted file mode 100644 index ce5e9ab..0000000 --- a/docs/config_ref.rst +++ /dev/null @@ -1,30 +0,0 @@ - -AWSM Configuration File Reference -================================= -The AWSM config file is a combination of SMRF and AWSM's own configuration -files. Pleas refer to the `Reference for SMRF Configuration Files`_ for any -questions regarding the configuration file sections from SMRF which are as -follows: - -* topo -* time -* stations -* csv -* gridded -* air_temperature -* vapor pressure -* wind -* precip -* albedo -* thermal -* soil_temp -* output -* logging -* system - -.. _Reference for SMRF Configuration Files: http://smrf.readthedocs.io/en/latest/auto_config.html - -All other sections and details for AWSM configuration files can be seen in the -below. - -.. include:: ./auto_config.rst diff --git a/docs/contributing.rst b/docs/contributing.rst deleted file mode 100644 index e582053..0000000 --- a/docs/contributing.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../CONTRIBUTING.rst diff --git a/docs/getting_started/installation.rst b/docs/getting_started/installation.rst deleted file mode 100644 index 17af66a..0000000 --- a/docs/getting_started/installation.rst +++ /dev/null @@ -1,92 +0,0 @@ -.. highlight:: shell - -============ -Installation -============ - -Installing Dependencies ------------------------ - -AWSM was designed to run simulations with SMRF_ and PySnobal_. These are -the two main dependencies for AWSM. To install the dependencies: - -.. code:: bash - - python3 -m pip install -r requirements.txt - -.. _SMRF: https://github.com/USDA-ARS-NWRC/smrf -.. _PySnobal: https://github.com/USDA-ARS-NWRC/pysnobal - -Installing AWSM ---------------- - -Once the dependencies have been installed for your respective system, the -following will install AWSM. It is preferable to use a Python -`virtual environment`_ to reduce the possibility of a dependency issue. You should -use the same virtual environment in which you installed SMRF. You can just -source your smrfenv instead of step number 1. - -.. _virtual environment: https://virtualenv.pypa.io - -1. Create a virtualenv and activate it. - - .. code:: bash - - virtualenv awsmenv - source awsmenv/bin/activate - -2. Clone AWSM source code from the ARS-NWRC github. - - .. code:: console - - git clone https://github.com/USDA-ARS-NWRC/AWSM.git - -3. Change directories into the AWSM directory. Install the python requirements. - After the requirements are done, install AWSM. - - .. code:: console - - cd awsm - pip install -r requirements.txt - python setup.py install - -4. (Optional) Generate a local copy of the documentation. - - .. code:: console - - cd docs - make html - - To view the documentation use the preferred browser to open up the files. - This can be done from the browser by opening the index.rst file directly or - by the commandline like the following: - - .. code:: console - - google-chrome _build/html/index.html - -Testing AWSM ---------------- - -Once everything is installed, you can run a quick test case over a small -catchment in Idaho called Reynolds Mountain East (RME). - -1. Move to config file and run case. Start in your AWSM directory - - .. code:: console - - cd test_data/RME_run/ - awsm config.ini - -2. Wait for the test run to finish and then view the results. - - .. code:: console - - cd output/rme/devel/wy1998/rme_test/ - -The iSnobal model outputs will be in the "runs" folder and the distributed -SMRF data will be in the "data" folder. Navigate around and see what the -outputs look like. You can visualize the .nc (netCDF) files with -the `ncview`_ utility. - -.. _ncview: http://meteora.ucsd.edu/~pierce/ncview_home_page.html diff --git a/docs/history.rst b/docs/history.rst deleted file mode 100644 index 2506499..0000000 --- a/docs/history.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../HISTORY.rst diff --git a/docs/readme.rst b/docs/readme.rst deleted file mode 100644 index b3331e2..0000000 --- a/docs/readme.rst +++ /dev/null @@ -1,125 +0,0 @@ -Automated Water Supply Model -============================ - -|GitHub version| |docs| |docker build| |docker automated| - -Automated Water Supply Model (AWSM) was developed at the USDA -Agricultural Research Service (ARS) in Boise, ID. AWSM was designed to -streamline the work flow used by the ARS to forecast the water supply of -multiple water basins. AWSM standardizes the steps needed to distribute -weather station data with SMRF, run an energy and mass balance with -iSnobal, and process the results, while maintaining the flexibility of -each program. - -.. |GitHub version| image:: https://badge.fury.io/gh/USDA-ARS-NWRC%2Fawsm.svg - :alt: AWSM Version - :target: https://badge.fury.io/gh/USDA-ARS-NWRC%2Fawsm - -.. |docs| image:: https://readthedocs.org/projects/awsm/badge/ - :alt: Documentation Status - :target: https://awsm.readthedocs.io - -.. |docker build| image:: https://img.shields.io/docker/build/usdaarsnwrc/awsm.svg - :alt: Docker Build Status - :target: https://hub.docker.com/r/usdaarsnwrc/awsm/ - -.. |docker automated| image:: https://img.shields.io/docker/automated/usdaarsnwrc/awsm.svg - :alt: Automated Docker Build Status - :target: https://hub.docker.com/r/usdaarsnwrc/awsm/ - -.. image::https://raw.githubusercontent.com/USDA-ARS-NWRC/awsm/master/docs/_static/ModelSystemOverview_new.png - -Quick Start ------------ - -The fastest way to get up and running with AWSM is to use the docker -images that are pre-built and can deployed cross platform. - -To build AWSM natively from source checkout the install instructions -`here`_. - -.. _here: https://awsm.readthedocs.io/en/latest/installation.html - -Docker -~~~~~~ - -Docker images are containers that allow us to ship the software to our -users seamlessly and without a headache. It is by far the easiest way to -use AWSM. If you are curious to read more about them, visit `Whats a -container`_ on docker’s website. - -.. _Whats a container: https://www.docker.com/what-container - -Using docker images comes with very minor quirks though, such as -requiring you to mount a volume to access the data when you are done -with your run. To mount a data volume, so that you can share data -between the local file system and the docker, the ``-v`` option must be -used. For a more in depth discussion and tutorial, read about `docker -volumes`_. The container has a shared data volume at ``/data`` where the -container can access the local file system. - -.. _docker volumes: https://docs.docker.com/storage/volumes/ - -**NOTE: On the host paths to the volume to mount, you must use full -absolute paths!** - -Running the Demo -~~~~~~~~~~~~~~~~ - -To simply run the AWSM demo; mount the desired directory as a volume and -run the image, using the following command: - -**For Linux:** - -.. code-block:: console - - docker run -v :/data -it usdaarsnwrc/awsm:develop - -**For MacOSX:** - -.. code-block:: console - - docker run -v /Users/:/data -it usdaarsnwrc/awsm:develop - -**For Windows:** - -.. code-block:: console - - docker run -v /c/Users/:/data -it usdaarsnwrc/awsm:develop - -The output netCDF files will be placed in the location you mounted -(using the -v option). We like to use `ncview`_ to view our netcdf files -quickly. - -.. _ncview: http://meteora.ucsd.edu/~pierce/ncview_home_page.html - -Setting Up Your Run -~~~~~~~~~~~~~~~~~~~ - -To use the AWSM docker image to create your own runs, you need to setup -a project folder containing all the files necessary to run the model. -Then using the same command above, mount your project folder and provide -a path to the configuration file. An example of a project folder might -like: - -.. code-block:: console - - My_Basin - ├── air_temp.csv - ├── cloud_factor.csv - ├── config.ini - ├── maxus.nc - ├── metadata.csv - ├── output - ├── precip.csv - ├── solar.csv - ├── topo.nc - ├── vapor_pressure.csv - ├── wind_direction.csv - └── wind_speed.csv - -Then the command would be: - -.. code-block:: console - - docker run -v /My_Basin:/data -it usdaarsnwrc/awsm:develop /My_Basin/config.ini diff --git a/docs/usage.rst b/docs/usage.rst deleted file mode 100644 index b3a4e2e..0000000 --- a/docs/usage.rst +++ /dev/null @@ -1,24 +0,0 @@ -===== -Usage -===== - -To run AWSM, a configuration is require and it's simply passed as the first -argument to the awsm command. If the configuration file was named config.ini -it could be used like the following. - -.. code-block:: bash - - awsm config.ini - -For configuring AWSM simulations refer to :ref:`using-configs`. If you are -interested in using AWSM in a project, getting started looks like this: - -.. code-block:: python - - import awsm - - with awsm.framework.framework.AWSM(configFile) as a: - # Specify functions to run - -Review the script for running AWSM in "./scripts/awsm" to get a better sense of -the methods used to run AWSM and use the :ref:`api-documentation` From 0530a7c6391fde5c45ac6e8c739abe788c384ced Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Mon, 9 Aug 2021 08:46:05 -0600 Subject: [PATCH 25/25] clarify basin setup --- docs/getting_started/basin_setup.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting_started/basin_setup.rst b/docs/getting_started/basin_setup.rst index cfec6a5..34da5c8 100644 --- a/docs/getting_started/basin_setup.rst +++ b/docs/getting_started/basin_setup.rst @@ -43,7 +43,7 @@ Creating the DEM The DEM was obtained from the USGS National Elevation `dataset`_ at ~10-m (9.26-m) resolution. The original tiles are provided in EPSG:4269 - NAD83 – Geographic projection. Download the necessary -tiles to cover the modeling domain. Join the tiles were in the native projection and resolution (EPSG 4269). +tiles to cover the modeling domain. Merge the tiles in the native projection and resolution (EPSG 4269, 10m). .. _dataset: https://catalog.data.gov/dataset/usgs-national-elevation-dataset-ned