First prototype of a code base

.github/workflows/ci.yaml

@@ -34,29 +34,31 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - name: Set up Python 3.11
+    - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
-        python-version: 3.11
+        python-version: ${{ matrix.python-version }}
     - name: Install hatch
       run: |
         pip install --upgrade pip
         pip install hatch
+        pip install . # to get verdi
     - name: Run test suite
       env:
         PYTEST_ADDOPTS: "--durations=0"
       run: |
+        verdi presto
         hatch test --cover
 
   docs:
     runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Python 3.11
+    - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
-        python-version: "3.11"
+        python-version: ${{ matrix.python-version }}
     - name: Install hatch
       run: |
         pip install --upgrade pip

README.rst

@@ -3,19 +3,49 @@
 Weather and Climate Workflow Tool based on AiiDA
 ================================================
 
-WcFlow a library for creating weather and climate workflows from a yaml format inspired by `cylc <https://cylc.github.io/>`_ using `AiiDA <https://www.aiida.net/>`_ as workflow engine.
+WcFlow is a library for creating weather and climate workflows from a yaml format inspired by `cylc <https://cylc.github.io/>`_ using `AiiDA <https://www.aiida.net/>`_ as workflow engine.
+
+.. marker-installation-aiida
+
+Installing AiiDA
+----------------
+
+To install AiiDA, it is first recommended to create a virtual environment, e.g. via ``venv`` and activate it:
+
+.. code-block:: bash
+
+    python -m venv ~/.python_venvs/wcflow
+    source ~/.python_venvs/wcflow/bin/activate
+
+The ``aiida-core`` package can then be installed with pip:
+
+.. code-block:: bash
+
+    pip install aiida-core
+
+Finally, ``verdi presto`` can be used to quickly set up an AiiDA profile using SQLite via:
+
+.. code-block:: bash
+
+    verdi presto --profile-name wcflow
+
+Further information on the installation can be found in the 
+`AiiDA documentation <https://aiida.readthedocs.io/projects/aiida-core/en/latest/installation/index.html>`_,
+which covers (among other topics) how to set up RabbitMQ, necessary to run processes in the background in a non-blocking manner, as well as how use a more performant PostgreSQL database.
 
 .. marker-installation
 
-Install
--------
+Installing the package for development
+--------------------------------------
 
-To install it please use
+To install the package please use
 
 .. code-block:: bash
 
     pip install -e .
 
+This will automatically install the additional, required packages, ``aiida-shell`` and ``aiida-workgraph`` at compatible versions.
+
 .. marker-developer-tools
 
 Developer tools

docs/src/api.rst

@@ -0,0 +1,20 @@
+API
+===
+
+wcflow.wc
+---------
+.. automodule:: wcflow.wc
+   :members:
+   :undoc-members:
+
+wcflow.core
+-----------
+.. automodule:: wcflow.core
+   :members: 
+   :undoc-members:
+
+wcflow.workgraph
+----------------
+.. automodule:: wcflow.workgraph
+   :members:
+   :undoc-members:

docs/src/conf.py

@@ -29,6 +29,7 @@
     'sphinx.ext.mathjax',
     'sphinx.ext.intersphinx',
     'sphinx.ext.viewcode',
+    'sphinx.ext.autodoc',
 ]
 
 # Add any paths that contain templates here, relative to this directory.

docs/src/getting_started.rst

@@ -4,3 +4,9 @@ Getting started
 .. include:: ../../README.rst
    :start-after: marker-installation
    :end-before: marker-developer-tools
+
+Example
+-------
+
+.. include:: ../../examples/README.rst
+   :start-after: marker-example-text

docs/src/index.rst

@@ -5,3 +5,4 @@
 .. toctree::
 
    getting_started.rst
+   api.rst

examples/README.rst

@@ -0,0 +1,12 @@
+Running examples
+================
+
+.. marker-example-text
+
+To run the example you can use the command
+
+.. code-block:: bash
+
+    wcflow files/configs/example_config.yaml
+
+It is important to execute it from the examples folder as the relative paths are hardcoded.

examples/files/configs/example_config.yaml

@@ -0,0 +1,181 @@
+start_date: '2026-01-01T00:00'
+end_date: '2027-01-01T00:00'
+cycles:
+  init:
+    tasks:
+      extpar:
+        inputs:
+          - obs_data:
+              arg_option: "--input"
+        outputs:
+          - extpar_file
+  icon_bimonthly:
+    period: 'P2M'
+    tasks:
+      preproc:
+        inputs:
+          - grid_file:
+              arg_option: "-g"
+          - extpar_file:
+              date: '2026-01-01T00:00'
+              arg_option: "-p"
+          - ERA5:
+              arg_option: "-e"
+        outputs:
+          - icon_input
+        depends:
+          - icon:
+              lag: '-P4M'
+      icon:
+        inputs:
+          - grid_file:
+              arg_option: "-g"
+          - icon_input:
+              arg_option: "--input"
+          #- icon_restart:
+          #    lag: '-P2M'
+        outputs:
+          - stream_1
+          - stream_2
+          #- icon_restart
+      postproc_1:
+        inputs:
+          - stream_1:
+              arg_option: "--input"
+        outputs:
+          - postout_1
+      store_and_clean_1:
+        inputs:
+          - postout_1:
+              arg_option: "--input"
+          - stream_1:
+              arg_option: "--stream"
+          - icon_input:
+              arg_option: "--icon_input"
+        outputs:
+          - stored_data_1
+  yearly:
+    period: 'P1Y'
+    tasks:
+      postproc_2:
+        inputs:
+          - stream_2:
+             lag: 
+               - 'P0M'
+             arg_option: "--input"
+        outputs:
+          - postout_2
+      store_and_clean_2:
+        inputs:
+          - postout_2:
+             arg_option: "--input"
+          - stream_2:
+             lag: 
+               - 'P0M'
+        outputs:
+          - stored_data_2
+# Each task and piece of data (input and output of tasks) used to
+# define the graph is described in that section
+tasks:
+  root:
+    # All tasks inherit the root task properties
+    host: santis
+    account: g110
+  extpar:
+    plugin: extpar
+    command: $PWD/files/scripts/extpar
+    command_option: "--verbose" # todo implement support
+    config: path/to/namelists/dir
+    uenv:
+      squashfs: path/to/squashfs
+      mount_point: runtime/mount/point
+    nodes: 1
+    walltime: '00:02:00'
+  preproc:
+    plugin: AiiDA Shell
+    command: $PWD/files/scripts/cleanup.sh
+    nodes: 4
+    walltime: '00:02:00'
+    config: path/to/config/dir
+    uenv:
+      squashfs: path/to/squashfs
+      mount_point: runtime/mount/point
+  icon:
+    plugin: icon
+    command: $PWD/files/scripts/icon
+    nodes: 40
+    walltime: '23:59:59'
+    config: path/to/namelists/dir
+    uenv:
+      squashfs: path/to/squashfs
+      mount_point: runtime/mount/point
+  postproc_1:
+    plugin: AiiDA Shell
+    command: $PWD/files/scripts/main_script_ocn.sh
+    nodes: 2
+    walltime: '00:05:00'
+    conda_env: path/to/yaml/env/file
+    uenv:
+      squashfs: path/to/squashfs
+      mount_point: runtime/mount/point
+  postproc_2:
+    plugin: AiiDA Shell
+    command: $PWD/files/scripts/main_script_atm.sh
+    nodes: 2
+    walltime: '00:05:00'
+    src: path/to/src/dir
+    conda_env: path/to/yaml/env/file
+    uenv:
+      squashfs: path/to/squashfs
+      mount_point: runtime/mount/point
+  store_and_clean_1:
+    plugin: AiiDA Shell
+    command: $PWD/files/scripts/post_clean.sh
+    nodes: 1
+    walltime: '00:01:00'
+  store_and_clean_2:
+    plugin: AiiDA Shell
+    command: $PWD/files/scripts/post_clean.sh
+    nodes: 1
+    walltime: '00:01:00'
+data:
+  preproc_output:
+    type: file
+    src: $PWD/files/data/file.ncdf
+  grid_file:
+    type: file
+    src: $PWD/files/data/grid
+  obs_data:
+    type: file
+    src: $PWD/files/data/obs_data
+  ERA5:
+    type: file
+    src: $PWD/files/data/era5
+  extpar_file:
+    type: file
+    src: output
+  icon_input:
+    type: file
+    src: output
+  icon_restart:
+    type: file
+    format: ncdf
+    src: restart
+  stream_1:
+    type: file
+    src: output_1
+  stream_2:
+    type: file
+    src: output_2
+  postout_1:
+    type: file
+    src: postout
+  postout_2:
+    type: file
+    src: postout
+  stored_data_1:
+    type: file
+    src: stored_data
+  stored_data_2:
+    type: file
+    src: stored_data

examples/files/scripts/cleanup.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+
+def main():
+    # Main script execution continues here
+    print("Cleaning")
+
+
+if __name__ == '__main__':
+    main()

examples/files/scripts/cleanup.sh

@@ -0,0 +1 @@
+echo "cleanup" > output

examples/files/scripts/extpar

@@ -0,0 +1 @@
+echo "extpar" > output

examples/files/scripts/icon

@@ -0,0 +1,4 @@
+echo "icon" > restart
+echo "icon" > output
+echo "icon" > output_1
+echo "icon" > output_2

examples/files/scripts/icon.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+
+import argparse
+from pathlib import Path
+
+def main():
+    parser = argparse.ArgumentParser(description='A simple script with an optional restart argument.')
+    parser.add_argument('icon_input', type=str, help='The icon input.')
+    parser.add_argument('--restart', nargs='?', type=str, help='The icon restart file.')
+    #parser.add_argument('--restart', nargs='?', const='default', type=str, help='Initiate a restart operation with an optional string argument.')
+
+
+    args = parser.parse_args()
+
+    output = Path('output')
+    output.write_text("")
+
+    if args.restart:
+        restart = Path(args.restart)
+        restart.read_text()
+        text = "Restart operation initiated..."
+        print(text)
+        with output.open("a") as f:
+            f.write(text)
+    else:
+        text = "No restart option provided. Continuing without restart."
+        print(text)
+        with output.open("a") as f:
+            f.write(text)
+
+    # Main script execution continues here
+    text = "Script execution continues..."
+    print(text)
+    with output.open("a") as f:
+        f.write(text)
+
+    restart = Path('restart')
+    restart.write_text("")
+
+if __name__ == '__main__':
+    main()