WIP: Shell tasks for WorkGraph with pinned old version

code.yaml

@@ -0,0 +1,8 @@
+append_text: ''
+computer: localhost
+default_calc_job_plugin: core.shell
+description: ''
+filepath_executable: /home/geiger_j/aiida_projects/swiss-twins/git-repos/Sirocco/tests/files/scripts/shell_task.sh
+label: tests/files/scripts/shell_task.sh
+prepend_text: ''
+use_double_quotes: 'False'

examples/files/configs/example_config.yaml

@@ -0,0 +1,181 @@
+start_date: '2026-01-01T00:00'
+end_date: '2027-01-01T00:00'
+cycles:
+  init:
+    tasks:
+      extpar:
+        inputs:
+          - obs_data:
+              arg_option: "--input"
+        outputs:
+          - extpar_file
+  icon_bimonthly:
+    period: 'P2M'
+    tasks:
+      preproc:
+        inputs:
+          - grid_file:
+              arg_option: "-g"
+          - extpar_file:
+              date: '2026-01-01T00:00'
+              arg_option: "-p"
+          - ERA5:
+              arg_option: "-e"
+        outputs:
+          - icon_input
+        depends:
+          - icon:
+              lag: '-P4M'
+      icon:
+        inputs:
+          - grid_file:
+              arg_option: "-g"
+          - icon_input:
+              arg_option: "--input"
+          #- icon_restart:
+          #    lag: '-P2M'
+        outputs:
+          - stream_1
+          - stream_2
+          #- icon_restart
+      postproc_1:
+        inputs:
+          - stream_1:
+              arg_option: "--input"
+        outputs:
+          - postout_1
+      store_and_clean_1:
+        inputs:
+          - postout_1:
+              arg_option: "--input"
+          - stream_1:
+              arg_option: "--stream"
+          - icon_input:
+              arg_option: "--icon_input"
+        outputs:
+          - stored_data_1
+  yearly:
+    period: 'P1Y'
+    tasks:
+      postproc_2:
+        inputs:
+          - stream_2:
+             lag: 
+               - 'P0M'
+             arg_option: "--input"
+        outputs:
+          - postout_2
+      store_and_clean_2:
+        inputs:
+          - postout_2:
+             arg_option: "--input"
+          - stream_2:
+             lag: 
+               - 'P0M'
+        outputs:
+          - stored_data_2
+# Each task and piece of data (input and output of tasks) used to
+# define the graph is described in that section
+tasks:
+  root:
+    # All tasks inherit the root task properties
+    host: santis
+    account: g110
+  extpar:
+    plugin: extpar
+    command: $PWD/files/scripts/extpar
+    command_option: "--verbose" # todo implement support
+    config: path/to/namelists/dir
+    uenv:
+      squashfs: path/to/squashfs
+      mount_point: runtime/mount/point
+    nodes: 1
+    walltime: '00:02:00'
+  preproc:
+    plugin: AiiDA Shell
+    command: $PWD/files/scripts/cleanup.sh
+    nodes: 4
+    walltime: '00:02:00'
+    config: path/to/config/dir
+    uenv:
+      squashfs: path/to/squashfs
+      mount_point: runtime/mount/point
+  icon:
+    plugin: icon
+    command: $PWD/files/scripts/icon
+    nodes: 40
+    walltime: '23:59:59'
+    config: path/to/namelists/dir
+    uenv:
+      squashfs: path/to/squashfs
+      mount_point: runtime/mount/point
+  postproc_1:
+    plugin: AiiDA Shell
+    command: $PWD/files/scripts/main_script_ocn.sh
+    nodes: 2
+    walltime: '00:05:00'
+    conda_env: path/to/yaml/env/file
+    uenv:
+      squashfs: path/to/squashfs
+      mount_point: runtime/mount/point
+  postproc_2:
+    plugin: AiiDA Shell
+    command: $PWD/files/scripts/main_script_atm.sh
+    nodes: 2
+    walltime: '00:05:00'
+    src: path/to/src/dir
+    conda_env: path/to/yaml/env/file
+    uenv:
+      squashfs: path/to/squashfs
+      mount_point: runtime/mount/point
+  store_and_clean_1:
+    plugin: AiiDA Shell
+    command: $PWD/files/scripts/post_clean.sh
+    nodes: 1
+    walltime: '00:01:00'
+  store_and_clean_2:
+    plugin: AiiDA Shell
+    command: $PWD/files/scripts/post_clean.sh
+    nodes: 1
+    walltime: '00:01:00'
+data:
+  preproc_output:
+    type: file
+    src: $PWD/files/data/file.ncdf
+  grid_file:
+    type: file
+    src: $PWD/files/data/grid
+  obs_data:
+    type: file
+    src: $PWD/files/data/obs_data
+  ERA5:
+    type: file
+    src: $PWD/files/data/era5
+  extpar_file:
+    type: file
+    src: output
+  icon_input:
+    type: file
+    src: output
+  icon_restart:
+    type: file
+    format: ncdf
+    src: restart
+  stream_1:
+    type: file
+    src: output_1
+  stream_2:
+    type: file
+    src: output_2
+  postout_1:
+    type: file
+    src: postout
+  postout_2:
+    type: file
+    src: postout
+  stored_data_1:
+    type: file
+    src: stored_data
+  stored_data_2:
+    type: file
+    src: stored_data

examples/files/scripts/cleanup.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+
+def main():
+    # Main script execution continues here
+    print("Cleaning")
+
+
+if __name__ == '__main__':
+    main()

examples/files/scripts/cleanup.sh

@@ -0,0 +1 @@
+echo "cleanup" > output

examples/files/scripts/extpar

@@ -0,0 +1 @@
+echo "extpar" > output

examples/files/scripts/icon

@@ -0,0 +1,4 @@
+echo "icon" > restart
+echo "icon" > output
+echo "icon" > output_1
+echo "icon" > output_2

examples/files/scripts/icon.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+
+import argparse
+from pathlib import Path
+
+def main():
+    parser = argparse.ArgumentParser(description='A simple script with an optional restart argument.')
+    parser.add_argument('icon_input', type=str, help='The icon input.')
+    parser.add_argument('--restart', nargs='?', type=str, help='The icon restart file.')
+    #parser.add_argument('--restart', nargs='?', const='default', type=str, help='Initiate a restart operation with an optional string argument.')
+
+
+    args = parser.parse_args()
+
+    output = Path('output')
+    output.write_text("")
+
+    if args.restart:
+        restart = Path(args.restart)
+        restart.read_text()
+        text = "Restart operation initiated..."
+        print(text)
+        with output.open("a") as f:
+            f.write(text)
+    else:
+        text = "No restart option provided. Continuing without restart."
+        print(text)
+        with output.open("a") as f:
+            f.write(text)
+
+    # Main script execution continues here
+    text = "Script execution continues..."
+    print(text)
+    with output.open("a") as f:
+        f.write(text)
+
+    restart = Path('restart')
+    restart.write_text("")
+
+if __name__ == '__main__':
+    main()

examples/files/scripts/icon.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Function to write text to the output file
+write_to_output() {
+  local text="$1"
+  echo "$text"
+  echo "$text" >> output
+}
+
+# Check if at least one argument is provided
+if [ "$#" -lt 1 ]; then
+  echo "Usage: $0 icon_input [--restart [restart_file]]"
+  exit 1
+fi
+
+# Positional argument
+icon_input="$1"
+
+# Optional restart argument
+restart_file=""
+
+if [ "$2" == "--restart" ]; then
+  if [ -n "$3" ]; then
+    restart_file="$3"
+  fi
+fi
+
+# Create/empty the output file
+> output
+
+# Handling restart if the argument is provided
+if [ -n "$restart_file" ]; then
+  if [ -f "$restart_file" ]; then
+    cat "$restart_file" > /dev/null
+    write_to_output "Restart operation initiated..."
+  else
+    echo "Restart file $restart_file does not exist."
+    exit 1
+  fi
+else
+  write_to_output "No restart option provided. Continuing without restart."
+fi
+
+# Main script execution continues here
+write_to_output "Script execution continues..."
+
+# Create/empty the restart file
+> restart

examples/files/scripts/main_script_atm.sh

@@ -0,0 +1 @@
+echo "main_script_atm.sh" > postout

examples/files/scripts/main_script_ocn.sh

@@ -0,0 +1 @@
+echo "python main_script_ocn.sh" > postout

examples/files/scripts/post_clean.sh

@@ -0,0 +1 @@
+echo "store_and_clean" > stored_data

pyproject.toml

@@ -32,8 +32,9 @@ dependencies = [
   "pydantic-yaml",
   "aiida-core>=2.5",
   "termcolor",
-  "pygraphviz",
-  "lxml"
+  "aiida-core~=2.5",
+  "aiida-workgraph==0.3.14",
+  "node_graph==0.0.11",
 ]
 license = {file = "LICENSE"}
 

src/sirocco/core/graph_items.py

@@ -2,6 +2,8 @@
 
 from dataclasses import dataclass, field
 from itertools import chain, product
+from os.path import expandvars
+from pathlib import Path
 from typing import TYPE_CHECKING, Any, ClassVar, Self
 
 from sirocco.parsing._yaml_data_models import (
@@ -106,6 +108,11 @@ def from_config(cls, config: ConfigBaseData, coordinates: dict) -> Self:
             coordinates=coordinates,
         )
 
+    @property
+    def path(self) -> Path:
+        # TODO yaml level?
+        return Path(expandvars(self.src))
+
 
 @dataclass
 class Cycle(GraphItem):