Merge pull request #101 from Project-OMOTES/97-integrate-changes-to-p…

…arams_dict-conversion-code-so-params_dict-now-allows-for-time-durations

97: Update all usages of SDK to latest one, change orchestrator confi…

CHANGELOG.md

@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [mvp.4.RC2] - 2024-0x-xx
 
+Compatible with Python SDK 3.1.2
+
+This version uses the following models:
+- Mesido 0.1.7.0
+- simulator-core 0.0.8
+
 ### Omotes-system
 
 * 47: Add README.md and ci scripts for system_tests and example_sdk_client providing easy info and
@@ -28,5 +34,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   @lfse-slafleur in https://github.com/Project-OMOTES/omotes-system/pull/93
 * 95: Update optimizer worker to 2.0.2 which includes mesido 0.1.7.0. by @lfse-slafleur
   in https://github.com/Project-OMOTES/omotes-system/pull/96
-* 99: Set the workflows definitions file via environment variable by @MarkTNO in
+* 99: Set the workflows definitions file via environment variable by @MarkTNO
   in https://github.com/Project-OMOTES/omotes-system/pull/100
+* 81: Make RabbitMQ port configurable as env var. by @lfse-slafleur in 
+  https://github.com/Project-OMOTES/omotes-system/pull/94
+* 98: Make RabbitMQ management port configurable through env var. by @lfse-slafleur in
+  https://github.com/Project-OMOTES/omotes-system/pull/102
+* 97: Update all usages of SDK to latest one, change orchestrator config by removing any test 
+  workers and fixing the parameters expected for simulator workflow. by @lfse-slafleur
+  in https://github.com/Project-OMOTES/omotes-system/pull/101
+  * This introduces also a delivery limit feature where jobs are retried until the delivery limit
+    is reached (can be configured in .env file). This is useful in case of OOM errors and other
+    hard crashes of the worker which may keep a job retried indefinitely.
+  * This also introduces a cleanup feature where old queues are removed after 48 hours of
+    inactivity. After 48 hours, we assume the SDK is not interested in the result of a job anymore.
+  * start_time, end_time and timestep parameters are now all submitted with a simulator workflow
+    job submission.

config/workflow_config_example.json

@@ -61,6 +61,14 @@
         "parameter_type": "datetime",
         "key_name": "datetime_example_with_default",
         "default": "2023-12-31T20:00:00"
+      },
+      {
+        "parameter_type": "duration",
+        "key_name": "duration_example_with_default",
+        "description": "The default is in seconds.",
+        "default": 900.0,
+        "minimum": 0.0,
+        "maximum": 1500.0
       }
     ]
   }

config/workflow_config_nwn.json

@@ -17,48 +17,21 @@
   },
   {
     "workflow_type_name": "simulator",
-    "workflow_type_description_name": "High fidelity simulator"
-  },
-  {
-    "workflow_type_name": "test_worker",
-    "workflow_type_description_name": "Used for testing purposes. Should not be used in production environments.",
+    "workflow_type_description_name": "High fidelity simulator",
     "workflow_parameters": [
+      {
+        "parameter_type": "duration",
+        "key_name": "timestep",
+        "description": "Duration of a timestep in seconds.",
+        "minimum": 0
+      },
       {
         "parameter_type": "datetime",
         "key_name": "start_time"
       },
       {
         "parameter_type": "datetime",
         "key_name": "end_time"
-      },
-      {
-        "parameter_type": "integer",
-        "key_name": "step_size_in_minutes",
-        "title": "This will override the 'capitalized, underscore to space', key_name",
-        "description": "Description/explanation.",
-        "default": 900,
-        "minimum": 1,
-        "maximum": 525600
-      },
-      {
-        "parameter_type": "integer",
-        "key_name": "number_of_steps",
-        "minimum": 1
-      },
-      {
-        "parameter_type": "string",
-        "key_name": "test_enum",
-        "description": "How to use enum options",
-        "enum_options": [
-          {
-            "key_name": "key1",
-            "display_name": "Display name 1"
-          },
-          {
-            "key_name": "key2",
-            "display_name": "Display name 2"
-          }
-        ]
       }
     ]
   }

config/workflow_config_nwn_9sept2024.json

@@ -17,6 +17,22 @@
   },
   {
     "workflow_type_name": "simulator",
-    "workflow_type_description_name": "Conceptual Design - Simulation"
+    "workflow_type_description_name": "Conceptual Design - Simulation",
+    "workflow_parameters": [
+      {
+        "parameter_type": "duration",
+        "key_name": "timestep",
+        "description": "Duration of a timestep in seconds.",
+        "minimum": 0
+      },
+      {
+        "parameter_type": "datetime",
+        "key_name": "start_time"
+      },
+      {
+        "parameter_type": "datetime",
+        "key_name": "end_time"
+      }
+    ]
   }
 ]

docker-compose.yml

@@ -86,7 +86,7 @@ services:
       - manual_dev
 
   orchestrator:
-    image: ghcr.io/project-omotes/omotes_orchestrator:1.0.0
+    image: ghcr.io/project-omotes/omotes_orchestrator:1.1.0
     restart: unless-stopped
     networks:
       - omotes
@@ -125,7 +125,7 @@ services:
         condition: service_healthy
 
   grow_worker_optimizer: &grow_worker
-    image: ghcr.io/project-omotes/omotes-grow-optimizer-worker:2.0.2
+    image: ghcr.io/project-omotes/omotes-grow-optimizer-worker:2.0.3
     restart: unless-stopped
     deploy:
       replicas: 2
@@ -168,7 +168,7 @@ services:
       GROW_TASK_TYPE: grow_optimizer_with_pressure
 
   omotes_simulator_worker:
-    image: ghcr.io/project-omotes/omotes-simulator-worker:0.0.9
+    image: ghcr.io/project-omotes/omotes-simulator-worker:0.0.11
     restart: unless-stopped
     deploy:
       replicas: 2

example_sdk_client/requirements.txt

@@ -1 +1 @@
-omotes_sdk_python==3.0.0
+omotes_sdk_python==3.1.2

system_tests/requirements.txt

@@ -1,4 +1,4 @@
-omotes_sdk_python==3.0.0
+omotes_sdk_python==3.1.2
 pytest ~= 8.1.2
 pytest-timeout
 xmltodict ~= 0.13.0