chore: Merge branch 'main' into schema-load-arg-struct

pywr-core/src/models/mod.rs

@@ -7,6 +7,7 @@ use crate::PywrError;
 pub use multi::{MultiNetworkModel, MultiNetworkTransferIndex};
 pub use simple::{Model, ModelState};
 
+#[derive(Debug)]
 pub struct ModelDomain {
     time: TimeDomain,
     scenarios: ScenarioDomain,

pywr-core/src/recorders/metric_set.rs

@@ -109,11 +109,27 @@ impl MetricSet {
                 .as_mut()
                 .expect("Aggregation state expected for metric set with aggregator!");
 
-            let agg_values = values
-                .into_iter()
-                .zip(aggregation_states.iter_mut())
-                .map(|(value, current_state)| aggregator.append_value(current_state, value))
-                .collect::<Option<Vec<_>>>();
+            // Collect any aggregated values. This will remain empty if the aggregator yields
+            // no values. However, if there are values we will expect the same number of aggregated
+            // values as the input values / metrics.
+            let mut agg_values = Vec::with_capacity(values.len());
+            // Use a for loop instead of using an iterator because we need to execute the
+            // `append_value` method on all aggregators.
+            for (value, current_state) in values.iter().zip(aggregation_states.iter_mut()) {
+                if let Some(agg_value) = aggregator.append_value(current_state, *value) {
+                    agg_values.push(agg_value);
+                }
+            }
+
+            let agg_values = if agg_values.is_empty() {
+                None
+            } else if agg_values.len() == values.len() {
+                Some(agg_values)
+            } else {
+                // This should never happen because the aggregator should either yield no values
+                // or the same number of values as the input metrics.
+                unreachable!("Some values were aggregated and some were not!");
+            };
 
             internal_state.current_values = agg_values;
         } else {

pywr-core/src/scenario.rs

@@ -31,6 +31,10 @@ pub struct ScenarioGroupCollection {
 }
 
 impl ScenarioGroupCollection {
+    pub fn new(groups: Vec<ScenarioGroup>) -> Self {
+        Self { groups }
+    }
+
     /// Number of [`ScenarioGroup`]s in the collection.
     pub fn len(&self) -> usize {
         self.groups.len()
@@ -105,6 +109,7 @@ impl ScenarioIndex {
     }
 }
 
+#[derive(Debug)]
 pub struct ScenarioDomain {
     scenario_indices: Vec<ScenarioIndex>,
     scenario_group_names: Vec<String>,

pywr-core/src/timestep.rs

@@ -54,10 +54,15 @@ impl PywrDuration {
         }
     }
 
-    // Returns the fractional number of days in the duration.
+    /// Returns the fractional number of days in the duration.
     pub fn fractional_days(&self) -> f64 {
         self.0.num_seconds() as f64 / SECS_IN_DAY as f64
     }
+
+    /// Returns the number of nanoseconds in the duration.
+    pub fn whole_nanoseconds(&self) -> Option<i64> {
+        self.0.num_nanoseconds()
+    }
 }
 
 type TimestepIndex = usize;
@@ -187,6 +192,7 @@ impl Timestepper {
 }
 
 /// The time domain that a model will be simulated over.
+#[derive(Debug)]
 pub struct TimeDomain {
     timesteps: Vec<Timestep>,
 }
@@ -209,6 +215,14 @@ impl TimeDomain {
         self.timesteps.len()
     }
 
+    pub fn first_timestep(&self) -> &Timestep {
+        self.timesteps.first().expect("No time-steps defined.")
+    }
+
+    pub fn last_timestep(&self) -> &Timestep {
+        self.timesteps.last().expect("No time-steps defined.")
+    }
+
     pub fn is_empty(&self) -> bool {
         self.timesteps.is_empty()
     }

pywr-python/tests/models/aggregated-node1/model.json

@@ -13,8 +13,12 @@
         "name": "input1",
         "type": "Input",
         "max_flow": {
-          "type": "Parameter",
-          "name": "inflow"
+          "type": "Timeseries",
+          "name": "inflow",
+          "columns": {
+            "type": "Column",
+            "name": "inflow"
+          }
         }
       },
       {
@@ -69,17 +73,14 @@
         "name": "demand",
         "type": "Constant",
         "value": 10.0
-      },
+      }
+    ],
+    "timeseries": [
       {
         "name": "inflow",
-        "type": "DataFrame",
-        "url": "inflow.csv",
-        "pandas_kwargs": {
-          "index_col": 0
-        },
-        "columns": {
-          "type": "Column",
-          "name": "inflow"
+        "provider": {
+          "type": "Polars",
+          "url": "inflow.csv"
         }
       }
     ],

pywr-python/tests/models/piecewise-link1/model.json

@@ -13,12 +13,20 @@
         "name": "input1",
         "type": "Input",
         "max_flow": {
-          "type": "Parameter",
-          "name": "inflow"
+          "type": "Timeseries",
+          "name": "inflow",
+          "columns": {
+            "type": "Column",
+            "name": "inflow"
+          }
         },
         "min_flow": {
-          "type": "Parameter",
-          "name": "inflow"
+          "type": "Timeseries",
+          "name": "inflow",
+          "columns": {
+            "type": "Column",
+            "name": "inflow"
+          }
         }
       },
       {
@@ -74,17 +82,14 @@
         "name": "demand",
         "type": "Constant",
         "value": 10.0
-      },
+      }
+    ],
+    "timeseries": [
       {
         "name": "inflow",
-        "type": "DataFrame",
-        "url": "inflow.csv",
-        "pandas_kwargs": {
-          "index_col": 0
-        },
-        "columns": {
-          "type": "Column",
-          "name": "inflow"
+        "provider": {
+          "type": "Polars",
+          "url": "inflow.csv"
         }
       }
     ],

pywr-python/tests/models/simple-custom-parameter/model.json

@@ -13,8 +13,12 @@
         "name": "input1",
         "type": "Input",
         "max_flow": {
-          "type": "Parameter",
-          "name": "inflow"
+          "type": "Timeseries",
+          "name": "inflow",
+          "columns": {
+            "type": "Column",
+            "name": "inflow"
+          }
         }
       },
       {
@@ -53,17 +57,14 @@
         "kwargs": {
           "multiplier": 2.0
         }
-      },
+      }
+    ],
+    "timeseries": [
       {
         "name": "inflow",
-        "type": "DataFrame",
-        "url": "inflow.csv",
-        "pandas_kwargs": {
-          "index_col": 0
-        },
-        "columns": {
-          "type": "Column",
-          "name": "inflow"
+        "provider": {
+          "type": "Polars",
+          "url": "inflow.csv"
         }
       }
     ],

pywr-python/tests/models/simple-timeseries/model.json

@@ -13,8 +13,12 @@
         "name": "input1",
         "type": "Input",
         "max_flow": {
-          "type": "Parameter",
-          "name": "inflow"
+          "type": "Timeseries",
+          "name": "inflow",
+          "columns": {
+            "type": "Column",
+            "name": "inflow"
+          }
         }
       },
       {
@@ -46,17 +50,14 @@
         "name": "demand",
         "type": "Constant",
         "value": 10.0
-      },
+      }
+    ],
+    "timeseries": [
       {
         "name": "inflow",
-        "type": "DataFrame",
-        "url": "inflow.csv",
-        "pandas_kwargs": {
-          "index_col": 0
-        },
-        "columns": {
-          "type": "Column",
-          "name": "inflow"
+        "provider": {
+          "type": "Polars",
+          "url": "inflow.csv"
         }
       }
     ],

pywr-python/tests/models/simple-wasm/model.json

@@ -10,8 +10,12 @@
         "name": "input1",
         "type": "input",
         "max_flow": {
-          "type": "Parameter",
-          "name": "inflow"
+          "type": "Timeseries",
+          "name": "inflow",
+          "columns": {
+            "type": "Column",
+            "name": "inflow"
+          }
         }
       },
       {
@@ -64,6 +68,15 @@
         "url": "inflow.csv.gz",
         "column": "inflow"
       }
+    ],
+    "timeseries": [
+      {
+        "name": "inflow",
+        "provider": {
+          "type": "Polars",
+          "url": "inflow.csv.gz"
+        }
+      }
     ]
   }
 }

pywr-schema/Cargo.toml

@@ -15,7 +15,7 @@ categories = ["science", "simulation"]
 
 [dependencies]
 svgbobdoc = { version = "0.3.0", features = ["enable"]  }
-polars = {  workspace = true }
+polars = {  workspace = true, features = ["csv", "diff", "dtype-datetime", "dtype-date", "dynamic_group_by"] }
 pyo3 = {  workspace = true }
 pyo3-polars = {  workspace = true }
 strum = "0.26"

pywr-schema/src/error.rs

@@ -1,5 +1,6 @@
 use crate::data_tables::TableError;
 use crate::nodes::NodeAttribute;
+use crate::timeseries::TimeseriesError;
 use pyo3::exceptions::PyRuntimeError;
 use pyo3::PyErr;
 use thiserror::Error;
@@ -54,6 +55,8 @@ pub enum SchemaError {
     InvalidRollingWindow { name: String },
     #[error("Failed to load parameter {name}: {error}")]
     LoadParameter { name: String, error: String },
+    #[error("Timeseries error: {0}")]
+    Timeseries(#[from] TimeseriesError),
 }
 
 impl From<SchemaError> for PyErr {
@@ -103,4 +106,10 @@ pub enum ConversionError {
     UnparseableDate(String),
     #[error("Chrono out of range error: {0}")]
     OutOfRange(#[from] chrono::OutOfRange),
+    #[error("The dataframe parameters '{0}' defines both a column and a scenario attribute. Only 1 is allowed.")]
+    AmbiguousColumnAndScenario(String),
+    #[error("The dataframe parameters '{0}' defines both a column and a scenario. Only 1 is allowed.")]
+    MissingColumnOrScenario(String),
+    #[error("Unable to create a timeseries for file: '{0}'. No name was found.")]
+    MissingTimeseriesName(String),
 }

pywr-schema/src/lib.rs

@@ -12,6 +12,7 @@ pub mod model;
 pub mod nodes;
 pub mod outputs;
 pub mod parameters;
+pub mod timeseries;
 
 pub use error::{ConversionError, SchemaError};
 pub use model::PywrModel;