feat: add timeseries section to model schema

pywr-core/src/models/mod.rs

@@ -7,6 +7,7 @@ use crate::PywrError;
 pub use multi::{MultiNetworkModel, MultiNetworkTransferIndex};
 pub use simple::{Model, ModelState};
 
+#[derive(Debug)]
 pub struct ModelDomain {
     time: TimeDomain,
     scenarios: ScenarioDomain,

pywr-core/src/scenario.rs

@@ -31,6 +31,10 @@ pub struct ScenarioGroupCollection {
 }
 
 impl ScenarioGroupCollection {
+    pub fn new(groups: Vec<ScenarioGroup>) -> Self {
+        Self { groups }
+    }
+
     /// Number of [`ScenarioGroup`]s in the collection.
     pub fn len(&self) -> usize {
         self.groups.len()
@@ -105,6 +109,7 @@ impl ScenarioIndex {
     }
 }
 
+#[derive(Debug)]
 pub struct ScenarioDomain {
     scenario_indices: Vec<ScenarioIndex>,
     scenario_group_names: Vec<String>,

pywr-core/src/timestep.rs

@@ -58,6 +58,10 @@ impl PywrDuration {
     pub fn fractional_days(&self) -> f64 {
         self.0.num_seconds() as f64 / SECS_IN_DAY as f64
     }
+
+    pub fn whole_nanoseconds(&self) -> Option<i64> {
+        self.0.num_nanoseconds()
+    }
 }
 
 type TimestepIndex = usize;
@@ -187,6 +191,7 @@ impl Timestepper {
 }
 
 /// The time domain that a model will be simulated over.
+#[derive(Debug)]
 pub struct TimeDomain {
     timesteps: Vec<Timestep>,
 }
@@ -209,6 +214,14 @@ impl TimeDomain {
         self.timesteps.len()
     }
 
+    pub fn first_timestep(&self) -> &Timestep {
+        self.timesteps.first().expect("No time-steps defined.")
+    }
+
+    pub fn last_timestep(&self) -> &Timestep {
+        self.timesteps.last().expect("No time-steps defined.")
+    }
+
     pub fn is_empty(&self) -> bool {
         self.timesteps.is_empty()
     }

pywr-schema/Cargo.toml

@@ -15,7 +15,7 @@ categories = ["science", "simulation"]
 
 [dependencies]
 svgbobdoc = { version = "0.3.0", features = ["enable"]  }
-polars = {  workspace = true }
+polars = {  workspace = true, features = ["csv", "diff", "dtype-datetime", "dtype-date", "dynamic_group_by"] }
 pyo3 = {  workspace = true }
 pyo3-polars = {  workspace = true }
 strum = "0.26"

pywr-schema/src/error.rs

@@ -1,5 +1,7 @@
 use crate::data_tables::TableError;
 use crate::nodes::NodeAttribute;
+use crate::timeseries::TimeseriesError;
+use polars::error::PolarsError;
 use pyo3::exceptions::PyRuntimeError;
 use pyo3::PyErr;
 use thiserror::Error;
@@ -54,6 +56,8 @@ pub enum SchemaError {
     InvalidRollingWindow { name: String },
     #[error("Failed to load parameter {name}: {error}")]
     LoadParameter { name: String, error: String },
+    #[error("Timeseries error: {0}")]
+    Timeseries(#[from] TimeseriesError),
 }
 
 impl From<SchemaError> for PyErr {
@@ -103,4 +107,8 @@ pub enum ConversionError {
     UnparseableDate(String),
     #[error("Chrono out of range error: {0}")]
     OutOfRange(#[from] chrono::OutOfRange),
+    #[error("The dataframe parameters '{0}' defines both a column and a scenario attribute. Only 1 is allowed.")]
+    AmbiguousColumnAndScenario(String),
+    #[error("The dataframe parameters '{0}' defines both a column and a scenario. Only 1 is allowed.")]
+    MissingColumnOrScenario(String),
 }

pywr-schema/src/lib.rs

@@ -12,6 +12,7 @@ pub mod model;
 pub mod nodes;
 pub mod outputs;
 pub mod parameters;
+pub mod timeseries;
 
 pub use error::{ConversionError, SchemaError};
 pub use model::PywrModel;

pywr-schema/src/model.rs

@@ -4,12 +4,21 @@ use super::parameters::Parameter;
 use crate::data_tables::{DataTable, LoadedTableCollection};
 use crate::error::{ConversionError, SchemaError};
 use crate::metric_sets::MetricSet;
+use crate::nodes::NodeAndTimeseries;
 use crate::outputs::Output;
-use crate::parameters::{MetricFloatReference, TryIntoV2Parameter};
+use crate::parameters::{
+    convert_parameter_v1_to_v2, DataFrameColumns, DynamicFloatValue, MetricFloatReference, MetricFloatValue,
+    TimeseriesReference, TryIntoV2Parameter,
+};
+use crate::timeseries::{convert_from_v1_data, LoadedTimeseriesCollection, Timeseries};
 use chrono::{NaiveDate, NaiveDateTime, NaiveTime};
+use polars::frame::DataFrame;
 use pywr_core::models::ModelDomain;
 use pywr_core::timestep::TimestepDuration;
 use pywr_core::PywrError;
+use serde::de;
+use std::collections::HashSet;
+use std::hash::Hash;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 
@@ -140,6 +149,7 @@ pub struct PywrNetwork {
     pub edges: Vec<Edge>,
     pub parameters: Option<Vec<Parameter>>,
     pub tables: Option<Vec<DataTable>>,
+    pub timeseries: Option<Vec<Timeseries>>,
     pub metric_sets: Option<Vec<MetricSet>>,
     pub outputs: Option<Vec<Output>>,
 }
@@ -192,16 +202,25 @@ impl PywrNetwork {
         // Load all the data tables
         let tables = LoadedTableCollection::from_schema(self.tables.as_deref(), data_path)?;
 
+        // Load all timeseries data
+        let timeseries = LoadedTimeseriesCollection::from_schema(self.timeseries.as_deref(), domain, data_path)?;
+
         // Create all the nodes
         let mut remaining_nodes = self.nodes.clone();
 
         while !remaining_nodes.is_empty() {
             let mut failed_nodes: Vec<Node> = Vec::new();
             let n = remaining_nodes.len();
             for node in remaining_nodes.into_iter() {
-                if let Err(e) =
-                    node.add_to_model(&mut network, self, domain, &tables, data_path, inter_network_transfers)
-                {
+                if let Err(e) = node.add_to_model(
+                    &mut network,
+                    &self,
+                    domain,
+                    &tables,
+                    data_path,
+                    inter_network_transfers,
+                    &timeseries,
+                ) {
                     // Adding the node failed!
                     match e {
                         SchemaError::PywrCore(core_err) => match core_err {
@@ -252,9 +271,15 @@ impl PywrNetwork {
                 let mut failed_parameters: Vec<Parameter> = Vec::new();
                 let n = remaining_parameters.len();
                 for parameter in remaining_parameters.into_iter() {
-                    if let Err(e) =
-                        parameter.add_to_model(&mut network, self, domain, &tables, data_path, inter_network_transfers)
-                    {
+                    if let Err(e) = parameter.add_to_model(
+                        &mut network,
+                        self,
+                        domain,
+                        &tables,
+                        data_path,
+                        inter_network_transfers,
+                        &timeseries,
+                    ) {
                         // Adding the parameter failed!
                         match e {
                             SchemaError::PywrCore(core_err) => match core_err {
@@ -280,7 +305,15 @@ impl PywrNetwork {
 
         // Apply the inline parameters & constraints to the nodes
         for node in &self.nodes {
-            node.set_constraints(&mut network, self, domain, &tables, data_path, inter_network_transfers)?;
+            node.set_constraints(
+                &mut network,
+                self,
+                domain,
+                &tables,
+                data_path,
+                inter_network_transfers,
+                &timeseries,
+            )?;
         }
 
         // Create all of the metric sets
@@ -412,8 +445,9 @@ impl PywrModel {
             Timestepper::default()
         });
 
-        let nodes = v1
+        let nodes_and_ts: Vec<NodeAndTimeseries> = v1
             .nodes
+            .clone()
             .into_iter()
             .filter_map(|n| match n.try_into() {
                 Ok(n) => Some(n),
@@ -424,22 +458,29 @@ impl PywrModel {
             })
             .collect::<Vec<_>>();
 
+        let mut ts_data = nodes_and_ts
+            .iter()
+            .filter_map(|n| n.timeseries.clone())
+            .flatten()
+            .collect::<Vec<_>>();
+
+        let nodes = nodes_and_ts.into_iter().map(|n| n.node).collect::<Vec<_>>();
+
         let edges = v1.edges.into_iter().map(|e| e.into()).collect();
 
-        let parameters = if let Some(v1_parameters) = v1.parameters {
+        let (parameters, param_ts_data) = if let Some(v1_parameters) = v1.parameters {
             let mut unnamed_count: usize = 0;
-            Some(
-                v1_parameters
-                    .into_iter()
-                    .filter_map(|p| match p.try_into_v2_parameter(None, &mut unnamed_count) {
-                        Ok(p) => Some(p),
-                        Err(e) => {
-                            errors.push(e);
-                            None
-                        }
-                    })
-                    .collect::<Vec<_>>(),
-            )
+            let (parameters, param_ts_data) = convert_parameter_v1_to_v2(v1_parameters, &mut unnamed_count, &mut errors);
+            (Some(parameters), Some(param_ts_data))
+        } else {
+            (None, None)
+        };
+
+        ts_data.extend(param_ts_data.into_iter().flatten());
+
+        let timeseries = if !ts_data.is_empty() {
+            let ts = convert_from_v1_data(&ts_data, &v1.tables);
+            Some(ts)
         } else {
             None
         };
@@ -453,6 +494,7 @@ impl PywrModel {
             edges,
             parameters,
             tables,
+            timeseries,
             metric_sets,
             outputs,
         };

pywr-schema/src/nodes/annual_virtual_storage.rs

@@ -4,6 +4,7 @@ use crate::model::PywrMultiNetworkTransfer;
 use crate::nodes::core::StorageInitialVolume;
 use crate::nodes::{NodeAttribute, NodeMeta};
 use crate::parameters::{DynamicFloatValue, TryIntoV2Parameter};
+use crate::timeseries::LoadedTimeseriesCollection;
 use pywr_core::derived_metric::DerivedMetric;
 use pywr_core::metric::Metric;
 use pywr_core::models::ModelDomain;
@@ -14,7 +15,7 @@ use pywr_v1_schema::nodes::AnnualVirtualStorageNode as AnnualVirtualStorageNodeV
 use std::collections::HashMap;
 use std::path::Path;
 
-#[derive(serde::Deserialize, serde::Serialize, Clone)]
+#[derive(serde::Deserialize, serde::Serialize, Clone, Debug)]
 pub struct AnnualReset {
     pub day: u8,
     pub month: chrono::Month,
@@ -31,7 +32,7 @@ impl Default for AnnualReset {
     }
 }
 
-#[derive(serde::Deserialize, serde::Serialize, Clone, Default, PywrNode)]
+#[derive(serde::Deserialize, serde::Serialize, Clone, Default, Debug, PywrNode)]
 pub struct AnnualVirtualStorageNode {
     #[serde(flatten)]
     pub meta: NodeMeta,
@@ -55,24 +56,49 @@ impl AnnualVirtualStorageNode {
         tables: &LoadedTableCollection,
         data_path: Option<&Path>,
         inter_network_transfers: &[PywrMultiNetworkTransfer],
+        timeseries: &LoadedTimeseriesCollection,
     ) -> Result<(), SchemaError> {
         let cost = match &self.cost {
             Some(v) => v
-                .load(network, schema, domain, tables, data_path, inter_network_transfers)?
+                .load(
+                    network,
+                    schema,
+                    domain,
+                    tables,
+                    data_path,
+                    inter_network_transfers,
+                    timeseries,
+                )?
                 .into(),
             None => ConstraintValue::Scalar(0.0),
         };
 
         let min_volume = match &self.min_volume {
             Some(v) => v
-                .load(network, schema, domain, tables, data_path, inter_network_transfers)?
+                .load(
+                    network,
+                    schema,
+                    domain,
+                    tables,
+                    data_path,
+                    inter_network_transfers,
+                    timeseries,
+                )?
                 .into(),
             None => ConstraintValue::Scalar(0.0),
         };
 
         let max_volume = match &self.max_volume {
             Some(v) => v
-                .load(network, schema, domain, tables, data_path, inter_network_transfers)?
+                .load(
+                    network,
+                    schema,
+                    domain,
+                    tables,
+                    data_path,
+                    inter_network_transfers,
+                    timeseries,
+                )?
                 .into(),
             None => ConstraintValue::None,
         };