From 32b5e5eb8c65d707b6445ead466b1d2e0155e527 Mon Sep 17 00:00:00 2001 From: James Batchelor Date: Wed, 7 Feb 2024 10:44:58 +0000 Subject: [PATCH 01/12] WIP: feat: add timeseries section to model schema --- pywr-schema/Cargo.toml | 2 +- pywr-schema/src/error.rs | 11 + pywr-schema/src/lib.rs | 1 + pywr-schema/src/model.rs | 10 + .../src/nodes/annual_virtual_storage.rs | 32 +- pywr-schema/src/nodes/core.rs | 192 ++++++++- pywr-schema/src/nodes/loss_link.rs | 32 +- pywr-schema/src/nodes/mod.rs | 207 ++++++++-- .../src/nodes/monthly_virtual_storage.rs | 32 +- pywr-schema/src/nodes/piecewise_link.rs | 32 +- pywr-schema/src/nodes/piecewise_storage.rs | 53 ++- pywr-schema/src/nodes/river_gauge.rs | 22 +- .../src/nodes/river_split_with_gauge.rs | 23 +- .../src/nodes/rolling_virtual_storage.rs | 32 +- pywr-schema/src/nodes/virtual_storage.rs | 32 +- .../src/nodes/water_treatment_works.rs | 62 ++- pywr-schema/src/parameters/aggregated.rs | 27 +- .../src/parameters/asymmetric_switch.rs | 26 +- pywr-schema/src/parameters/control_curves.rs | 77 +++- pywr-schema/src/parameters/core.rs | 65 +++- pywr-schema/src/parameters/delay.rs | 14 +- pywr-schema/src/parameters/discount_factor.rs | 14 +- pywr-schema/src/parameters/indexed_array.rs | 26 +- pywr-schema/src/parameters/interpolated.rs | 38 +- pywr-schema/src/parameters/mod.rs | 71 +++- pywr-schema/src/parameters/offset.rs | 14 +- pywr-schema/src/parameters/python.rs | 26 +- pywr-schema/src/parameters/thresholds.rs | 26 +- pywr-schema/src/test_models/inflow.csv | 366 ++++++++++++++++++ pywr-schema/src/test_models/timeseries.json | 82 ++++ pywr-schema/src/timeseries/mod.rs | 153 ++++++++ pywr-schema/src/timeseries/pandas.rs | 0 pywr-schema/src/timeseries/polars_dataset.rs | 56 +++ 33 files changed, 1688 insertions(+), 168 deletions(-) create mode 100644 pywr-schema/src/test_models/inflow.csv create mode 100644 pywr-schema/src/test_models/timeseries.json create mode 100644 pywr-schema/src/timeseries/mod.rs create mode 100644 pywr-schema/src/timeseries/pandas.rs create mode 100644 pywr-schema/src/timeseries/polars_dataset.rs diff --git a/pywr-schema/Cargo.toml b/pywr-schema/Cargo.toml index 90adafdf..e9f44778 100644 --- a/pywr-schema/Cargo.toml +++ b/pywr-schema/Cargo.toml @@ -15,7 +15,7 @@ categories = ["science", "simulation"] [dependencies] svgbobdoc = { version = "0.3.0", features = ["enable"] } -polars = { workspace = true } +polars = { workspace = true, features = ["csv"] } pyo3 = { workspace = true } pyo3-polars = { workspace = true } strum = "0.26" diff --git a/pywr-schema/src/error.rs b/pywr-schema/src/error.rs index 8f94a89e..80f49008 100644 --- a/pywr-schema/src/error.rs +++ b/pywr-schema/src/error.rs @@ -1,5 +1,6 @@ use crate::data_tables::TableError; use crate::nodes::NodeAttribute; +use polars::error::PolarsError; use pyo3::exceptions::PyRuntimeError; use pyo3::PyErr; use thiserror::Error; @@ -28,6 +29,16 @@ pub enum SchemaError { PywrCore(#[from] pywr_core::PywrError), #[error("data table error: {0}")] DataTable(#[from] TableError), + #[error("Timeseries '{0} not found")] + TimeseriesNotFound(String), + #[error("Column '{col}' not found in timeseries input '{name}'")] + ColumnNotFound { col: String, name: String }, + #[error("Timeseries provider '{provider}' does not support '{fmt}' file types")] + TimeseriesUnsupportedFileFormat { provider: String, fmt: String }, + #[error("Timeseries provider '{provider}' cannot parse file: '{path}'")] + TimeseriesUnparsableFileFormat { provider: String, path: String }, + #[error("Polars error: {0}")] + PolarsError(#[from] PolarsError), #[error("Circular node reference(s) found.")] CircularNodeReference, #[error("Circular parameters reference(s) found.")] diff --git a/pywr-schema/src/lib.rs b/pywr-schema/src/lib.rs index dbe4c3ec..0413d44e 100644 --- a/pywr-schema/src/lib.rs +++ b/pywr-schema/src/lib.rs @@ -12,6 +12,7 @@ pub mod model; pub mod nodes; pub mod outputs; pub mod parameters; +pub mod timeseries; pub use error::{ConversionError, SchemaError}; pub use model::PywrModel; diff --git a/pywr-schema/src/model.rs b/pywr-schema/src/model.rs index d780da16..12e122f2 100644 --- a/pywr-schema/src/model.rs +++ b/pywr-schema/src/model.rs @@ -6,6 +6,7 @@ use crate::error::{ConversionError, SchemaError}; use crate::metric_sets::MetricSet; use crate::outputs::Output; use crate::parameters::{MetricFloatReference, TryIntoV2Parameter}; +use crate::timeseries::{LoadedTimeseriesCollection, Timeseries}; use pywr_core::models::ModelDomain; use pywr_core::PywrError; use std::path::{Path, PathBuf}; @@ -91,6 +92,7 @@ pub struct PywrNetwork { pub edges: Vec, pub parameters: Option>, pub tables: Option>, + pub timeseries: Option>, pub metric_sets: Option>, pub outputs: Option>, } @@ -139,6 +141,9 @@ impl PywrNetwork { // Load all the data tables let tables = LoadedTableCollection::from_schema(self.tables.as_deref(), data_path)?; + // Load all timeseries data + let timeseries = LoadedTimeseriesCollection::from_schema(self.timeseries.as_deref(), domain, data_path)?; + // Create all the nodes let mut remaining_nodes = self.nodes.clone(); @@ -153,6 +158,7 @@ impl PywrNetwork { &tables, data_path, inter_network_transfers, + ×eries, ) { // Adding the node failed! match e { @@ -211,6 +217,7 @@ impl PywrNetwork { &tables, data_path, inter_network_transfers, + ×eries, ) { // Adding the parameter failed! match e { @@ -244,6 +251,7 @@ impl PywrNetwork { &tables, data_path, inter_network_transfers, + ×eries, )?; } @@ -382,6 +390,7 @@ impl TryFrom for PywrModel { // TODO convert v1 tables! let tables = None; + let timeseries = None; let outputs = None; let metric_sets = None; let network = PywrNetwork { @@ -389,6 +398,7 @@ impl TryFrom for PywrModel { edges, parameters, tables, + timeseries, metric_sets, outputs, }; diff --git a/pywr-schema/src/nodes/annual_virtual_storage.rs b/pywr-schema/src/nodes/annual_virtual_storage.rs index d10f98f6..1e60b5f6 100644 --- a/pywr-schema/src/nodes/annual_virtual_storage.rs +++ b/pywr-schema/src/nodes/annual_virtual_storage.rs @@ -4,6 +4,7 @@ use crate::model::PywrMultiNetworkTransfer; use crate::nodes::core::StorageInitialVolume; use crate::nodes::{NodeAttribute, NodeMeta}; use crate::parameters::{DynamicFloatValue, TryIntoV2Parameter}; +use crate::timeseries::LoadedTimeseriesCollection; use pywr_core::derived_metric::DerivedMetric; use pywr_core::metric::Metric; use pywr_core::models::ModelDomain; @@ -53,24 +54,49 @@ impl AnnualVirtualStorageNode { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result<(), SchemaError> { let cost = match &self.cost { Some(v) => v - .load(network, schema, domain, tables, data_path, inter_network_transfers)? + .load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )? .into(), None => ConstraintValue::Scalar(0.0), }; let min_volume = match &self.min_volume { Some(v) => v - .load(network, schema, domain, tables, data_path, inter_network_transfers)? + .load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )? .into(), None => ConstraintValue::Scalar(0.0), }; let max_volume = match &self.max_volume { Some(v) => v - .load(network, schema, domain, tables, data_path, inter_network_transfers)? + .load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )? .into(), None => ConstraintValue::None, }; diff --git a/pywr-schema/src/nodes/core.rs b/pywr-schema/src/nodes/core.rs index 38488d27..33b72aca 100644 --- a/pywr-schema/src/nodes/core.rs +++ b/pywr-schema/src/nodes/core.rs @@ -3,6 +3,7 @@ use crate::error::{ConversionError, SchemaError}; use crate::model::PywrMultiNetworkTransfer; use crate::nodes::{NodeAttribute, NodeMeta}; use crate::parameters::{DynamicFloatValue, TryIntoV2Parameter}; +use crate::timeseries::LoadedTimeseriesCollection; use pywr_core::derived_metric::DerivedMetric; use pywr_core::metric::Metric; use pywr_core::models::ModelDomain; @@ -55,19 +56,44 @@ impl InputNode { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result<(), SchemaError> { if let Some(cost) = &self.cost { - let value = cost.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = cost.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_cost(self.meta.name.as_str(), None, value.into())?; } if let Some(max_flow) = &self.max_flow { - let value = max_flow.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = max_flow.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_max_flow(self.meta.name.as_str(), None, value.into())?; } if let Some(min_flow) = &self.min_flow { - let value = min_flow.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = min_flow.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_min_flow(self.meta.name.as_str(), None, value.into())?; } @@ -177,19 +203,44 @@ impl LinkNode { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result<(), SchemaError> { if let Some(cost) = &self.cost { - let value = cost.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = cost.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_cost(self.meta.name.as_str(), None, value.into())?; } if let Some(max_flow) = &self.max_flow { - let value = max_flow.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = max_flow.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_max_flow(self.meta.name.as_str(), None, value.into())?; } if let Some(min_flow) = &self.min_flow { - let value = min_flow.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = min_flow.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_min_flow(self.meta.name.as_str(), None, value.into())?; } @@ -299,19 +350,44 @@ impl OutputNode { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result<(), SchemaError> { if let Some(cost) = &self.cost { - let value = cost.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = cost.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_cost(self.meta.name.as_str(), None, value.into())?; } if let Some(max_flow) = &self.max_flow { - let value = max_flow.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = max_flow.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_max_flow(self.meta.name.as_str(), None, value.into())?; } if let Some(min_flow) = &self.min_flow { - let value = min_flow.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = min_flow.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_min_flow(self.meta.name.as_str(), None, value.into())?; } @@ -438,17 +514,34 @@ impl StorageNode { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result<(), SchemaError> { let min_volume = match &self.min_volume { Some(v) => v - .load(network, schema, domain, tables, data_path, inter_network_transfers)? + .load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )? .into(), None => ConstraintValue::Scalar(0.0), }; let max_volume = match &self.max_volume { Some(v) => v - .load(network, schema, domain, tables, data_path, inter_network_transfers)? + .load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )? .into(), None => ConstraintValue::None, }; @@ -471,9 +564,18 @@ impl StorageNode { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result<(), SchemaError> { if let Some(cost) = &self.cost { - let value = cost.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = cost.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_cost(self.meta.name.as_str(), None, value.into())?; } @@ -654,14 +756,31 @@ impl CatchmentNode { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result<(), SchemaError> { if let Some(cost) = &self.cost { - let value = cost.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = cost.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_cost(self.meta.name.as_str(), None, value.into())?; } if let Some(flow) = &self.flow { - let value = flow.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = flow.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_min_flow(self.meta.name.as_str(), None, value.clone().into())?; network.set_node_max_flow(self.meta.name.as_str(), None, value.into())?; } @@ -764,14 +883,31 @@ impl AggregatedNode { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result<(), SchemaError> { if let Some(max_flow) = &self.max_flow { - let value = max_flow.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = max_flow.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_aggregated_node_max_flow(self.meta.name.as_str(), None, value.into())?; } if let Some(min_flow) = &self.min_flow { - let value = min_flow.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = min_flow.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_aggregated_node_min_flow(self.meta.name.as_str(), None, value.into())?; } @@ -780,13 +916,33 @@ impl AggregatedNode { Factors::Proportion { factors } => pywr_core::aggregated_node::Factors::Proportion( factors .iter() - .map(|f| f.load(network, schema, domain, tables, data_path, inter_network_transfers)) + .map(|f| { + f.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ) + }) .collect::, _>>()?, ), Factors::Ratio { factors } => pywr_core::aggregated_node::Factors::Ratio( factors .iter() - .map(|f| f.load(network, schema, domain, tables, data_path, inter_network_transfers)) + .map(|f| { + f.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ) + }) .collect::, _>>()?, ), }; diff --git a/pywr-schema/src/nodes/loss_link.rs b/pywr-schema/src/nodes/loss_link.rs index d41ae412..1a3b8718 100644 --- a/pywr-schema/src/nodes/loss_link.rs +++ b/pywr-schema/src/nodes/loss_link.rs @@ -3,6 +3,7 @@ use crate::error::{ConversionError, SchemaError}; use crate::model::PywrMultiNetworkTransfer; use crate::nodes::{NodeAttribute, NodeMeta}; use crate::parameters::{DynamicFloatValue, TryIntoV2Parameter}; +use crate::timeseries::LoadedTimeseriesCollection; use pywr_core::metric::Metric; use pywr_core::models::ModelDomain; use pywr_v1_schema::nodes::LossLinkNode as LossLinkNodeV1; @@ -64,19 +65,44 @@ impl LossLinkNode { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result<(), SchemaError> { if let Some(cost) = &self.net_cost { - let value = cost.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = cost.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_cost(self.meta.name.as_str(), Self::net_sub_name(), value.into())?; } if let Some(max_flow) = &self.max_net_flow { - let value = max_flow.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = max_flow.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_max_flow(self.meta.name.as_str(), Self::net_sub_name(), value.into())?; } if let Some(min_flow) = &self.min_net_flow { - let value = min_flow.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = min_flow.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_min_flow(self.meta.name.as_str(), Self::net_sub_name(), value.into())?; } diff --git a/pywr-schema/src/nodes/mod.rs b/pywr-schema/src/nodes/mod.rs index 116f765e..1ba68f87 100644 --- a/pywr-schema/src/nodes/mod.rs +++ b/pywr-schema/src/nodes/mod.rs @@ -22,6 +22,7 @@ pub use crate::nodes::delay::DelayNode; pub use crate::nodes::river::RiverNode; use crate::nodes::rolling_virtual_storage::RollingVirtualStorageNode; use crate::parameters::DynamicFloatValue; +use crate::timeseries::LoadedTimeseriesCollection; pub use annual_virtual_storage::AnnualVirtualStorageNode; pub use loss_link::LossLinkNode; pub use monthly_virtual_storage::MonthlyVirtualStorageNode; @@ -302,12 +303,21 @@ impl Node { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result<(), SchemaError> { match self { Node::Input(n) => n.add_to_model(network), Node::Link(n) => n.add_to_model(network), Node::Output(n) => n.add_to_model(network), - Node::Storage(n) => n.add_to_model(network, schema, domain, tables, data_path, inter_network_transfers), + Node::Storage(n) => n.add_to_model( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ), Node::Catchment(n) => n.add_to_model(network), Node::RiverGauge(n) => n.add_to_model(network), Node::LossLink(n) => n.add_to_model(network), @@ -316,23 +326,53 @@ impl Node { Node::WaterTreatmentWorks(n) => n.add_to_model(network), Node::Aggregated(n) => n.add_to_model(network), Node::AggregatedStorage(n) => n.add_to_model(network), - Node::VirtualStorage(n) => { - n.add_to_model(network, schema, domain, tables, data_path, inter_network_transfers) - } - Node::AnnualVirtualStorage(n) => { - n.add_to_model(network, schema, domain, tables, data_path, inter_network_transfers) - } + Node::VirtualStorage(n) => n.add_to_model( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ), + Node::AnnualVirtualStorage(n) => n.add_to_model( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ), Node::PiecewiseLink(n) => n.add_to_model(network), - Node::PiecewiseStorage(n) => { - n.add_to_model(network, schema, domain, tables, data_path, inter_network_transfers) - } + Node::PiecewiseStorage(n) => n.add_to_model( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ), Node::Delay(n) => n.add_to_model(network), - Node::MonthlyVirtualStorage(n) => { - n.add_to_model(network, schema, domain, tables, data_path, inter_network_transfers) - } - Node::RollingVirtualStorage(n) => { - n.add_to_model(network, schema, domain, tables, data_path, inter_network_transfers) - } + Node::MonthlyVirtualStorage(n) => n.add_to_model( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ), + Node::RollingVirtualStorage(n) => n.add_to_model( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ), } } @@ -344,38 +384,121 @@ impl Node { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result<(), SchemaError> { match self { - Node::Input(n) => n.set_constraints(network, schema, domain, tables, data_path, inter_network_transfers), - Node::Link(n) => n.set_constraints(network, schema, domain, tables, data_path, inter_network_transfers), - Node::Output(n) => n.set_constraints(network, schema, domain, tables, data_path, inter_network_transfers), - Node::Storage(n) => n.set_constraints(network, schema, domain, tables, data_path, inter_network_transfers), - Node::Catchment(n) => { - n.set_constraints(network, schema, domain, tables, data_path, inter_network_transfers) - } - Node::RiverGauge(n) => { - n.set_constraints(network, schema, domain, tables, data_path, inter_network_transfers) - } - Node::LossLink(n) => n.set_constraints(network, schema, domain, tables, data_path, inter_network_transfers), + Node::Input(n) => n.set_constraints( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ), + Node::Link(n) => n.set_constraints( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ), + Node::Output(n) => n.set_constraints( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ), + Node::Storage(n) => n.set_constraints( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ), + Node::Catchment(n) => n.set_constraints( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ), + Node::RiverGauge(n) => n.set_constraints( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ), + Node::LossLink(n) => n.set_constraints( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ), Node::River(_) => Ok(()), // No constraints on river node - Node::RiverSplitWithGauge(n) => { - n.set_constraints(network, schema, domain, tables, data_path, inter_network_transfers) - } - Node::WaterTreatmentWorks(n) => { - n.set_constraints(network, schema, domain, tables, data_path, inter_network_transfers) - } - Node::Aggregated(n) => { - n.set_constraints(network, schema, domain, tables, data_path, inter_network_transfers) - } + Node::RiverSplitWithGauge(n) => n.set_constraints( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ), + Node::WaterTreatmentWorks(n) => n.set_constraints( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ), + Node::Aggregated(n) => n.set_constraints( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ), Node::AggregatedStorage(_) => Ok(()), // No constraints on aggregated storage nodes. Node::VirtualStorage(_) => Ok(()), // TODO Node::AnnualVirtualStorage(_) => Ok(()), // TODO - Node::PiecewiseLink(n) => { - n.set_constraints(network, schema, domain, tables, data_path, inter_network_transfers) - } - Node::PiecewiseStorage(n) => { - n.set_constraints(network, schema, domain, tables, data_path, inter_network_transfers) - } + Node::PiecewiseLink(n) => n.set_constraints( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ), + Node::PiecewiseStorage(n) => n.set_constraints( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ), Node::Delay(n) => n.set_constraints(network, tables), Node::MonthlyVirtualStorage(_) => Ok(()), // TODO Node::RollingVirtualStorage(_) => Ok(()), // TODO diff --git a/pywr-schema/src/nodes/monthly_virtual_storage.rs b/pywr-schema/src/nodes/monthly_virtual_storage.rs index 8673c003..08aa889e 100644 --- a/pywr-schema/src/nodes/monthly_virtual_storage.rs +++ b/pywr-schema/src/nodes/monthly_virtual_storage.rs @@ -4,6 +4,7 @@ use crate::model::PywrMultiNetworkTransfer; use crate::nodes::core::StorageInitialVolume; use crate::nodes::{NodeAttribute, NodeMeta}; use crate::parameters::{DynamicFloatValue, TryIntoV2Parameter}; +use crate::timeseries::LoadedTimeseriesCollection; use pywr_core::derived_metric::DerivedMetric; use pywr_core::metric::Metric; use pywr_core::models::ModelDomain; @@ -47,24 +48,49 @@ impl MonthlyVirtualStorageNode { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result<(), SchemaError> { let cost = match &self.cost { Some(v) => v - .load(network, schema, domain, tables, data_path, inter_network_transfers)? + .load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )? .into(), None => ConstraintValue::Scalar(0.0), }; let min_volume = match &self.min_volume { Some(v) => v - .load(network, schema, domain, tables, data_path, inter_network_transfers)? + .load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )? .into(), None => ConstraintValue::Scalar(0.0), }; let max_volume = match &self.max_volume { Some(v) => v - .load(network, schema, domain, tables, data_path, inter_network_transfers)? + .load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )? .into(), None => ConstraintValue::None, }; diff --git a/pywr-schema/src/nodes/piecewise_link.rs b/pywr-schema/src/nodes/piecewise_link.rs index e014dae4..17149714 100644 --- a/pywr-schema/src/nodes/piecewise_link.rs +++ b/pywr-schema/src/nodes/piecewise_link.rs @@ -3,6 +3,7 @@ use crate::error::{ConversionError, SchemaError}; use crate::model::PywrMultiNetworkTransfer; use crate::nodes::{NodeAttribute, NodeMeta}; use crate::parameters::{DynamicFloatValue, TryIntoV2Parameter}; +use crate::timeseries::LoadedTimeseriesCollection; use pywr_core::metric::Metric; use pywr_core::models::ModelDomain; use pywr_v1_schema::nodes::PiecewiseLinkNode as PiecewiseLinkNodeV1; @@ -67,22 +68,47 @@ impl PiecewiseLinkNode { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result<(), SchemaError> { for (i, step) in self.steps.iter().enumerate() { let sub_name = Self::step_sub_name(i); if let Some(cost) = &step.cost { - let value = cost.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = cost.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_cost(self.meta.name.as_str(), sub_name.as_deref(), value.into())?; } if let Some(max_flow) = &step.max_flow { - let value = max_flow.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = max_flow.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_max_flow(self.meta.name.as_str(), sub_name.as_deref(), value.into())?; } if let Some(min_flow) = &step.min_flow { - let value = min_flow.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = min_flow.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_min_flow(self.meta.name.as_str(), sub_name.as_deref(), value.into())?; } } diff --git a/pywr-schema/src/nodes/piecewise_storage.rs b/pywr-schema/src/nodes/piecewise_storage.rs index f57b3d8c..c1662382 100644 --- a/pywr-schema/src/nodes/piecewise_storage.rs +++ b/pywr-schema/src/nodes/piecewise_storage.rs @@ -3,6 +3,7 @@ use crate::error::SchemaError; use crate::model::PywrMultiNetworkTransfer; use crate::nodes::{NodeAttribute, NodeMeta}; use crate::parameters::DynamicFloatValue; +use crate::timeseries::LoadedTimeseriesCollection; use pywr_core::derived_metric::DerivedMetric; use pywr_core::metric::Metric; use pywr_core::models::ModelDomain; @@ -74,11 +75,18 @@ impl PiecewiseStorageNode { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result<(), SchemaError> { // These are the min and max volume of the overall node - let max_volume = self - .max_volume - .load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let max_volume = self.max_volume.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; let mut store_node_indices = Vec::new(); @@ -94,14 +102,21 @@ impl PiecewiseStorageNode { tables, data_path, inter_network_transfers, + timeseries, )?) } else { None }; - let upper = step - .control_curve - .load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let upper = step.control_curve.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; let max_volume_parameter = VolumeBetweenControlCurvesParameter::new( format!("{}-{}-max-volume", self.meta.name, Self::step_sub_name(i).unwrap()).as_str(), @@ -136,12 +151,15 @@ impl PiecewiseStorageNode { // The volume of this store the remain proportion above the last control curve let lower = match self.steps.last() { - Some(step) => { - Some( - step.control_curve - .load(network, schema, domain, tables, data_path, inter_network_transfers)?, - ) - } + Some(step) => Some(step.control_curve.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?), None => None, }; @@ -197,12 +215,21 @@ impl PiecewiseStorageNode { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result<(), SchemaError> { for (i, step) in self.steps.iter().enumerate() { let sub_name = Self::step_sub_name(i); if let Some(cost) = &step.cost { - let value = cost.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = cost.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_cost(self.meta.name.as_str(), sub_name.as_deref(), value.into())?; } } diff --git a/pywr-schema/src/nodes/river_gauge.rs b/pywr-schema/src/nodes/river_gauge.rs index 58921b52..f4aee624 100644 --- a/pywr-schema/src/nodes/river_gauge.rs +++ b/pywr-schema/src/nodes/river_gauge.rs @@ -3,6 +3,7 @@ use crate::error::{ConversionError, SchemaError}; use crate::model::PywrMultiNetworkTransfer; use crate::nodes::{NodeAttribute, NodeMeta}; use crate::parameters::{DynamicFloatValue, TryIntoV2Parameter}; +use crate::timeseries::LoadedTimeseriesCollection; use pywr_core::metric::Metric; use pywr_core::models::ModelDomain; use pywr_v1_schema::nodes::RiverGaugeNode as RiverGaugeNodeV1; @@ -57,15 +58,32 @@ impl RiverGaugeNode { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result<(), SchemaError> { // MRF applies as a maximum on the MRF node. if let Some(cost) = &self.mrf_cost { - let value = cost.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = cost.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_cost(self.meta.name.as_str(), Self::mrf_sub_name(), value.into())?; } if let Some(mrf) = &self.mrf { - let value = mrf.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = mrf.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_max_flow(self.meta.name.as_str(), Self::mrf_sub_name(), value.into())?; } diff --git a/pywr-schema/src/nodes/river_split_with_gauge.rs b/pywr-schema/src/nodes/river_split_with_gauge.rs index 95240da2..9ac72260 100644 --- a/pywr-schema/src/nodes/river_split_with_gauge.rs +++ b/pywr-schema/src/nodes/river_split_with_gauge.rs @@ -3,6 +3,7 @@ use crate::error::{ConversionError, SchemaError}; use crate::model::PywrMultiNetworkTransfer; use crate::nodes::{NodeAttribute, NodeMeta}; use crate::parameters::{DynamicFloatValue, TryIntoV2Parameter}; +use crate::timeseries::LoadedTimeseriesCollection; use pywr_core::aggregated_node::Factors; use pywr_core::metric::Metric; use pywr_core::models::ModelDomain; @@ -88,15 +89,32 @@ impl RiverSplitWithGaugeNode { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result<(), SchemaError> { // MRF applies as a maximum on the MRF node. if let Some(cost) = &self.mrf_cost { - let value = cost.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = cost.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_cost(self.meta.name.as_str(), Self::mrf_sub_name(), value.into())?; } if let Some(mrf) = &self.mrf { - let value = mrf.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = mrf.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_max_flow(self.meta.name.as_str(), Self::mrf_sub_name(), value.into())?; } @@ -109,6 +127,7 @@ impl RiverSplitWithGaugeNode { tables, data_path, inter_network_transfers, + timeseries, )?]); network.set_aggregated_node_factors( self.meta.name.as_str(), diff --git a/pywr-schema/src/nodes/rolling_virtual_storage.rs b/pywr-schema/src/nodes/rolling_virtual_storage.rs index 4e3c3ba5..6d4a4112 100644 --- a/pywr-schema/src/nodes/rolling_virtual_storage.rs +++ b/pywr-schema/src/nodes/rolling_virtual_storage.rs @@ -3,6 +3,7 @@ use crate::error::{ConversionError, SchemaError}; use crate::model::PywrMultiNetworkTransfer; use crate::nodes::{NodeAttribute, NodeMeta}; use crate::parameters::{DynamicFloatValue, TryIntoV2Parameter}; +use crate::timeseries::LoadedTimeseriesCollection; use pywr_core::derived_metric::DerivedMetric; use pywr_core::metric::Metric; use pywr_core::models::ModelDomain; @@ -80,6 +81,7 @@ impl RollingVirtualStorageNode { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result<(), SchemaError> { let initial_volume = if let Some(iv) = self.initial_volume { StorageInitialVolume::Absolute(iv) @@ -91,21 +93,45 @@ impl RollingVirtualStorageNode { let cost = match &self.cost { Some(v) => v - .load(network, schema, domain, tables, data_path, inter_network_transfers)? + .load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )? .into(), None => ConstraintValue::Scalar(0.0), }; let min_volume = match &self.min_volume { Some(v) => v - .load(network, schema, domain, tables, data_path, inter_network_transfers)? + .load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )? .into(), None => ConstraintValue::Scalar(0.0), }; let max_volume = match &self.max_volume { Some(v) => v - .load(network, schema, domain, tables, data_path, inter_network_transfers)? + .load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )? .into(), None => ConstraintValue::None, }; diff --git a/pywr-schema/src/nodes/virtual_storage.rs b/pywr-schema/src/nodes/virtual_storage.rs index 2c247117..e354fe62 100644 --- a/pywr-schema/src/nodes/virtual_storage.rs +++ b/pywr-schema/src/nodes/virtual_storage.rs @@ -4,6 +4,7 @@ use crate::model::PywrMultiNetworkTransfer; use crate::nodes::core::StorageInitialVolume; use crate::nodes::{NodeAttribute, NodeMeta}; use crate::parameters::{DynamicFloatValue, TryIntoV2Parameter}; +use crate::timeseries::LoadedTimeseriesCollection; use pywr_core::derived_metric::DerivedMetric; use pywr_core::metric::Metric; use pywr_core::models::ModelDomain; @@ -35,24 +36,49 @@ impl VirtualStorageNode { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result<(), SchemaError> { let cost = match &self.cost { Some(v) => v - .load(network, schema, domain, tables, data_path, inter_network_transfers)? + .load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )? .into(), None => ConstraintValue::Scalar(0.0), }; let min_volume = match &self.min_volume { Some(v) => v - .load(network, schema, domain, tables, data_path, inter_network_transfers)? + .load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )? .into(), None => ConstraintValue::Scalar(0.0), }; let max_volume = match &self.max_volume { Some(v) => v - .load(network, schema, domain, tables, data_path, inter_network_transfers)? + .load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )? .into(), None => ConstraintValue::None, }; diff --git a/pywr-schema/src/nodes/water_treatment_works.rs b/pywr-schema/src/nodes/water_treatment_works.rs index a4fce5ca..83d647f8 100644 --- a/pywr-schema/src/nodes/water_treatment_works.rs +++ b/pywr-schema/src/nodes/water_treatment_works.rs @@ -3,6 +3,7 @@ use crate::error::SchemaError; use crate::model::PywrMultiNetworkTransfer; use crate::nodes::{NodeAttribute, NodeMeta}; use crate::parameters::DynamicFloatValue; +use crate::timeseries::LoadedTimeseriesCollection; use num::Zero; use pywr_core::aggregated_node::Factors; use pywr_core::metric::Metric; @@ -110,26 +111,59 @@ impl WaterTreatmentWorks { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result<(), SchemaError> { if let Some(cost) = &self.cost { - let value = cost.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = cost.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_cost(self.meta.name.as_str(), Self::net_sub_name(), value.into())?; } if let Some(max_flow) = &self.max_flow { - let value = max_flow.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = max_flow.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_max_flow(self.meta.name.as_str(), Self::net_sub_name(), value.into())?; } if let Some(min_flow) = &self.min_flow { - let value = min_flow.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = min_flow.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_min_flow(self.meta.name.as_str(), Self::net_sub_name(), value.into())?; } // soft min flow constraints; This typically applies a negative cost upto a maximum // defined by the `soft_min_flow` if let Some(cost) = &self.soft_min_flow_cost { - let value = cost.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = cost.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_cost( self.meta.name.as_str(), Self::net_soft_min_flow_sub_name(), @@ -137,7 +171,15 @@ impl WaterTreatmentWorks { )?; } if let Some(min_flow) = &self.soft_min_flow { - let value = min_flow.load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let value = min_flow.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; network.set_node_max_flow( self.meta.name.as_str(), Self::net_soft_min_flow_sub_name(), @@ -148,7 +190,15 @@ impl WaterTreatmentWorks { if let Some(loss_factor) = &self.loss_factor { // Handle the case where we a given a zero loss factor // The aggregated node does not support zero loss factors so filter them here. - let lf = match loss_factor.load(network, schema, domain, tables, data_path, inter_network_transfers)? { + let lf = match loss_factor.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )? { Metric::Constant(f) => { if f.is_zero() { None diff --git a/pywr-schema/src/parameters/aggregated.rs b/pywr-schema/src/parameters/aggregated.rs index 945a1543..a90819f9 100644 --- a/pywr-schema/src/parameters/aggregated.rs +++ b/pywr-schema/src/parameters/aggregated.rs @@ -5,6 +5,7 @@ use crate::parameters::{ DynamicFloatValue, DynamicFloatValueType, DynamicIndexValue, IntoV2Parameter, ParameterMeta, TryFromV1Parameter, TryIntoV2Parameter, }; +use crate::timeseries::LoadedTimeseriesCollection; use pywr_core::models::ModelDomain; use pywr_core::parameters::{IndexParameterIndex, ParameterIndex}; use pywr_v1_schema::parameters::{ @@ -99,11 +100,22 @@ impl AggregatedParameter { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result { let metrics = self .metrics .iter() - .map(|v| v.load(network, schema, domain, tables, data_path, inter_network_transfers)) + .map(|v| { + v.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ) + }) .collect::, _>>()?; let p = pywr_core::parameters::AggregatedParameter::new(&self.meta.name, &metrics, self.agg_func.into()); @@ -206,11 +218,22 @@ impl AggregatedIndexParameter { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result { let parameters = self .parameters .iter() - .map(|v| v.load(network, schema, domain, tables, data_path, inter_network_transfers)) + .map(|v| { + v.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ) + }) .collect::, _>>()?; let p = pywr_core::parameters::AggregatedIndexParameter::new(&self.meta.name, parameters, self.agg_func.into()); diff --git a/pywr-schema/src/parameters/asymmetric_switch.rs b/pywr-schema/src/parameters/asymmetric_switch.rs index 554b4506..60cdd29f 100644 --- a/pywr-schema/src/parameters/asymmetric_switch.rs +++ b/pywr-schema/src/parameters/asymmetric_switch.rs @@ -4,6 +4,7 @@ use crate::model::PywrMultiNetworkTransfer; use crate::parameters::{ DynamicFloatValueType, DynamicIndexValue, IntoV2Parameter, ParameterMeta, TryFromV1Parameter, TryIntoV2Parameter, }; +use crate::timeseries::LoadedTimeseriesCollection; use pywr_core::models::ModelDomain; use pywr_core::parameters::IndexParameterIndex; use pywr_v1_schema::parameters::AsymmetricSwitchIndexParameter as AsymmetricSwitchIndexParameterV1; @@ -34,13 +35,26 @@ impl AsymmetricSwitchIndexParameter { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result { - let on_index_parameter = - self.on_index_parameter - .load(network, schema, domain, tables, data_path, inter_network_transfers)?; - let off_index_parameter = - self.off_index_parameter - .load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let on_index_parameter = self.on_index_parameter.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; + let off_index_parameter = self.off_index_parameter.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; let p = pywr_core::parameters::AsymmetricSwitchIndexParameter::new( &self.meta.name, diff --git a/pywr-schema/src/parameters/control_curves.rs b/pywr-schema/src/parameters/control_curves.rs index 9e3477b2..3b80b8fe 100644 --- a/pywr-schema/src/parameters/control_curves.rs +++ b/pywr-schema/src/parameters/control_curves.rs @@ -5,6 +5,7 @@ use crate::nodes::NodeAttribute; use crate::parameters::{ DynamicFloatValue, IntoV2Parameter, NodeReference, ParameterMeta, TryFromV1Parameter, TryIntoV2Parameter, }; +use crate::timeseries::{self, LoadedTimeseriesCollection}; use pywr_core::models::ModelDomain; use pywr_core::parameters::{IndexParameterIndex, ParameterIndex}; use pywr_v1_schema::parameters::{ @@ -33,19 +34,40 @@ impl ControlCurveInterpolatedParameter { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result { let metric = self.storage_node.load(network, schema)?; let control_curves = self .control_curves .iter() - .map(|cc| cc.load(network, schema, domain, tables, data_path, inter_network_transfers)) + .map(|cc| { + cc.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ) + }) .collect::>()?; let values = self .values .iter() - .map(|val| val.load(network, schema, domain, tables, data_path, inter_network_transfers)) + .map(|val| { + val.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ) + }) .collect::>()?; let p = pywr_core::parameters::ControlCurveInterpolatedParameter::new( @@ -136,13 +158,24 @@ impl ControlCurveIndexParameter { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result { let metric = self.storage_node.load(network, schema)?; let control_curves = self .control_curves .iter() - .map(|cc| cc.load(network, schema, domain, tables, data_path, inter_network_transfers)) + .map(|cc| { + cc.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ) + }) .collect::>()?; let p = pywr_core::parameters::ControlCurveIndexParameter::new(&self.meta.name, metric, control_curves); @@ -247,19 +280,40 @@ impl ControlCurveParameter { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result { let metric = self.storage_node.load(network, schema)?; let control_curves = self .control_curves .iter() - .map(|cc| cc.load(network, schema, domain, tables, data_path, inter_network_transfers)) + .map(|cc| { + cc.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ) + }) .collect::>()?; let values = self .values .iter() - .map(|val| val.load(network, schema, domain, tables, data_path, inter_network_transfers)) + .map(|val| { + val.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ) + }) .collect::>()?; let p = pywr_core::parameters::ControlCurveParameter::new(&self.meta.name, metric, control_curves, values); @@ -341,13 +395,24 @@ impl ControlCurvePiecewiseInterpolatedParameter { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result { let metric = self.storage_node.load(network, schema)?; let control_curves = self .control_curves .iter() - .map(|cc| cc.load(network, schema, domain, tables, data_path, inter_network_transfers)) + .map(|cc| { + cc.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ) + }) .collect::>()?; let values = match &self.values { diff --git a/pywr-schema/src/parameters/core.rs b/pywr-schema/src/parameters/core.rs index a6606497..aa034685 100644 --- a/pywr-schema/src/parameters/core.rs +++ b/pywr-schema/src/parameters/core.rs @@ -5,6 +5,7 @@ use crate::parameters::{ ConstantValue, DynamicFloatValue, DynamicFloatValueType, IntoV2Parameter, ParameterMeta, TryFromV1Parameter, TryIntoV2Parameter, }; +use crate::timeseries::LoadedTimeseriesCollection; use pywr_core::models::ModelDomain; use pywr_core::parameters::ParameterIndex; use pywr_v1_schema::parameters::{ @@ -237,10 +238,17 @@ impl MaxParameter { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result { - let idx = self - .parameter - .load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let idx = self.parameter.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; let threshold = self.threshold.unwrap_or(0.0); let p = pywr_core::parameters::MaxParameter::new(&self.meta.name, idx, threshold); @@ -312,13 +320,26 @@ impl DivisionParameter { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result { - let n = self - .numerator - .load(network, schema, domain, tables, data_path, inter_network_transfers)?; - let d = self - .denominator - .load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let n = self.numerator.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; + let d = self.denominator.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; let p = pywr_core::parameters::DivisionParameter::new(&self.meta.name, n, d); Ok(network.add_parameter(Box::new(p))?) @@ -387,10 +408,17 @@ impl MinParameter { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result { - let idx = self - .parameter - .load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let idx = self.parameter.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; let threshold = self.threshold.unwrap_or(0.0); let p = pywr_core::parameters::MinParameter::new(&self.meta.name, idx, threshold); @@ -444,10 +472,17 @@ impl NegativeParameter { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result { - let idx = self - .parameter - .load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let idx = self.parameter.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; let p = pywr_core::parameters::NegativeParameter::new(&self.meta.name, idx); Ok(network.add_parameter(Box::new(p))?) diff --git a/pywr-schema/src/parameters/delay.rs b/pywr-schema/src/parameters/delay.rs index 79021b85..c0957e99 100644 --- a/pywr-schema/src/parameters/delay.rs +++ b/pywr-schema/src/parameters/delay.rs @@ -2,6 +2,7 @@ use crate::data_tables::LoadedTableCollection; use crate::error::SchemaError; use crate::model::PywrMultiNetworkTransfer; use crate::parameters::{DynamicFloatValue, DynamicFloatValueType, ParameterMeta}; +use crate::timeseries::LoadedTimeseriesCollection; use pywr_core::models::ModelDomain; use pywr_core::parameters::ParameterIndex; use std::collections::HashMap; @@ -39,10 +40,17 @@ impl DelayParameter { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result { - let metric = self - .metric - .load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let metric = self.metric.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; let p = pywr_core::parameters::DelayParameter::new(&self.meta.name, metric, self.delay, self.initial_value); Ok(network.add_parameter(Box::new(p))?) } diff --git a/pywr-schema/src/parameters/discount_factor.rs b/pywr-schema/src/parameters/discount_factor.rs index bef78112..75d8d303 100644 --- a/pywr-schema/src/parameters/discount_factor.rs +++ b/pywr-schema/src/parameters/discount_factor.rs @@ -2,6 +2,7 @@ use crate::data_tables::LoadedTableCollection; use crate::error::SchemaError; use crate::model::PywrMultiNetworkTransfer; use crate::parameters::{DynamicFloatValue, DynamicFloatValueType, IntoV2Parameter, ParameterMeta, TryFromV1Parameter}; +use crate::timeseries::LoadedTimeseriesCollection; use crate::ConversionError; use pywr_core::models::ModelDomain; use pywr_core::parameters::ParameterIndex; @@ -40,10 +41,17 @@ impl DiscountFactorParameter { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result { - let discount_rate = - self.discount_rate - .load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let discount_rate = self.discount_rate.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; let p = pywr_core::parameters::DiscountFactorParameter::new(&self.meta.name, discount_rate, self.base_year); Ok(network.add_parameter(Box::new(p))?) } diff --git a/pywr-schema/src/parameters/indexed_array.rs b/pywr-schema/src/parameters/indexed_array.rs index 3534db62..dc36a449 100644 --- a/pywr-schema/src/parameters/indexed_array.rs +++ b/pywr-schema/src/parameters/indexed_array.rs @@ -5,6 +5,7 @@ use crate::parameters::{ DynamicFloatValue, DynamicFloatValueType, DynamicIndexValue, IntoV2Parameter, ParameterMeta, TryFromV1Parameter, TryIntoV2Parameter, }; +use crate::timeseries::LoadedTimeseriesCollection; use pywr_core::models::ModelDomain; use pywr_core::parameters::ParameterIndex; use pywr_v1_schema::parameters::IndexedArrayParameter as IndexedArrayParameterV1; @@ -42,15 +43,32 @@ impl IndexedArrayParameter { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result { - let index_parameter = - self.index_parameter - .load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let index_parameter = self.index_parameter.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; let metrics = self .metrics .iter() - .map(|v| v.load(network, schema, domain, tables, data_path, inter_network_transfers)) + .map(|v| { + v.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ) + }) .collect::, _>>()?; let p = pywr_core::parameters::IndexedArrayParameter::new(&self.meta.name, index_parameter, &metrics); diff --git a/pywr-schema/src/parameters/interpolated.rs b/pywr-schema/src/parameters/interpolated.rs index 0622c80c..c9c4b5cb 100644 --- a/pywr-schema/src/parameters/interpolated.rs +++ b/pywr-schema/src/parameters/interpolated.rs @@ -5,6 +5,7 @@ use crate::parameters::{ DynamicFloatValue, DynamicFloatValueType, IntoV2Parameter, MetricFloatReference, MetricFloatValue, NodeReference, ParameterMeta, TryFromV1Parameter, TryIntoV2Parameter, }; +use crate::timeseries::LoadedTimeseriesCollection; use crate::ConversionError; use pywr_core::models::ModelDomain; use pywr_core::parameters::ParameterIndex; @@ -58,10 +59,17 @@ impl InterpolatedParameter { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result { - let x = self - .x - .load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let x = self.x.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; // Sense check the points if self.xp.len() != self.fp.len() { @@ -74,12 +82,32 @@ impl InterpolatedParameter { let xp = self .xp .iter() - .map(|p| p.load(network, schema, domain, tables, data_path, inter_network_transfers)) + .map(|p| { + p.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ) + }) .collect::, _>>()?; let fp = self .fp .iter() - .map(|p| p.load(network, schema, domain, tables, data_path, inter_network_transfers)) + .map(|p| { + p.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + ) + }) .collect::, _>>()?; let points = xp diff --git a/pywr-schema/src/parameters/mod.rs b/pywr-schema/src/parameters/mod.rs index dc3266c4..b2daac0a 100644 --- a/pywr-schema/src/parameters/mod.rs +++ b/pywr-schema/src/parameters/mod.rs @@ -48,8 +48,9 @@ use crate::error::{ConversionError, SchemaError}; use crate::model::PywrMultiNetworkTransfer; use crate::nodes::NodeAttribute; use crate::parameters::core::DivisionParameter; -pub use crate::parameters::data_frame::DataFrameParameter; +pub use crate::parameters::data_frame::{DataFrameColumns, DataFrameParameter}; use crate::parameters::interpolated::InterpolatedParameter; +use crate::timeseries::{self, LoadedTimeseriesCollection}; pub use offset::OffsetParameter; use pywr_core::metric::Metric; use pywr_core::models::{ModelDomain, MultiNetworkTransferIndex}; @@ -241,6 +242,7 @@ impl Parameter { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result { let ty = match self { Self::Constant(p) => ParameterType::Parameter(p.add_to_model(network, tables)?), @@ -251,6 +253,7 @@ impl Parameter { tables, data_path, inter_network_transfers, + timeseries, )?), Self::Aggregated(p) => ParameterType::Parameter(p.add_to_model( network, @@ -259,6 +262,7 @@ impl Parameter { tables, data_path, inter_network_transfers, + timeseries, )?), Self::AggregatedIndex(p) => ParameterType::Index(p.add_to_model( network, @@ -267,6 +271,7 @@ impl Parameter { tables, data_path, inter_network_transfers, + timeseries, )?), Self::AsymmetricSwitchIndex(p) => ParameterType::Index(p.add_to_model( network, @@ -275,6 +280,7 @@ impl Parameter { tables, data_path, inter_network_transfers, + timeseries, )?), Self::ControlCurvePiecewiseInterpolated(p) => ParameterType::Parameter(p.add_to_model( network, @@ -283,6 +289,7 @@ impl Parameter { tables, data_path, inter_network_transfers, + timeseries, )?), Self::ControlCurveIndex(p) => ParameterType::Index(p.add_to_model( network, @@ -291,6 +298,7 @@ impl Parameter { tables, data_path, inter_network_transfers, + timeseries, )?), Self::ControlCurve(p) => ParameterType::Parameter(p.add_to_model( network, @@ -299,6 +307,7 @@ impl Parameter { tables, data_path, inter_network_transfers, + timeseries, )?), Self::DailyProfile(p) => ParameterType::Parameter(p.add_to_model(network, tables)?), Self::IndexedArray(p) => ParameterType::Parameter(p.add_to_model( @@ -308,6 +317,7 @@ impl Parameter { tables, data_path, inter_network_transfers, + timeseries, )?), Self::MonthlyProfile(p) => ParameterType::Parameter(p.add_to_model(network, tables)?), Self::UniformDrawdownProfile(p) => ParameterType::Parameter(p.add_to_model(network, tables)?), @@ -318,6 +328,7 @@ impl Parameter { tables, data_path, inter_network_transfers, + timeseries, )?), Self::Min(p) => ParameterType::Parameter(p.add_to_model( network, @@ -326,6 +337,7 @@ impl Parameter { tables, data_path, inter_network_transfers, + timeseries, )?), Self::Negative(p) => ParameterType::Parameter(p.add_to_model( network, @@ -334,6 +346,7 @@ impl Parameter { tables, data_path, inter_network_transfers, + timeseries, )?), Self::Polynomial1D(p) => ParameterType::Parameter(p.add_to_model(network)?), Self::ParameterThreshold(p) => ParameterType::Index(p.add_to_model( @@ -343,9 +356,18 @@ impl Parameter { tables, data_path, inter_network_transfers, + timeseries, )?), Self::TablesArray(p) => ParameterType::Parameter(p.add_to_model(network, domain, data_path)?), - Self::Python(p) => p.add_to_model(network, schema, domain, tables, data_path, inter_network_transfers)?, + Self::Python(p) => p.add_to_model( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?, Self::DataFrame(p) => ParameterType::Parameter(p.add_to_model(network, domain, data_path)?), Self::Delay(p) => ParameterType::Parameter(p.add_to_model( network, @@ -354,6 +376,7 @@ impl Parameter { tables, data_path, inter_network_transfers, + timeseries, )?), Self::Division(p) => ParameterType::Parameter(p.add_to_model( network, @@ -362,6 +385,7 @@ impl Parameter { tables, data_path, inter_network_transfers, + timeseries, )?), Self::Offset(p) => ParameterType::Parameter(p.add_to_model( network, @@ -370,6 +394,7 @@ impl Parameter { tables, data_path, inter_network_transfers, + timeseries, )?), Self::DiscountFactor(p) => ParameterType::Parameter(p.add_to_model( network, @@ -378,6 +403,7 @@ impl Parameter { tables, data_path, inter_network_transfers, + timeseries, )?), Self::Interpolated(p) => ParameterType::Parameter(p.add_to_model( network, @@ -386,6 +412,7 @@ impl Parameter { tables, data_path, inter_network_transfers, + timeseries, )?), Self::RbfProfile(p) => ParameterType::Parameter(p.add_to_model(network)?), }; @@ -643,6 +670,7 @@ impl MetricFloatReference { pub enum MetricFloatValue { Reference(MetricFloatReference), InlineParameter { definition: Box }, + Timeseries { name: String, columns: DataFrameColumns }, } impl MetricFloatValue { @@ -655,6 +683,7 @@ impl MetricFloatValue { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result { match self { Self::Reference(reference) => Ok(reference.load(network, schema, inter_network_transfers)?), @@ -673,7 +702,7 @@ impl MetricFloatValue { } Err(_) => { // An error retrieving a parameter with this name; assume it needs creating. - match definition.add_to_model(network, schema, &domain, tables, data_path, inter_network_transfers)? { + match definition.add_to_model(network, schema, &domain, tables, data_path, inter_network_transfers, timeseries)? { ParameterType::Parameter(idx) => Ok(Metric::ParameterValue(idx)), ParameterType::Index(_) => Err(SchemaError::UnexpectedParameterType(format!( "Found index parameter of type '{}' with name '{}' where an float parameter was expected.", @@ -689,6 +718,13 @@ impl MetricFloatValue { } } } + Self::Timeseries { name, columns } => { + let param_idx = match columns { + DataFrameColumns::Scenario(scenario) => timeseries.load_df(network, name, domain, scenario)?, + DataFrameColumns::Column(col) => timeseries.load_column(network, name, col)?, + }; + Ok(Metric::ParameterValue(param_idx)) + } } } } @@ -710,6 +746,7 @@ impl ParameterIndexValue { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result { match self { Self::Reference(name) => { @@ -718,7 +755,7 @@ impl ParameterIndexValue { } Self::Inline(parameter) => { // Inline parameter needs to be added - match parameter.add_to_model(network, schema, domain, tables, data_path, inter_network_transfers)? { + match parameter.add_to_model(network, schema, domain, tables, data_path, inter_network_transfers, timeseries)? { ParameterType::Index(idx) => Ok(idx), ParameterType::Parameter(_) => Err(SchemaError::UnexpectedParameterType(format!( "Found float parameter of type '{}' with name '{}' where an index parameter was expected.", @@ -766,12 +803,19 @@ impl DynamicFloatValue { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result { let parameter_ref = match self { DynamicFloatValue::Constant(v) => Metric::Constant(v.load(tables)?), - DynamicFloatValue::Dynamic(v) => { - v.load(network, schema, domain, tables, data_path, inter_network_transfers)? - } + DynamicFloatValue::Dynamic(v) => v.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?, }; Ok(parameter_ref) } @@ -827,12 +871,19 @@ impl DynamicIndexValue { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result { let parameter_ref = match self { DynamicIndexValue::Constant(v) => IndexValue::Constant(v.load(tables)?), - DynamicIndexValue::Dynamic(v) => { - IndexValue::Dynamic(v.load(network, schema, domain, tables, data_path, inter_network_transfers)?) - } + DynamicIndexValue::Dynamic(v) => IndexValue::Dynamic(v.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?), }; Ok(parameter_ref) } diff --git a/pywr-schema/src/parameters/offset.rs b/pywr-schema/src/parameters/offset.rs index deca374c..bd062168 100644 --- a/pywr-schema/src/parameters/offset.rs +++ b/pywr-schema/src/parameters/offset.rs @@ -1,5 +1,6 @@ use crate::data_tables::LoadedTableCollection; use crate::parameters::{ConstantValue, DynamicFloatValue, DynamicFloatValueType, ParameterMeta, VariableSettings}; +use crate::timeseries::LoadedTimeseriesCollection; use pywr_core::parameters::ParameterIndex; use crate::error::SchemaError; @@ -57,6 +58,7 @@ impl OffsetParameter { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result { let variable = match &self.variable { None => None, @@ -70,9 +72,15 @@ impl OffsetParameter { } }; - let idx = self - .metric - .load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let idx = self.metric.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; let p = pywr_core::parameters::OffsetParameter::new(&self.meta.name, idx, self.offset.load(tables)?, variable); Ok(network.add_parameter(Box::new(p))?) diff --git a/pywr-schema/src/parameters/python.rs b/pywr-schema/src/parameters/python.rs index 52b62f61..f3794ff2 100644 --- a/pywr-schema/src/parameters/python.rs +++ b/pywr-schema/src/parameters/python.rs @@ -2,6 +2,7 @@ use crate::data_tables::{make_path, LoadedTableCollection}; use crate::error::SchemaError; use crate::model::PywrMultiNetworkTransfer; use crate::parameters::{DynamicFloatValue, DynamicFloatValueType, DynamicIndexValue, ParameterMeta}; +use crate::timeseries::LoadedTimeseriesCollection; use pyo3::prelude::PyModule; use pyo3::types::{PyDict, PyTuple}; use pyo3::{IntoPy, PyErr, PyObject, Python, ToPyObject}; @@ -131,6 +132,7 @@ impl PythonParameter { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result { pyo3::prepare_freethreaded_python(); @@ -172,7 +174,15 @@ impl PythonParameter { .map(|(k, v)| { Ok(( k.to_string(), - v.load(network, schema, domain, tables, data_path, inter_network_transfers)?, + v.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?, )) }) .collect::, SchemaError>>()?, @@ -185,7 +195,15 @@ impl PythonParameter { .map(|(k, v)| { Ok(( k.to_string(), - v.load(network, schema, domain, tables, data_path, inter_network_transfers)?, + v.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?, )) }) .collect::, SchemaError>>()?, @@ -208,6 +226,7 @@ mod tests { use crate::data_tables::LoadedTableCollection; use crate::model::PywrNetwork; use crate::parameters::python::PythonParameter; + use crate::timeseries::LoadedTimeseriesCollection; use pywr_core::models::ModelDomain; use pywr_core::network::Network; use pywr_core::test_utils::default_time_domain; @@ -259,8 +278,9 @@ class MyParameter: let schema = PywrNetwork::default(); let mut network = Network::default(); let tables = LoadedTableCollection::from_schema(None, None).unwrap(); + let timeseries = LoadedTimeseriesCollection::from_schema(None, &domain, None).unwrap(); param - .add_to_model(&mut network, &schema, &domain, &tables, None, &[]) + .add_to_model(&mut network, &schema, &domain, &tables, None, &[], ×eries) .unwrap(); } } diff --git a/pywr-schema/src/parameters/thresholds.rs b/pywr-schema/src/parameters/thresholds.rs index e24ba041..e10da05e 100644 --- a/pywr-schema/src/parameters/thresholds.rs +++ b/pywr-schema/src/parameters/thresholds.rs @@ -4,6 +4,7 @@ use crate::model::PywrMultiNetworkTransfer; use crate::parameters::{ DynamicFloatValue, DynamicFloatValueType, IntoV2Parameter, ParameterMeta, TryFromV1Parameter, TryIntoV2Parameter, }; +use crate::timeseries::LoadedTimeseriesCollection; use pywr_core::models::ModelDomain; use pywr_core::parameters::IndexParameterIndex; use pywr_v1_schema::parameters::{ @@ -77,13 +78,26 @@ impl ParameterThresholdParameter { tables: &LoadedTableCollection, data_path: Option<&Path>, inter_network_transfers: &[PywrMultiNetworkTransfer], + timeseries: &LoadedTimeseriesCollection, ) -> Result { - let metric = self - .parameter - .load(network, schema, domain, tables, data_path, inter_network_transfers)?; - let threshold = self - .threshold - .load(network, schema, domain, tables, data_path, inter_network_transfers)?; + let metric = self.parameter.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; + let threshold = self.threshold.load( + network, + schema, + domain, + tables, + data_path, + inter_network_transfers, + timeseries, + )?; let p = pywr_core::parameters::ThresholdParameter::new( &self.meta.name, diff --git a/pywr-schema/src/test_models/inflow.csv b/pywr-schema/src/test_models/inflow.csv new file mode 100644 index 00000000..40df8051 --- /dev/null +++ b/pywr-schema/src/test_models/inflow.csv @@ -0,0 +1,366 @@ +date,inflow1,inflow2 +01/01/2021,1,2 +02/01/2021,2,2 +03/01/2021,3,2 +04/01/2021,4,2 +05/01/2021,5,2 +06/01/2021,6,2 +07/01/2021,7,2 +08/01/2021,8,2 +09/01/2021,9,2 +10/01/2021,10,2 +11/01/2021,11,2 +12/01/2021,12,2 +13/01/2021,13,2 +14/01/2021,14,2 +15/01/2021,15,2 +16/01/2021,16,2 +17/01/2021,17,2 +18/01/2021,18,2 +19/01/2021,19,2 +20/01/2021,20,2 +21/01/2021,21,2 +22/01/2021,22,2 +23/01/2021,23,2 +24/01/2021,24,2 +25/01/2021,25,2 +26/01/2021,26,2 +27/01/2021,27,2 +28/01/2021,28,2 +29/01/2021,29,2 +30/01/2021,30,2 +31/01/2021,31,2 +01/02/2021,1,2 +02/02/2021,2,2 +03/02/2021,3,2 +04/02/2021,4,2 +05/02/2021,5,2 +06/02/2021,6,2 +07/02/2021,7,2 +08/02/2021,8,2 +09/02/2021,9,2 +10/02/2021,10,2 +11/02/2021,11,2 +12/02/2021,12,2 +13/02/2021,13,2 +14/02/2021,14,2 +15/02/2021,15,2 +16/02/2021,16,2 +17/02/2021,17,2 +18/02/2021,18,2 +19/02/2021,19,2 +20/02/2021,20,2 +21/02/2021,21,2 +22/02/2021,22,2 +23/02/2021,23,2 +24/02/2021,24,2 +25/02/2021,25,2 +26/02/2021,26,2 +27/02/2021,27,2 +28/02/2021,28,2 +01/03/2021,1,2 +02/03/2021,2,2 +03/03/2021,3,2 +04/03/2021,4,2 +05/03/2021,5,2 +06/03/2021,6,2 +07/03/2021,7,2 +08/03/2021,8,2 +09/03/2021,9,2 +10/03/2021,10,2 +11/03/2021,11,2 +12/03/2021,12,2 +13/03/2021,13,2 +14/03/2021,14,2 +15/03/2021,15,2 +16/03/2021,16,2 +17/03/2021,17,2 +18/03/2021,18,2 +19/03/2021,19,2 +20/03/2021,20,2 +21/03/2021,21,2 +22/03/2021,22,2 +23/03/2021,23,2 +24/03/2021,24,2 +25/03/2021,25,2 +26/03/2021,26,2 +27/03/2021,27,2 +28/03/2021,28,2 +29/03/2021,29,2 +30/03/2021,30,2 +31/03/2021,31,2 +01/04/2021,1,2 +02/04/2021,2,2 +03/04/2021,3,2 +04/04/2021,4,2 +05/04/2021,5,2 +06/04/2021,6,2 +07/04/2021,7,2 +08/04/2021,8,2 +09/04/2021,9,2 +10/04/2021,10,2 +11/04/2021,11,2 +12/04/2021,12,2 +13/04/2021,13,2 +14/04/2021,14,2 +15/04/2021,15,2 +16/04/2021,16,2 +17/04/2021,17,2 +18/04/2021,18,2 +19/04/2021,19,2 +20/04/2021,20,2 +21/04/2021,21,2 +22/04/2021,22,2 +23/04/2021,23,2 +24/04/2021,24,2 +25/04/2021,25,2 +26/04/2021,26,2 +27/04/2021,27,2 +28/04/2021,28,2 +29/04/2021,29,2 +30/04/2021,30,2 +01/05/2021,1,2 +02/05/2021,2,2 +03/05/2021,3,2 +04/05/2021,4,2 +05/05/2021,5,2 +06/05/2021,6,2 +07/05/2021,7,2 +08/05/2021,8,2 +09/05/2021,9,2 +10/05/2021,10,2 +11/05/2021,11,2 +12/05/2021,12,2 +13/05/2021,13,2 +14/05/2021,14,2 +15/05/2021,15,2 +16/05/2021,16,2 +17/05/2021,17,2 +18/05/2021,18,2 +19/05/2021,19,2 +20/05/2021,20,2 +21/05/2021,21,2 +22/05/2021,22,2 +23/05/2021,23,2 +24/05/2021,24,2 +25/05/2021,25,2 +26/05/2021,26,2 +27/05/2021,27,2 +28/05/2021,28,2 +29/05/2021,29,2 +30/05/2021,30,2 +31/05/2021,31,2 +01/06/2021,1,2 +02/06/2021,2,2 +03/06/2021,3,2 +04/06/2021,4,2 +05/06/2021,5,2 +06/06/2021,6,2 +07/06/2021,7,2 +08/06/2021,8,2 +09/06/2021,9,2 +10/06/2021,10,2 +11/06/2021,11,2 +12/06/2021,12,2 +13/06/2021,13,2 +14/06/2021,14,2 +15/06/2021,15,2 +16/06/2021,16,2 +17/06/2021,17,2 +18/06/2021,18,2 +19/06/2021,19,2 +20/06/2021,20,2 +21/06/2021,21,2 +22/06/2021,22,2 +23/06/2021,23,2 +24/06/2021,24,2 +25/06/2021,25,2 +26/06/2021,26,2 +27/06/2021,27,2 +28/06/2021,28,2 +29/06/2021,29,2 +30/06/2021,30,2 +01/07/2021,1,2 +02/07/2021,2,2 +03/07/2021,3,2 +04/07/2021,4,2 +05/07/2021,5,2 +06/07/2021,6,2 +07/07/2021,7,2 +08/07/2021,8,2 +09/07/2021,9,2 +10/07/2021,10,2 +11/07/2021,11,2 +12/07/2021,12,2 +13/07/2021,13,2 +14/07/2021,14,2 +15/07/2021,15,2 +16/07/2021,16,2 +17/07/2021,17,2 +18/07/2021,18,2 +19/07/2021,19,2 +20/07/2021,20,2 +21/07/2021,21,2 +22/07/2021,22,2 +23/07/2021,23,2 +24/07/2021,24,2 +25/07/2021,25,2 +26/07/2021,26,2 +27/07/2021,27,2 +28/07/2021,28,2 +29/07/2021,29,2 +30/07/2021,30,2 +31/07/2021,31,2 +01/08/2021,1,2 +02/08/2021,2,2 +03/08/2021,3,2 +04/08/2021,4,2 +05/08/2021,5,2 +06/08/2021,6,2 +07/08/2021,7,2 +08/08/2021,8,2 +09/08/2021,9,2 +10/08/2021,10,2 +11/08/2021,11,2 +12/08/2021,12,2 +13/08/2021,13,2 +14/08/2021,14,2 +15/08/2021,15,2 +16/08/2021,16,2 +17/08/2021,17,2 +18/08/2021,18,2 +19/08/2021,19,2 +20/08/2021,20,2 +21/08/2021,21,2 +22/08/2021,22,2 +23/08/2021,23,2 +24/08/2021,24,2 +25/08/2021,25,2 +26/08/2021,26,2 +27/08/2021,27,2 +28/08/2021,28,2 +29/08/2021,29,2 +30/08/2021,30,2 +31/08/2021,31,2 +01/09/2021,1,2 +02/09/2021,2,2 +03/09/2021,3,2 +04/09/2021,4,2 +05/09/2021,5,2 +06/09/2021,6,2 +07/09/2021,7,2 +08/09/2021,8,2 +09/09/2021,9,2 +10/09/2021,10,2 +11/09/2021,11,2 +12/09/2021,12,2 +13/09/2021,13,2 +14/09/2021,14,2 +15/09/2021,15,2 +16/09/2021,16,2 +17/09/2021,17,2 +18/09/2021,18,2 +19/09/2021,19,2 +20/09/2021,20,2 +21/09/2021,21,2 +22/09/2021,22,2 +23/09/2021,23,2 +24/09/2021,24,2 +25/09/2021,25,2 +26/09/2021,26,2 +27/09/2021,27,2 +28/09/2021,28,2 +29/09/2021,29,2 +30/09/2021,30,2 +01/10/2021,1,2 +02/10/2021,2,2 +03/10/2021,3,2 +04/10/2021,4,2 +05/10/2021,5,2 +06/10/2021,6,2 +07/10/2021,7,2 +08/10/2021,8,2 +09/10/2021,9,2 +10/10/2021,10,2 +11/10/2021,11,2 +12/10/2021,12,2 +13/10/2021,13,2 +14/10/2021,14,2 +15/10/2021,15,2 +16/10/2021,16,2 +17/10/2021,17,2 +18/10/2021,18,2 +19/10/2021,19,2 +20/10/2021,20,2 +21/10/2021,21,2 +22/10/2021,22,2 +23/10/2021,23,2 +24/10/2021,24,2 +25/10/2021,25,2 +26/10/2021,26,2 +27/10/2021,27,2 +28/10/2021,28,2 +29/10/2021,29,2 +30/10/2021,30,2 +31/10/2021,31,2 +01/11/2021,1,2 +02/11/2021,2,2 +03/11/2021,3,2 +04/11/2021,4,2 +05/11/2021,5,2 +06/11/2021,6,2 +07/11/2021,7,2 +08/11/2021,8,2 +09/11/2021,9,2 +10/11/2021,10,2 +11/11/2021,11,2 +12/11/2021,12,2 +13/11/2021,13,2 +14/11/2021,14,2 +15/11/2021,15,2 +16/11/2021,16,2 +17/11/2021,17,2 +18/11/2021,18,2 +19/11/2021,19,2 +20/11/2021,20,2 +21/11/2021,21,2 +22/11/2021,22,2 +23/11/2021,23,2 +24/11/2021,24,2 +25/11/2021,25,2 +26/11/2021,26,2 +27/11/2021,27,2 +28/11/2021,28,2 +29/11/2021,29,2 +30/11/2021,30,2 +01/12/2021,1,2 +02/12/2021,2,2 +03/12/2021,3,2 +04/12/2021,4,2 +05/12/2021,5,2 +06/12/2021,6,2 +07/12/2021,7,2 +08/12/2021,8,2 +09/12/2021,9,2 +10/12/2021,10,2 +11/12/2021,11,2 +12/12/2021,12,2 +13/12/2021,13,2 +14/12/2021,14,2 +15/12/2021,15,2 +16/12/2021,16,2 +17/12/2021,17,2 +18/12/2021,18,2 +19/12/2021,19,2 +20/12/2021,20,2 +21/12/2021,21,2 +22/12/2021,22,2 +23/12/2021,23,2 +24/12/2021,24,2 +25/12/2021,25,2 +26/12/2021,26,2 +27/12/2021,27,2 +28/12/2021,28,2 +29/12/2021,29,2 +30/12/2021,30,2 +31/12/2021,31,2 diff --git a/pywr-schema/src/test_models/timeseries.json b/pywr-schema/src/test_models/timeseries.json new file mode 100644 index 00000000..1b1e7b03 --- /dev/null +++ b/pywr-schema/src/test_models/timeseries.json @@ -0,0 +1,82 @@ +{ + "metadata": { + "title": "Simple timeseries" + }, + "timestepper": { + "start": "2021-01-01", + "end": "2021-12-31", + "timestep": 1 + }, + "network": { + "nodes": [ + { + "name": "input1", + "type": "Input", + "max_flow": { + "type": "Timeseries", + "name": "inflow", + "columns": { + "type": "Column", + "name": "inflow1" + } + } + }, + { + "name": "input2", + "type": "Input", + "max_flow": { + "type": "Timeseries", + "name": "inflow", + "columns": { + "type": "Column", + "name": "inflow2" + } + } + }, + { + "name": "link1", + "type": "Link" + }, + { + "name": "output1", + "type": "Output", + "cost": -10.0, + "max_flow": { + "type": "Parameter", + "name": "demand" + } + } + ], + "edges": [ + { + "from_node": "input1", + "to_node": "link1" + }, + { + "from_node": "input2", + "to_node": "link1" + }, + { + "from_node": "link1", + "to_node": "output1" + } + ], + "parameters": [ + { + "name": "demand", + "type": "Constant", + "value": 100.0 + } + ], + "timeseries": [ + { + "name": "inflow", + "provider": { + "type": "Polars", + "index_col": 0 + }, + "url": "../test_models/inflow.csv" + } + ] + } +} diff --git a/pywr-schema/src/timeseries/mod.rs b/pywr-schema/src/timeseries/mod.rs new file mode 100644 index 00000000..7fe9f64f --- /dev/null +++ b/pywr-schema/src/timeseries/mod.rs @@ -0,0 +1,153 @@ +mod polars_dataset; + +use ndarray::{Array1, Array2}; +use polars::error::PolarsError; +use polars::prelude::DataType::Float64; +use polars::prelude::{DataFrame, Float64Type, IndexOrder, Schema}; +use pywr_core::models::ModelDomain; +use pywr_core::parameters::{Array1Parameter, Array2Parameter, ParameterIndex}; +use std::{ + collections::HashMap, + path::{Path, PathBuf}, +}; +use thiserror::Error; + +use crate::{parameters::ParameterMeta, SchemaError}; + +use self::polars_dataset::PolarsDataset; + +#[derive(serde::Deserialize, serde::Serialize, Debug, Clone)] +#[serde(tag = "type")] +enum TimeseriesProvider { + Pandas, + Polars(PolarsDataset), +} + +struct ResamplingArgs {} + +#[derive(serde::Deserialize, serde::Serialize, Debug, Clone)] +pub struct Timeseries { + #[serde(flatten)] + names: ParameterMeta, + provider: TimeseriesProvider, + url: PathBuf, +} + +impl Timeseries { + pub fn load(&self, domain: &ModelDomain, data_path: Option<&Path>) -> Result { + match &self.provider { + TimeseriesProvider::Polars(dataset) => dataset.load(self.url.as_path(), data_path), + TimeseriesProvider::Pandas => todo!(), + } + } +} + +fn align_and_resample(df: &DataFrame, domain: &ModelDomain) -> Result { + // 1. check that time column contains datetime + // 2. get domain timestep frequency + // 3. get df timestep frequency + // 4. resample df if they don't match + // 5. slice df to align with domain start and end dates + + todo!() +} + +pub struct LoadedTimeseriesCollection { + timeseries: HashMap, +} + +impl LoadedTimeseriesCollection { + pub fn from_schema( + timeseries_defs: Option<&[Timeseries]>, + domain: &ModelDomain, + data_path: Option<&Path>, + ) -> Result { + let mut timeseries = HashMap::new(); + if let Some(timeseries_defs) = timeseries_defs { + for ts in timeseries_defs { + let df = ts.load(domain, data_path)?; + timeseries.insert(ts.names.name.clone(), df); + } + } + Ok(Self { timeseries }) + } + + pub fn load_column( + &self, + network: &mut pywr_core::network::Network, + name: &str, + col: &str, + ) -> Result { + let df = self + .timeseries + .get(name) + .ok_or(SchemaError::TimeseriesNotFound(name.to_string()))?; + let series = df.column(col)?; + + let array = series.cast(&Float64)?.f64()?.to_ndarray()?.to_owned(); + let name = format!("{}_{}", name, col); + let p = Array1Parameter::new(&name, array, None); + Ok(network.add_parameter(Box::new(p))?) + } + + pub fn load_df( + &self, + network: &mut pywr_core::network::Network, + name: &str, + domain: &ModelDomain, + scenario: &str, + ) -> Result { + let scenario_group_index = domain + .scenarios() + .group_index(scenario) + .ok_or(SchemaError::ScenarioGroupNotFound(scenario.to_string()))?; + + let df = self + .timeseries + .get(name) + .ok_or(SchemaError::TimeseriesNotFound(name.to_string()))?; + + let array: Array2 = df.to_ndarray::(IndexOrder::default()).unwrap(); + let name = format!("{}_{}", name, scenario); + let p = Array2Parameter::new(&name, array, scenario_group_index, None); + Ok(network.add_parameter(Box::new(p))?) + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use ndarray::Array; + use pywr_core::{metric::Metric, recorders::AssertionRecorder, test_utils::run_all_solvers}; + use time::Date; + + use crate::PywrModel; + + fn model_str() -> &'static str { + include_str!("../test_models/timeseries.json") + } + + #[test] + fn test_timeseries_polars() { + let cargo_manifest_dir = env!("CARGO_MANIFEST_DIR"); + + let model_dir = PathBuf::from(cargo_manifest_dir).join("src/test_models"); + + dbg!(&model_dir); + + let data = model_str(); + let schema: PywrModel = serde_json::from_str(data).unwrap(); + let mut model = schema.build_model(Some(model_dir.as_path()), None).unwrap(); + + let expected = Array::from_shape_fn((365, 1), |(x, _)| { + (Date::from_ordinal_date(2021, (x + 1) as u16).unwrap().day() + 2) as f64 + }); + let idx = model.network().get_node_by_name("output1", None).unwrap().index(); + + let recorder = AssertionRecorder::new("output-flow", Metric::NodeInFlow(idx), expected.clone(), None, None); + model.network_mut().add_recorder(Box::new(recorder)).unwrap(); + + run_all_solvers(&model) + } +} diff --git a/pywr-schema/src/timeseries/pandas.rs b/pywr-schema/src/timeseries/pandas.rs new file mode 100644 index 00000000..e69de29b diff --git a/pywr-schema/src/timeseries/polars_dataset.rs b/pywr-schema/src/timeseries/polars_dataset.rs new file mode 100644 index 00000000..f41973f3 --- /dev/null +++ b/pywr-schema/src/timeseries/polars_dataset.rs @@ -0,0 +1,56 @@ +use std::path::Path; + +use polars::{frame::DataFrame, prelude::*}; + +use crate::SchemaError; + +#[derive(serde::Deserialize, serde::Serialize, Debug, Clone)] +pub struct PolarsDataset { + #[serde(flatten)] + index_col: Option, +} + +impl PolarsDataset { + pub fn load(&self, url: &Path, data_path: Option<&Path>) -> Result { + let fp = if url.is_absolute() { + url.to_path_buf() + } else if let Some(data_path) = data_path { + data_path.join(url) + } else { + url.to_path_buf() + }; + + let df = match fp.extension() { + Some(ext) => match ext.to_str() { + Some("csv") => CsvReader::from_path(fp)? + .infer_schema(None) + .with_try_parse_dates(true) + .has_header(true) + .finish()?, + Some("parquet") => { + todo!() + } + Some(other_ext) => { + return Err(SchemaError::TimeseriesUnsupportedFileFormat { + provider: "polars".to_string(), + fmt: other_ext.to_string(), + }) + } + None => { + return Err(SchemaError::TimeseriesUnparsableFileFormat { + provider: "polars".to_string(), + path: url.to_string_lossy().to_string(), + }) + } + }, + None => { + return Err(SchemaError::TimeseriesUnparsableFileFormat { + provider: "polars".to_string(), + path: url.to_string_lossy().to_string(), + }) + } + }; + + Ok(df) + } +} From 2b308ecc44d15c28f0c6b05a8f911498a319c639 Mon Sep 17 00:00:00 2001 From: James Batchelor Date: Sat, 10 Feb 2024 23:54:03 +0000 Subject: [PATCH 02/12] add a align_and_resample timeseries mod --- pywr-core/src/models/mod.rs | 1 + pywr-core/src/scenario.rs | 6 + pywr-core/src/timestep.rs | 9 + pywr-schema/Cargo.toml | 3 +- pywr-schema/src/error.rs | 2 + pywr-schema/src/test_models/timeseries.json | 6 +- .../src/timeseries/align_and_resample.rs | 248 ++++++++++++++++++ pywr-schema/src/timeseries/mod.rs | 21 +- pywr-schema/src/timeseries/polars_dataset.rs | 70 ++--- 9 files changed, 315 insertions(+), 51 deletions(-) create mode 100644 pywr-schema/src/timeseries/align_and_resample.rs diff --git a/pywr-core/src/models/mod.rs b/pywr-core/src/models/mod.rs index 36cd5999..e32db4c0 100644 --- a/pywr-core/src/models/mod.rs +++ b/pywr-core/src/models/mod.rs @@ -6,6 +6,7 @@ use crate::timestep::{TimeDomain, Timestepper}; pub use multi::{MultiNetworkModel, MultiNetworkTransferIndex}; pub use simple::Model; +#[derive(Debug)] pub struct ModelDomain { time: TimeDomain, scenarios: ScenarioDomain, diff --git a/pywr-core/src/scenario.rs b/pywr-core/src/scenario.rs index 0dbc7c84..de8e43d7 100644 --- a/pywr-core/src/scenario.rs +++ b/pywr-core/src/scenario.rs @@ -31,6 +31,11 @@ pub struct ScenarioGroupCollection { } impl ScenarioGroupCollection { + pub fn new(groups: Vec) -> Self { + Self { groups } + } + + /// Number of [`ScenarioGroup`]s in the collection. pub fn len(&self) -> usize { self.groups.len() @@ -103,6 +108,7 @@ impl ScenarioIndex { } } +#[derive(Debug)] pub struct ScenarioDomain { scenario_indices: Vec, scenario_group_names: Vec, diff --git a/pywr-core/src/timestep.rs b/pywr-core/src/timestep.rs index 326e0c07..ce2988d9 100644 --- a/pywr-core/src/timestep.rs +++ b/pywr-core/src/timestep.rs @@ -77,6 +77,7 @@ impl Timestepper { } /// The time domain that a model will be simulated over. +#[derive(Debug)] pub struct TimeDomain { timesteps: Vec, } @@ -98,6 +99,14 @@ impl TimeDomain { pub fn len(&self) -> usize { self.timesteps.len() } + + pub fn first_timestep(&self) -> &Timestep { + self.timesteps.first().expect("No time-steps defined.") + } + + pub fn last_timestep(&self) -> &Timestep { + self.timesteps.last().expect("No time-steps defined.") + } } impl From for TimeDomain { diff --git a/pywr-schema/Cargo.toml b/pywr-schema/Cargo.toml index e9f44778..e71152f1 100644 --- a/pywr-schema/Cargo.toml +++ b/pywr-schema/Cargo.toml @@ -15,7 +15,7 @@ categories = ["science", "simulation"] [dependencies] svgbobdoc = { version = "0.3.0", features = ["enable"] } -polars = { workspace = true, features = ["csv"] } +polars = { workspace = true, features = ["csv", "diff", "dtype-datetime", "dtype-date", "dynamic_group_by"] } pyo3 = { workspace = true } pyo3-polars = { workspace = true } strum = "0.26" @@ -32,6 +32,7 @@ thiserror = { workspace = true } pywr-v1-schema = { workspace = true } time = { workspace = true, features = ["serde", "serde-well-known", "serde-human-readable", "macros"] } pywr-core = { path="../pywr-core" } +chrono = "0.4.33" [dev-dependencies] tempfile = "3.3.0" diff --git a/pywr-schema/src/error.rs b/pywr-schema/src/error.rs index 80f49008..35da8488 100644 --- a/pywr-schema/src/error.rs +++ b/pywr-schema/src/error.rs @@ -31,6 +31,8 @@ pub enum SchemaError { DataTable(#[from] TableError), #[error("Timeseries '{0} not found")] TimeseriesNotFound(String), + #[error("The duration of timeseries '{0}' could not be determined.")] + TimeseriesDurationNotFound(String), #[error("Column '{col}' not found in timeseries input '{name}'")] ColumnNotFound { col: String, name: String }, #[error("Timeseries provider '{provider}' does not support '{fmt}' file types")] diff --git a/pywr-schema/src/test_models/timeseries.json b/pywr-schema/src/test_models/timeseries.json index 1b1e7b03..76845396 100644 --- a/pywr-schema/src/test_models/timeseries.json +++ b/pywr-schema/src/test_models/timeseries.json @@ -73,9 +73,9 @@ "name": "inflow", "provider": { "type": "Polars", - "index_col": 0 - }, - "url": "../test_models/inflow.csv" + "time_col": "date", + "url": "../test_models/inflow.csv" + } } ] } diff --git a/pywr-schema/src/timeseries/align_and_resample.rs b/pywr-schema/src/timeseries/align_and_resample.rs new file mode 100644 index 00000000..edcb1d47 --- /dev/null +++ b/pywr-schema/src/timeseries/align_and_resample.rs @@ -0,0 +1,248 @@ +use std::{cmp::Ordering, ops::Deref}; + +use chrono::{format, NaiveDateTime}; + +use polars::{prelude::*, series::ops::NullBehavior}; +use pywr_core::models::ModelDomain; +use pywr_v1_schema::model; + +use crate::SchemaError; + +pub fn align_and_resample( + name: &str, + df: DataFrame, + time_col: &str, + domain: &ModelDomain, +) -> Result { + // Ensure type of time column is datetime and that it is sorted + let df = df + .clone() + .lazy() + .with_columns([col(time_col).cast(DataType::Datetime(TimeUnit::Nanoseconds, None))]) + .collect()? + .sort([time_col], false, true)?; + + // Ensure that df start aligns with models start for any resampling + let df = slice_start(df, time_col, domain)?; + + // Get the durations of the time column + let durations = df + .clone() + .lazy() + .select([col(time_col).diff(1, NullBehavior::Drop).unique().alias("duration")]) + .collect()?; + let durations = durations.column("duration")?.duration()?.deref(); + + if durations.len() > 1 { + // Non-uniform timestep are not yet supported + todo!(); + } + + let timeseries_duration = match durations.get(0) { + Some(duration) => duration, + None => return Err(SchemaError::TimeseriesDurationNotFound(name.to_string())), + }; + + let model_duration = domain.time().step_duration().whole_nanoseconds() as i64; + + let df = match model_duration.cmp(×eries_duration) { + Ordering::Greater => { + // Downsample + df.clone() + .lazy() + .group_by_dynamic( + col(time_col), + [], + DynamicGroupOptions { + every: Duration::new(model_duration), + period: Duration::new(model_duration), + offset: Duration::new(0), + start_by: StartBy::DataPoint, + ..Default::default() + }, + ) + .agg([col("*").exclude([time_col]).mean()]) + .collect()? + } + Ordering::Less => { + // Upsample + // TODO: this does not extend the dataframe beyond its original end date. Should it do when using a forward fill strategy? + // The df could be extend by the length of the duration it is being resampled to. + df.clone() + .upsample::<[String; 0]>([], "time", Duration::new(model_duration), Duration::new(0))? + .fill_null(FillNullStrategy::Forward(None))? + } + Ordering::Equal => df, + }; + + let df = slice_end(df, time_col, domain)?; + + // TODO check df length equals number of model timesteps + Ok(df) +} + +fn slice_start(df: DataFrame, time_col: &str, domain: &ModelDomain) -> Result { + let start = domain.time().first_timestep().date.midnight().assume_utc(); + let start = NaiveDateTime::from_timestamp_opt(start.unix_timestamp(), 0).unwrap(); + let df = df.clone().lazy().filter(col(time_col).gt_eq(lit(start))).collect()?; + Ok(df) +} + +fn slice_end(df: DataFrame, time_col: &str, domain: &ModelDomain) -> Result { + let end = domain.time().last_timestep().date.midnight().assume_utc(); + let end = NaiveDateTime::from_timestamp_opt(end.unix_timestamp(), 0).unwrap(); + let df = df.clone().lazy().filter(col(time_col).lt_eq(lit(end))).collect()?; + Ok(df) +} + +#[cfg(test)] +mod tests { + //use polars::{datatypes::TimeUnit, time::{ClosedWindow, Duration}}; + use chrono::{NaiveDate, NaiveDateTime}; + use polars::prelude::*; + use pywr_core::{ + models::ModelDomain, + scenario::{ScenarioDomain, ScenarioGroupCollection}, + timestep::{TimeDomain, Timestepper}, + }; + use time::{Date, Month}; + + use crate::timeseries::{align_and_resample::align_and_resample, tests}; + + #[test] + fn test_downsample_and_slice() { + let time_domain: TimeDomain = Timestepper::new( + Date::from_calendar_date(2021, Month::January, 7).unwrap(), + Date::from_calendar_date(2021, Month::January, 20).unwrap(), + 7, + ) + .into(); + + let scenario_domain: ScenarioDomain = ScenarioGroupCollection::new(vec![]).into(); + + let domain = ModelDomain::new(time_domain, scenario_domain); + + let time = polars::time::date_range( + "time", + NaiveDate::from_ymd_opt(2021, 1, 1).unwrap().into(), + NaiveDate::from_ymd_opt(2021, 1, 31).unwrap().into(), + Duration::parse("1d"), + ClosedWindow::Both, + TimeUnit::Milliseconds, + None, + ) + .unwrap(); + //let values: Vec = vec![1.0; 31]; + let values: Vec = (1..32).map(|x| x as f64).collect(); + let mut df = df!( + "time" => time, + "values" => values + ) + .unwrap(); + + df = align_and_resample("test", df, "time", &domain).unwrap(); + + let expected_dates = Series::new( + "time", + vec![ + NaiveDateTime::parse_from_str("2021-01-07 00:00:00", "%Y-%m-%d %H:%M:%S").unwrap(), + NaiveDateTime::parse_from_str("2021-01-14 00:00:00", "%Y-%m-%d %H:%M:%S").unwrap(), + ], + ) + .cast(&DataType::Datetime(TimeUnit::Nanoseconds, None)) + .unwrap(); + let resampled_dates = df.column("time").unwrap(); + assert!(resampled_dates.equals(&expected_dates)); + + let expected_values = Series::new( + "values", + vec![ + 10.0, // mean of 7, 8, 9, 10, 11, 12, 13 + 17.0, // mean of 14, 15, 16, 17, 18, 19, 20 + ], + ); + let resampled_values = df.column("values").unwrap(); + assert!(resampled_values.equals(&expected_values)); + } + + #[test] + fn test_upsample_and_slice() { + let time_domain: TimeDomain = Timestepper::new( + Date::from_calendar_date(2021, Month::January, 1).unwrap(), + Date::from_calendar_date(2021, Month::January, 14).unwrap(), + 1, + ) + .into(); + let scenario_domain: ScenarioDomain = ScenarioGroupCollection::new(vec![]).into(); + let domain = ModelDomain::new(time_domain, scenario_domain); + + let time = polars::time::date_range( + "time", + NaiveDate::from_ymd_opt(2021, 1, 1).unwrap().into(), + NaiveDate::from_ymd_opt(2021, 1, 15).unwrap().into(), + Duration::parse("7d"), + ClosedWindow::Both, + TimeUnit::Milliseconds, + None, + ) + .unwrap(); + + let values: Vec = vec![1.0, 2.0, 3.0]; + let mut df = df!( + "time" => time, + "values" => values + ) + .unwrap(); + + df = align_and_resample("test", df, "time", &domain).unwrap(); + + let expected_values = Series::new( + "values", + vec![1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0], + ); + let resampled_values = df.column("values").unwrap(); + assert!(resampled_values.equals(&expected_values)); + } + + #[test] + fn test_no_resample_slice() { + let time_domain: TimeDomain = Timestepper::new( + Date::from_calendar_date(2021, Month::January, 1).unwrap(), + Date::from_calendar_date(2021, Month::January, 3).unwrap(), + 1, + ) + .into(); + let scenario_domain: ScenarioDomain = ScenarioGroupCollection::new(vec![]).into(); + let domain = ModelDomain::new(time_domain, scenario_domain); + + let time = polars::time::date_range( + "time", + NaiveDate::from_ymd_opt(2021, 1, 1).unwrap().into(), + NaiveDate::from_ymd_opt(2021, 1, 3).unwrap().into(), + Duration::parse("1d"), + ClosedWindow::Both, + TimeUnit::Milliseconds, + None, + ) + .unwrap(); + + let values: Vec = vec![1.0, 2.0, 3.0]; + let mut df = df!( + "time" => time.clone(), + "values" => values.clone() + ) + .unwrap(); + + df = align_and_resample("test", df, "time", &domain).unwrap(); + + let expected_values = Series::new("values", values); + let resampled_values = df.column("values").unwrap(); + assert!(resampled_values.equals(&expected_values)); + + let expected_dates = Series::new("time", time) + .cast(&DataType::Datetime(TimeUnit::Nanoseconds, None)) + .unwrap(); + let resampled_dates = df.column("time").unwrap(); + assert!(resampled_dates.equals(&expected_dates)); + } +} diff --git a/pywr-schema/src/timeseries/mod.rs b/pywr-schema/src/timeseries/mod.rs index 7fe9f64f..c3d4ffc0 100644 --- a/pywr-schema/src/timeseries/mod.rs +++ b/pywr-schema/src/timeseries/mod.rs @@ -1,3 +1,4 @@ +mod align_and_resample; mod polars_dataset; use ndarray::{Array1, Array2}; @@ -23,35 +24,22 @@ enum TimeseriesProvider { Polars(PolarsDataset), } -struct ResamplingArgs {} - #[derive(serde::Deserialize, serde::Serialize, Debug, Clone)] pub struct Timeseries { #[serde(flatten)] - names: ParameterMeta, + meta: ParameterMeta, provider: TimeseriesProvider, - url: PathBuf, } impl Timeseries { pub fn load(&self, domain: &ModelDomain, data_path: Option<&Path>) -> Result { match &self.provider { - TimeseriesProvider::Polars(dataset) => dataset.load(self.url.as_path(), data_path), + TimeseriesProvider::Polars(dataset) => dataset.load(self.meta.name.as_str(), data_path, domain), TimeseriesProvider::Pandas => todo!(), } } } -fn align_and_resample(df: &DataFrame, domain: &ModelDomain) -> Result { - // 1. check that time column contains datetime - // 2. get domain timestep frequency - // 3. get df timestep frequency - // 4. resample df if they don't match - // 5. slice df to align with domain start and end dates - - todo!() -} - pub struct LoadedTimeseriesCollection { timeseries: HashMap, } @@ -66,7 +54,8 @@ impl LoadedTimeseriesCollection { if let Some(timeseries_defs) = timeseries_defs { for ts in timeseries_defs { let df = ts.load(domain, data_path)?; - timeseries.insert(ts.names.name.clone(), df); + // TODO error if key already exists + timeseries.insert(ts.meta.name.clone(), df); } } Ok(Self { timeseries }) diff --git a/pywr-schema/src/timeseries/polars_dataset.rs b/pywr-schema/src/timeseries/polars_dataset.rs index f41973f3..c8d624a1 100644 --- a/pywr-schema/src/timeseries/polars_dataset.rs +++ b/pywr-schema/src/timeseries/polars_dataset.rs @@ -1,56 +1,64 @@ -use std::path::Path; +use std::path::{Path, PathBuf}; use polars::{frame::DataFrame, prelude::*}; +use pywr_core::models::ModelDomain; use crate::SchemaError; +use super::align_and_resample::align_and_resample; + #[derive(serde::Deserialize, serde::Serialize, Debug, Clone)] pub struct PolarsDataset { - #[serde(flatten)] - index_col: Option, + time_col: String, + url: PathBuf, } impl PolarsDataset { - pub fn load(&self, url: &Path, data_path: Option<&Path>) -> Result { - let fp = if url.is_absolute() { - url.to_path_buf() + pub fn load(&self, name: &str, data_path: Option<&Path>, domain: &ModelDomain) -> Result { + let fp = if self.url.is_absolute() { + self.url.clone() } else if let Some(data_path) = data_path { - data_path.join(url) + data_path.join(self.url.as_path()) } else { - url.to_path_buf() + self.url.clone() }; - let df = match fp.extension() { - Some(ext) => match ext.to_str() { - Some("csv") => CsvReader::from_path(fp)? - .infer_schema(None) - .with_try_parse_dates(true) - .has_header(true) - .finish()?, - Some("parquet") => { - todo!() - } - Some(other_ext) => { - return Err(SchemaError::TimeseriesUnsupportedFileFormat { - provider: "polars".to_string(), - fmt: other_ext.to_string(), - }) - } - None => { - return Err(SchemaError::TimeseriesUnparsableFileFormat { - provider: "polars".to_string(), - path: url.to_string_lossy().to_string(), - }) + let mut df = match fp.extension() { + Some(ext) => { + let ext = ext.to_str().map(|s| s.to_lowercase()); + match ext.as_deref() { + Some("csv") => CsvReader::from_path(fp)? + .infer_schema(None) + .with_try_parse_dates(true) + .has_header(true) + .finish()?, + Some("parquet") => { + todo!() + } + Some(other_ext) => { + return Err(SchemaError::TimeseriesUnsupportedFileFormat { + provider: "polars".to_string(), + fmt: other_ext.to_string(), + }) + } + None => { + return Err(SchemaError::TimeseriesUnparsableFileFormat { + provider: "polars".to_string(), + path: self.url.to_string_lossy().to_string(), + }) + } } - }, + } None => { return Err(SchemaError::TimeseriesUnparsableFileFormat { provider: "polars".to_string(), - path: url.to_string_lossy().to_string(), + path: self.url.to_string_lossy().to_string(), }) } }; + df = align_and_resample(name, df, self.time_col.as_str(), domain)?; + Ok(df) } } From 1b5c5b1888be9abe2bb82bcf7865f70bc2417c64 Mon Sep 17 00:00:00 2001 From: James Batchelor Date: Sun, 3 Mar 2024 14:02:40 +0000 Subject: [PATCH 03/12] fix: check if param exists before creating param from timeseries input --- .../src/timeseries/align_and_resample.rs | 8 +--- pywr-schema/src/timeseries/mod.rs | 40 +++++++++++++------ 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/pywr-schema/src/timeseries/align_and_resample.rs b/pywr-schema/src/timeseries/align_and_resample.rs index 884a793b..156210b1 100644 --- a/pywr-schema/src/timeseries/align_and_resample.rs +++ b/pywr-schema/src/timeseries/align_and_resample.rs @@ -1,10 +1,6 @@ -use std::{cmp::Ordering, ops::Deref}; - -use chrono::{format, NaiveDateTime}; - use polars::{prelude::*, series::ops::NullBehavior}; use pywr_core::models::ModelDomain; -use pywr_v1_schema::model; +use std::{cmp::Ordering, ops::Deref}; use crate::SchemaError; @@ -47,7 +43,7 @@ pub fn align_and_resample( .time() .step_duration() .whole_nanoseconds() - .expect("Nano seconds could not be extracted from model step duration") as i64; + .expect("Nano seconds could not be extracted from model step duration"); let df = match model_duration.cmp(×eries_duration) { Ordering::Greater => { diff --git a/pywr-schema/src/timeseries/mod.rs b/pywr-schema/src/timeseries/mod.rs index 83d8f5e0..1643b858 100644 --- a/pywr-schema/src/timeseries/mod.rs +++ b/pywr-schema/src/timeseries/mod.rs @@ -1,17 +1,13 @@ mod align_and_resample; mod polars_dataset; -use ndarray::{Array1, Array2}; -use polars::error::PolarsError; +use ndarray::Array2; use polars::prelude::DataType::Float64; -use polars::prelude::{DataFrame, Float64Type, IndexOrder, Schema}; +use polars::prelude::{DataFrame, Float64Type, IndexOrder}; use pywr_core::models::ModelDomain; use pywr_core::parameters::{Array1Parameter, Array2Parameter, ParameterIndex}; -use std::{ - collections::HashMap, - path::{Path, PathBuf}, -}; -use thiserror::Error; +use pywr_core::PywrError; +use std::{collections::HashMap, path::Path}; use crate::{parameters::ParameterMeta, SchemaError}; @@ -75,8 +71,17 @@ impl LoadedTimeseriesCollection { let array = series.cast(&Float64)?.f64()?.to_ndarray()?.to_owned(); let name = format!("{}_{}", name, col); - let p = Array1Parameter::new(&name, array, None); - Ok(network.add_parameter(Box::new(p))?) + + match network.get_parameter_index_by_name(&name) { + Ok(idx) => Ok(idx), + Err(e) => match e { + PywrError::ParameterNotFound(_) => { + let p = Array1Parameter::new(&name, array, None); + Ok(network.add_parameter(Box::new(p))?) + } + _ => Err(SchemaError::PywrCore(e)), + }, + } } pub fn load_df( @@ -97,9 +102,18 @@ impl LoadedTimeseriesCollection { .ok_or(SchemaError::TimeseriesNotFound(name.to_string()))?; let array: Array2 = df.to_ndarray::(IndexOrder::default()).unwrap(); - let name = format!("{}_{}", name, scenario); - let p = Array2Parameter::new(&name, array, scenario_group_index, None); - Ok(network.add_parameter(Box::new(p))?) + let name = format!("timeseries.{}_{}", name, scenario); + + match network.get_parameter_index_by_name(&name) { + Ok(idx) => Ok(idx), + Err(e) => match e { + PywrError::ParameterNotFound(_) => { + let p = Array2Parameter::new(&name, array, scenario_group_index, None); + Ok(network.add_parameter(Box::new(p))?) + } + _ => Err(SchemaError::PywrCore(e)), + }, + } } } From 72fe90ef2162b92a006d854fd297de9b4aa5fbcc Mon Sep 17 00:00:00 2001 From: James Batchelor Date: Sun, 3 Mar 2024 15:01:49 +0000 Subject: [PATCH 04/12] create seperate timeseries error enum --- pywr-schema/src/error.rs | 15 ++----- .../src/timeseries/align_and_resample.rs | 10 ++--- pywr-schema/src/timeseries/mod.rs | 42 ++++++++++++++----- pywr-schema/src/timeseries/polars_dataset.rs | 15 ++++--- 4 files changed, 50 insertions(+), 32 deletions(-) diff --git a/pywr-schema/src/error.rs b/pywr-schema/src/error.rs index 616c629f..49942886 100644 --- a/pywr-schema/src/error.rs +++ b/pywr-schema/src/error.rs @@ -1,5 +1,6 @@ use crate::data_tables::TableError; use crate::nodes::NodeAttribute; +use crate::timeseries::TimeseriesError; use polars::error::PolarsError; use pyo3::exceptions::PyRuntimeError; use pyo3::PyErr; @@ -29,18 +30,6 @@ pub enum SchemaError { PywrCore(#[from] pywr_core::PywrError), #[error("data table error: {0}")] DataTable(#[from] TableError), - #[error("Timeseries '{0} not found")] - TimeseriesNotFound(String), - #[error("The duration of timeseries '{0}' could not be determined.")] - TimeseriesDurationNotFound(String), - #[error("Column '{col}' not found in timeseries input '{name}'")] - ColumnNotFound { col: String, name: String }, - #[error("Timeseries provider '{provider}' does not support '{fmt}' file types")] - TimeseriesUnsupportedFileFormat { provider: String, fmt: String }, - #[error("Timeseries provider '{provider}' cannot parse file: '{path}'")] - TimeseriesUnparsableFileFormat { provider: String, path: String }, - #[error("Polars error: {0}")] - PolarsError(#[from] PolarsError), #[error("Circular node reference(s) found.")] CircularNodeReference, #[error("Circular parameters reference(s) found.")] @@ -67,6 +56,8 @@ pub enum SchemaError { InvalidRollingWindow { name: String }, #[error("Failed to load parameter {name}: {error}")] LoadParameter { name: String, error: String }, + #[error("Timeseries error: {0}")] + Timeseries(#[from] TimeseriesError), } impl From for PyErr { diff --git a/pywr-schema/src/timeseries/align_and_resample.rs b/pywr-schema/src/timeseries/align_and_resample.rs index 156210b1..b7158b5c 100644 --- a/pywr-schema/src/timeseries/align_and_resample.rs +++ b/pywr-schema/src/timeseries/align_and_resample.rs @@ -2,14 +2,14 @@ use polars::{prelude::*, series::ops::NullBehavior}; use pywr_core::models::ModelDomain; use std::{cmp::Ordering, ops::Deref}; -use crate::SchemaError; +use crate::timeseries::TimeseriesError; pub fn align_and_resample( name: &str, df: DataFrame, time_col: &str, domain: &ModelDomain, -) -> Result { +) -> Result { // Ensure type of time column is datetime and that it is sorted let df = df .clone() @@ -36,7 +36,7 @@ pub fn align_and_resample( let timeseries_duration = match durations.get(0) { Some(duration) => duration, - None => return Err(SchemaError::TimeseriesDurationNotFound(name.to_string())), + None => return Err(TimeseriesError::TimeseriesDurationNotFound(name.to_string())), }; let model_duration = domain @@ -81,13 +81,13 @@ pub fn align_and_resample( Ok(df) } -fn slice_start(df: DataFrame, time_col: &str, domain: &ModelDomain) -> Result { +fn slice_start(df: DataFrame, time_col: &str, domain: &ModelDomain) -> Result { let start = domain.time().first_timestep().date; let df = df.clone().lazy().filter(col(time_col).gt_eq(lit(start))).collect()?; Ok(df) } -fn slice_end(df: DataFrame, time_col: &str, domain: &ModelDomain) -> Result { +fn slice_end(df: DataFrame, time_col: &str, domain: &ModelDomain) -> Result { let end = domain.time().last_timestep().date; let df = df.clone().lazy().filter(col(time_col).lt_eq(lit(end))).collect()?; Ok(df) diff --git a/pywr-schema/src/timeseries/mod.rs b/pywr-schema/src/timeseries/mod.rs index 1643b858..f206fdfa 100644 --- a/pywr-schema/src/timeseries/mod.rs +++ b/pywr-schema/src/timeseries/mod.rs @@ -2,17 +2,39 @@ mod align_and_resample; mod polars_dataset; use ndarray::Array2; +use polars::error::PolarsError; use polars::prelude::DataType::Float64; use polars::prelude::{DataFrame, Float64Type, IndexOrder}; use pywr_core::models::ModelDomain; use pywr_core::parameters::{Array1Parameter, Array2Parameter, ParameterIndex}; use pywr_core::PywrError; use std::{collections::HashMap, path::Path}; +use thiserror::Error; -use crate::{parameters::ParameterMeta, SchemaError}; +use crate::parameters::ParameterMeta; use self::polars_dataset::PolarsDataset; +#[derive(Error, Debug)] +pub enum TimeseriesError { + #[error("Timeseries '{0} not found")] + TimeseriesNotFound(String), + #[error("The duration of timeseries '{0}' could not be determined.")] + TimeseriesDurationNotFound(String), + #[error("Column '{col}' not found in timeseries input '{name}'")] + ColumnNotFound { col: String, name: String }, + #[error("Timeseries provider '{provider}' does not support '{fmt}' file types")] + TimeseriesUnsupportedFileFormat { provider: String, fmt: String }, + #[error("Timeseries provider '{provider}' cannot parse file: '{path}'")] + TimeseriesUnparsableFileFormat { provider: String, path: String }, + #[error("A scenario group with name '{0}' was not found")] + ScenarioGroupNotFound(String), + #[error("Polars error: {0}")] + PolarsError(#[from] PolarsError), + #[error("Pywr core error: {0}")] + PywrCore(#[from] pywr_core::PywrError), +} + #[derive(serde::Deserialize, serde::Serialize, Debug, Clone)] #[serde(tag = "type")] enum TimeseriesProvider { @@ -28,7 +50,7 @@ pub struct Timeseries { } impl Timeseries { - pub fn load(&self, domain: &ModelDomain, data_path: Option<&Path>) -> Result { + pub fn load(&self, domain: &ModelDomain, data_path: Option<&Path>) -> Result { match &self.provider { TimeseriesProvider::Polars(dataset) => dataset.load(self.meta.name.as_str(), data_path, domain), TimeseriesProvider::Pandas => todo!(), @@ -45,7 +67,7 @@ impl LoadedTimeseriesCollection { timeseries_defs: Option<&[Timeseries]>, domain: &ModelDomain, data_path: Option<&Path>, - ) -> Result { + ) -> Result { let mut timeseries = HashMap::new(); if let Some(timeseries_defs) = timeseries_defs { for ts in timeseries_defs { @@ -62,11 +84,11 @@ impl LoadedTimeseriesCollection { network: &mut pywr_core::network::Network, name: &str, col: &str, - ) -> Result { + ) -> Result { let df = self .timeseries .get(name) - .ok_or(SchemaError::TimeseriesNotFound(name.to_string()))?; + .ok_or(TimeseriesError::TimeseriesNotFound(name.to_string()))?; let series = df.column(col)?; let array = series.cast(&Float64)?.f64()?.to_ndarray()?.to_owned(); @@ -79,7 +101,7 @@ impl LoadedTimeseriesCollection { let p = Array1Parameter::new(&name, array, None); Ok(network.add_parameter(Box::new(p))?) } - _ => Err(SchemaError::PywrCore(e)), + _ => Err(TimeseriesError::PywrCore(e)), }, } } @@ -90,16 +112,16 @@ impl LoadedTimeseriesCollection { name: &str, domain: &ModelDomain, scenario: &str, - ) -> Result { + ) -> Result { let scenario_group_index = domain .scenarios() .group_index(scenario) - .ok_or(SchemaError::ScenarioGroupNotFound(scenario.to_string()))?; + .ok_or(TimeseriesError::ScenarioGroupNotFound(scenario.to_string()))?; let df = self .timeseries .get(name) - .ok_or(SchemaError::TimeseriesNotFound(name.to_string()))?; + .ok_or(TimeseriesError::TimeseriesNotFound(name.to_string()))?; let array: Array2 = df.to_ndarray::(IndexOrder::default()).unwrap(); let name = format!("timeseries.{}_{}", name, scenario); @@ -111,7 +133,7 @@ impl LoadedTimeseriesCollection { let p = Array2Parameter::new(&name, array, scenario_group_index, None); Ok(network.add_parameter(Box::new(p))?) } - _ => Err(SchemaError::PywrCore(e)), + _ => Err(TimeseriesError::PywrCore(e)), }, } } diff --git a/pywr-schema/src/timeseries/polars_dataset.rs b/pywr-schema/src/timeseries/polars_dataset.rs index c8d624a1..1ed9ba56 100644 --- a/pywr-schema/src/timeseries/polars_dataset.rs +++ b/pywr-schema/src/timeseries/polars_dataset.rs @@ -3,7 +3,7 @@ use std::path::{Path, PathBuf}; use polars::{frame::DataFrame, prelude::*}; use pywr_core::models::ModelDomain; -use crate::SchemaError; +use crate::timeseries::TimeseriesError; use super::align_and_resample::align_and_resample; @@ -14,7 +14,12 @@ pub struct PolarsDataset { } impl PolarsDataset { - pub fn load(&self, name: &str, data_path: Option<&Path>, domain: &ModelDomain) -> Result { + pub fn load( + &self, + name: &str, + data_path: Option<&Path>, + domain: &ModelDomain, + ) -> Result { let fp = if self.url.is_absolute() { self.url.clone() } else if let Some(data_path) = data_path { @@ -36,13 +41,13 @@ impl PolarsDataset { todo!() } Some(other_ext) => { - return Err(SchemaError::TimeseriesUnsupportedFileFormat { + return Err(TimeseriesError::TimeseriesUnsupportedFileFormat { provider: "polars".to_string(), fmt: other_ext.to_string(), }) } None => { - return Err(SchemaError::TimeseriesUnparsableFileFormat { + return Err(TimeseriesError::TimeseriesUnparsableFileFormat { provider: "polars".to_string(), path: self.url.to_string_lossy().to_string(), }) @@ -50,7 +55,7 @@ impl PolarsDataset { } } None => { - return Err(SchemaError::TimeseriesUnparsableFileFormat { + return Err(TimeseriesError::TimeseriesUnparsableFileFormat { provider: "polars".to_string(), path: self.url.to_string_lossy().to_string(), }) From 53209838e6e3c9f047ebc426d1ba5585186d4547 Mon Sep 17 00:00:00 2001 From: James Batchelor Date: Thu, 7 Mar 2024 00:10:15 +0000 Subject: [PATCH 05/12] wip: conversion of v1 dataframe parameters to timeseries inputs --- pywr-cli/src/main.rs | 1 + pywr-schema/src/error.rs | 4 + pywr-schema/src/model.rs | 75 +++++- .../src/nodes/annual_virtual_storage.rs | 4 +- pywr-schema/src/nodes/core.rs | 29 ++- pywr-schema/src/nodes/delay.rs | 2 +- pywr-schema/src/nodes/loss_link.rs | 2 +- pywr-schema/src/nodes/mod.rs | 52 +++- .../src/nodes/monthly_virtual_storage.rs | 4 +- pywr-schema/src/nodes/piecewise_link.rs | 4 +- pywr-schema/src/nodes/piecewise_storage.rs | 4 +- pywr-schema/src/nodes/river.rs | 2 +- pywr-schema/src/nodes/river_gauge.rs | 2 +- .../src/nodes/river_split_with_gauge.rs | 2 +- .../src/nodes/rolling_virtual_storage.rs | 4 +- pywr-schema/src/nodes/virtual_storage.rs | 2 +- .../src/nodes/water_treatment_works.rs | 2 +- pywr-schema/src/parameters/mod.rs | 229 ++++++++++++++---- .../src/timeseries/align_and_resample.rs | 1 + pywr-schema/src/timeseries/mod.rs | 55 ++++- pywr-schema/src/timeseries/polars_dataset.rs | 15 +- 21 files changed, 414 insertions(+), 81 deletions(-) diff --git a/pywr-cli/src/main.rs b/pywr-cli/src/main.rs index f49c3124..80c91c86 100644 --- a/pywr-cli/src/main.rs +++ b/pywr-cli/src/main.rs @@ -179,6 +179,7 @@ fn v1_to_v2(path: &Path) -> std::result::Result<(), ConversionError> { let data = std::fs::read_to_string(path).unwrap(); let schema: pywr_v1_schema::PywrModel = serde_json::from_str(data.as_str()).unwrap(); + let schema_v2: PywrModel = schema.try_into()?; // There must be a better way to do this!! diff --git a/pywr-schema/src/error.rs b/pywr-schema/src/error.rs index 49942886..06c146ab 100644 --- a/pywr-schema/src/error.rs +++ b/pywr-schema/src/error.rs @@ -107,4 +107,8 @@ pub enum ConversionError { UnparseableDate(String), #[error("Chrono out of range error: {0}")] OutOfRange(#[from] chrono::OutOfRange), + #[error("The dataframe parameters '{0}' defines both a column and a scenario attribute. Only 1 is allowed.")] + AmbiguousColumnAndScenario(String), + #[error("The dataframe parameters '{0}' defines both a column and a scenario. Only 1 is allowed.")] + MissingColumnOrScenario(String), } diff --git a/pywr-schema/src/model.rs b/pywr-schema/src/model.rs index a2bd881a..dd12d6f5 100644 --- a/pywr-schema/src/model.rs +++ b/pywr-schema/src/model.rs @@ -4,13 +4,21 @@ use super::parameters::Parameter; use crate::data_tables::{DataTable, LoadedTableCollection}; use crate::error::{ConversionError, SchemaError}; use crate::metric_sets::MetricSet; +use crate::nodes::NodeAndTimeseries; use crate::outputs::Output; -use crate::parameters::{MetricFloatReference, TryIntoV2Parameter}; -use crate::timeseries::{LoadedTimeseriesCollection, Timeseries}; +use crate::parameters::{ + convert_parameter_v1_to_v2, DataFrameColumns, DynamicFloatValue, MetricFloatReference, MetricFloatValue, + TimeseriesReference, TryIntoV2Parameter, +}; +use crate::timeseries::{convert_from_v1_data, LoadedTimeseriesCollection, Timeseries}; use chrono::{NaiveDate, NaiveDateTime, NaiveTime}; +use polars::frame::DataFrame; use pywr_core::models::ModelDomain; use pywr_core::timestep::TimestepDuration; use pywr_core::PywrError; +use serde::de; +use std::collections::HashSet; +use std::hash::Hash; use std::path::{Path, PathBuf}; use std::str::FromStr; @@ -405,29 +413,74 @@ impl TryFrom for PywrModel { let metadata = v1.metadata.try_into()?; let timestepper = v1.timestepper.try_into()?; - let nodes = v1 + let nodes_and_ts: Vec = v1 .nodes + .clone() .into_iter() .map(|n| n.try_into()) .collect::, _>>()?; + let mut ts_data = nodes_and_ts + .iter() + .filter_map(|n| n.timeseries.clone()) + .flatten() + .collect::>(); + + let mut nodes = nodes_and_ts.into_iter().map(|n| n.node).collect::>(); + let edges = v1.edges.into_iter().map(|e| e.into()).collect(); - let parameters = if let Some(v1_parameters) = v1.parameters { + let (parameters, param_ts_data) = if let Some(v1_parameters) = v1.parameters { let mut unnamed_count: usize = 0; - Some( - v1_parameters - .into_iter() - .map(|p| p.try_into_v2_parameter(None, &mut unnamed_count)) - .collect::, _>>()?, - ) + let (parameters, param_ts_data) = convert_parameter_v1_to_v2(v1_parameters, &mut unnamed_count)?; + (Some(parameters), Some(param_ts_data)) + } else { + (None, None) + }; + + ts_data.extend(param_ts_data.into_iter().flatten()); + + let timeseries = if !ts_data.is_empty() { + let ts = convert_from_v1_data(&ts_data, &v1.tables); + let ts_names: HashSet<&str> = ts.iter().map(|t| t.name()).collect(); + + // Update node param references + for n in nodes.iter_mut() { + let mut params = n.parameters_mut(); + for param in params.values_mut() { + if let DynamicFloatValue::Dynamic(MetricFloatValue::Reference(MetricFloatReference::Parameter { + name, + key: _, + })) = param + { + // If the parameter name matches one of the timeseries names, assume parameter reference needs updating to a timeseries reference. + // This should be fine as the sources of all the names is the v1 parameter list which should have unique names. + if ts_names.contains(name.as_str()) { + if let Some(data) = ts_data.iter().find(|t| t.name.as_ref().unwrap() == name) { + let col = match (&data.column, &data.scenario) { + (Some(col), None) => DataFrameColumns::Column(col.clone()), + (None, Some(scenario)) => DataFrameColumns::Scenario(scenario.clone()), + (Some(_), Some(_)) => { + return Err(ConversionError::AmbiguousColumnAndScenario(name.clone())) + } + (None, None) => return Err(ConversionError::MissingColumnOrScenario(name.clone())), + }; + let ts_ref = DynamicFloatValue::Dynamic(MetricFloatValue::Timeseries( + TimeseriesReference::new(name.clone(), col), + )); + **param = ts_ref; + } + } + } + } + } + Some(ts) } else { None }; // TODO convert v1 tables! let tables = None; - let timeseries = None; let outputs = None; let metric_sets = None; let network = PywrNetwork { diff --git a/pywr-schema/src/nodes/annual_virtual_storage.rs b/pywr-schema/src/nodes/annual_virtual_storage.rs index 63c4fc68..1c8df0e7 100644 --- a/pywr-schema/src/nodes/annual_virtual_storage.rs +++ b/pywr-schema/src/nodes/annual_virtual_storage.rs @@ -13,7 +13,7 @@ use pywr_core::virtual_storage::VirtualStorageReset; use pywr_v1_schema::nodes::AnnualVirtualStorageNode as AnnualVirtualStorageNodeV1; use std::path::Path; -#[derive(serde::Deserialize, serde::Serialize, Clone)] +#[derive(serde::Deserialize, serde::Serialize, Clone, Debug)] pub struct AnnualReset { pub day: u8, pub month: chrono::Month, @@ -30,7 +30,7 @@ impl Default for AnnualReset { } } -#[derive(serde::Deserialize, serde::Serialize, Clone, Default)] +#[derive(serde::Deserialize, serde::Serialize, Clone, Default, Debug)] pub struct AnnualVirtualStorageNode { #[serde(flatten)] pub meta: NodeMeta, diff --git a/pywr-schema/src/nodes/core.rs b/pywr-schema/src/nodes/core.rs index a4843a55..869301e6 100644 --- a/pywr-schema/src/nodes/core.rs +++ b/pywr-schema/src/nodes/core.rs @@ -16,7 +16,7 @@ use pywr_v1_schema::nodes::{ use std::collections::HashMap; use std::path::Path; -#[derive(serde::Deserialize, serde::Serialize, Clone, Default)] +#[derive(serde::Deserialize, serde::Serialize, Clone, Default, Debug)] pub struct InputNode { #[serde(flatten)] pub meta: NodeMeta, @@ -43,6 +43,21 @@ impl InputNode { attributes } + pub fn parameters_mut(&mut self) -> HashMap<&str, &mut DynamicFloatValue> { + let mut attributes = HashMap::new(); + if let Some(p) = &mut self.max_flow { + attributes.insert("max_flow", p); + } + if let Some(p) = &mut self.min_flow { + attributes.insert("min_flow", p); + } + if let Some(p) = &mut self.cost { + attributes.insert("cost", p); + } + + attributes + } + pub fn add_to_model(&self, network: &mut pywr_core::network::Network) -> Result<(), SchemaError> { network.add_input_node(self.meta.name.as_str(), None)?; Ok(()) @@ -163,7 +178,7 @@ impl TryFrom for InputNode { } } -#[derive(serde::Deserialize, serde::Serialize, Clone, Default)] +#[derive(serde::Deserialize, serde::Serialize, Clone, Default, Debug)] pub struct LinkNode { #[serde(flatten)] pub meta: NodeMeta, @@ -310,7 +325,7 @@ impl TryFrom for LinkNode { } } -#[derive(serde::Deserialize, serde::Serialize, Clone, Default)] +#[derive(serde::Deserialize, serde::Serialize, Clone, Default, Debug)] pub struct OutputNode { #[serde(flatten)] pub meta: NodeMeta, @@ -732,7 +747,7 @@ impl TryFrom for StorageNode { /// ``` /// )] -#[derive(serde::Deserialize, serde::Serialize, Clone, Default)] +#[derive(serde::Deserialize, serde::Serialize, Clone, Default, Debug)] pub struct CatchmentNode { #[serde(flatten)] pub meta: NodeMeta, @@ -842,14 +857,14 @@ impl TryFrom for CatchmentNode { } } -#[derive(serde::Deserialize, serde::Serialize, Clone)] +#[derive(serde::Deserialize, serde::Serialize, Clone, Debug)] #[serde(tag = "type")] pub enum Factors { Proportion { factors: Vec }, Ratio { factors: Vec }, } -#[derive(serde::Deserialize, serde::Serialize, Clone, Default)] +#[derive(serde::Deserialize, serde::Serialize, Clone, Default, Debug)] pub struct AggregatedNode { #[serde(flatten)] pub meta: NodeMeta, @@ -1028,7 +1043,7 @@ impl TryFrom for AggregatedNode { } } -#[derive(serde::Deserialize, serde::Serialize, Clone, Default)] +#[derive(serde::Deserialize, serde::Serialize, Clone, Default, Debug)] pub struct AggregatedStorageNode { #[serde(flatten)] pub meta: NodeMeta, diff --git a/pywr-schema/src/nodes/delay.rs b/pywr-schema/src/nodes/delay.rs index 78335ed3..e70efc2e 100644 --- a/pywr-schema/src/nodes/delay.rs +++ b/pywr-schema/src/nodes/delay.rs @@ -24,7 +24,7 @@ use pywr_v1_schema::nodes::DelayNode as DelayNodeV1; /// ``` /// )] -#[derive(serde::Deserialize, serde::Serialize, Clone, Default)] +#[derive(serde::Deserialize, serde::Serialize, Clone, Default, Debug)] pub struct DelayNode { #[serde(flatten)] pub meta: NodeMeta, diff --git a/pywr-schema/src/nodes/loss_link.rs b/pywr-schema/src/nodes/loss_link.rs index 2c11ce5b..10e95af5 100644 --- a/pywr-schema/src/nodes/loss_link.rs +++ b/pywr-schema/src/nodes/loss_link.rs @@ -26,7 +26,7 @@ use std::path::Path; /// ``` /// )] -#[derive(serde::Deserialize, serde::Serialize, Clone, Default)] +#[derive(serde::Deserialize, serde::Serialize, Clone, Default, Debug)] pub struct LossLinkNode { #[serde(flatten)] pub meta: NodeMeta, diff --git a/pywr-schema/src/nodes/mod.rs b/pywr-schema/src/nodes/mod.rs index d9908235..7d4ee00e 100644 --- a/pywr-schema/src/nodes/mod.rs +++ b/pywr-schema/src/nodes/mod.rs @@ -21,7 +21,7 @@ pub use crate::nodes::core::{ pub use crate::nodes::delay::DelayNode; pub use crate::nodes::river::RiverNode; use crate::nodes::rolling_virtual_storage::RollingVirtualStorageNode; -use crate::parameters::DynamicFloatValue; +use crate::parameters::{DynamicFloatValue, TimeseriesV1Data}; use crate::timeseries::LoadedTimeseriesCollection; pub use annual_virtual_storage::AnnualVirtualStorageNode; pub use loss_link::LossLinkNode; @@ -33,6 +33,9 @@ use pywr_core::models::ModelDomain; use pywr_v1_schema::nodes::{ CoreNode as CoreNodeV1, Node as NodeV1, NodeMeta as NodeMetaV1, NodePosition as NodePositionV1, }; +use pywr_v1_schema::parameters::{ + CoreParameter as CoreParameterV1, Parameter as ParameterV1, ParameterValue as ParameterValueV1, +}; pub use river_gauge::RiverGaugeNode; pub use river_split_with_gauge::RiverSplitWithGaugeNode; use std::collections::HashMap; @@ -220,7 +223,7 @@ impl NodeBuilder { } } -#[derive(serde::Deserialize, serde::Serialize, Clone, EnumDiscriminants)] +#[derive(serde::Deserialize, serde::Serialize, Clone, EnumDiscriminants, Debug)] #[serde(tag = "type")] #[strum_discriminants(derive(Display, IntoStaticStr, EnumString, VariantNames))] // This creates a separate enum called `NodeType` that is available in this module. @@ -295,6 +298,13 @@ impl Node { } } + pub fn parameters_mut(&mut self) -> HashMap<&str, &mut DynamicFloatValue> { + match self { + Node::Input(n) => n.parameters_mut(), + _ => HashMap::new(), // TODO complete + } + } + pub fn add_to_model( &self, network: &mut pywr_core::network::Network, @@ -634,3 +644,41 @@ impl TryFrom> for Node { Ok(n) } } + +#[derive(Debug)] +pub struct NodeAndTimeseries { + pub node: Node, + pub timeseries: Option>, +} + +impl TryFrom for NodeAndTimeseries { + type Error = ConversionError; + + fn try_from(v1: NodeV1) -> Result { + let mut ts_vec = Vec::new(); + for param_value in v1.parameters().values() { + match param_value { + pywr_v1_schema::parameters::ParameterValueType::Single(param) => { + if let ParameterValueV1::Inline(p) = param { + if let ParameterV1::Core(CoreParameterV1::DataFrame(df_param)) = p.as_ref() { + let mut ts_data: TimeseriesV1Data = df_param.clone().into(); + + if ts_data.name.is_none() { + let name = format!("{}.timeseries", v1.name()); + ts_data.name = Some(name); + } + + ts_vec.push(ts_data); + } + } + } + pywr_v1_schema::parameters::ParameterValueType::List(_) => todo!(), + } + } + + let timeseries = if ts_vec.is_empty() { None } else { Some(ts_vec) }; + + let node = Node::try_from(v1)?; + Ok(Self { node, timeseries }) + } +} diff --git a/pywr-schema/src/nodes/monthly_virtual_storage.rs b/pywr-schema/src/nodes/monthly_virtual_storage.rs index 08aa889e..37aad7fb 100644 --- a/pywr-schema/src/nodes/monthly_virtual_storage.rs +++ b/pywr-schema/src/nodes/monthly_virtual_storage.rs @@ -13,7 +13,7 @@ use pywr_core::virtual_storage::VirtualStorageReset; use pywr_v1_schema::nodes::MonthlyVirtualStorageNode as MonthlyVirtualStorageNodeV1; use std::path::Path; -#[derive(serde::Deserialize, serde::Serialize, Clone)] +#[derive(serde::Deserialize, serde::Serialize, Clone, Debug)] pub struct NumberOfMonthsReset { pub months: u8, } @@ -24,7 +24,7 @@ impl Default for NumberOfMonthsReset { } } -#[derive(serde::Deserialize, serde::Serialize, Clone, Default)] +#[derive(serde::Deserialize, serde::Serialize, Clone, Default, Debug)] pub struct MonthlyVirtualStorageNode { #[serde(flatten)] pub meta: NodeMeta, diff --git a/pywr-schema/src/nodes/piecewise_link.rs b/pywr-schema/src/nodes/piecewise_link.rs index ea244fd9..04ee37ca 100644 --- a/pywr-schema/src/nodes/piecewise_link.rs +++ b/pywr-schema/src/nodes/piecewise_link.rs @@ -9,7 +9,7 @@ use pywr_core::models::ModelDomain; use pywr_v1_schema::nodes::PiecewiseLinkNode as PiecewiseLinkNodeV1; use std::path::Path; -#[derive(serde::Deserialize, serde::Serialize, Clone)] +#[derive(serde::Deserialize, serde::Serialize, Clone, Debug)] pub struct PiecewiseLinkStep { pub max_flow: Option, pub min_flow: Option, @@ -38,7 +38,7 @@ pub struct PiecewiseLinkStep { /// ``` /// )] -#[derive(serde::Deserialize, serde::Serialize, Clone, Default)] +#[derive(serde::Deserialize, serde::Serialize, Clone, Default, Debug)] pub struct PiecewiseLinkNode { #[serde(flatten)] pub meta: NodeMeta, diff --git a/pywr-schema/src/nodes/piecewise_storage.rs b/pywr-schema/src/nodes/piecewise_storage.rs index c1662382..c44c3589 100644 --- a/pywr-schema/src/nodes/piecewise_storage.rs +++ b/pywr-schema/src/nodes/piecewise_storage.rs @@ -11,7 +11,7 @@ use pywr_core::node::{ConstraintValue, StorageInitialVolume}; use pywr_core::parameters::VolumeBetweenControlCurvesParameter; use std::path::Path; -#[derive(serde::Deserialize, serde::Serialize, Clone)] +#[derive(serde::Deserialize, serde::Serialize, Clone, Debug)] pub struct PiecewiseStore { pub control_curve: DynamicFloatValue, pub cost: Option, @@ -43,7 +43,7 @@ pub struct PiecewiseStore { /// ``` /// )] -#[derive(serde::Deserialize, serde::Serialize, Clone, Default)] +#[derive(serde::Deserialize, serde::Serialize, Clone, Default, Debug)] pub struct PiecewiseStorageNode { #[serde(flatten)] pub meta: NodeMeta, diff --git a/pywr-schema/src/nodes/river.rs b/pywr-schema/src/nodes/river.rs index 55587875..d47655fa 100644 --- a/pywr-schema/src/nodes/river.rs +++ b/pywr-schema/src/nodes/river.rs @@ -5,7 +5,7 @@ use pywr_core::metric::Metric; use pywr_v1_schema::nodes::LinkNode as LinkNodeV1; use std::collections::HashMap; -#[derive(serde::Deserialize, serde::Serialize, Clone, Default)] +#[derive(serde::Deserialize, serde::Serialize, Clone, Default, Debug)] pub struct RiverNode { #[serde(flatten)] pub meta: NodeMeta, diff --git a/pywr-schema/src/nodes/river_gauge.rs b/pywr-schema/src/nodes/river_gauge.rs index f4aee624..9a13118d 100644 --- a/pywr-schema/src/nodes/river_gauge.rs +++ b/pywr-schema/src/nodes/river_gauge.rs @@ -24,7 +24,7 @@ use std::path::Path; /// ``` /// )] -#[derive(serde::Deserialize, serde::Serialize, Clone, Default)] +#[derive(serde::Deserialize, serde::Serialize, Clone, Default, Debug)] pub struct RiverGaugeNode { #[serde(flatten)] pub meta: NodeMeta, diff --git a/pywr-schema/src/nodes/river_split_with_gauge.rs b/pywr-schema/src/nodes/river_split_with_gauge.rs index 26fd9aaf..9101055c 100644 --- a/pywr-schema/src/nodes/river_split_with_gauge.rs +++ b/pywr-schema/src/nodes/river_split_with_gauge.rs @@ -33,7 +33,7 @@ use std::path::Path; /// ``` /// )] -#[derive(serde::Deserialize, serde::Serialize, Clone, Default)] +#[derive(serde::Deserialize, serde::Serialize, Clone, Default, Debug)] pub struct RiverSplitWithGaugeNode { #[serde(flatten)] pub meta: NodeMeta, diff --git a/pywr-schema/src/nodes/rolling_virtual_storage.rs b/pywr-schema/src/nodes/rolling_virtual_storage.rs index 889bedfa..443c8907 100644 --- a/pywr-schema/src/nodes/rolling_virtual_storage.rs +++ b/pywr-schema/src/nodes/rolling_virtual_storage.rs @@ -17,7 +17,7 @@ use std::path::Path; /// The length of the rolling window. /// /// This can be specified in either days or time-steps. -#[derive(serde::Deserialize, serde::Serialize, Clone)] +#[derive(serde::Deserialize, serde::Serialize, Clone, Debug)] pub enum RollingWindow { Days(NonZeroUsize), Timesteps(NonZeroUsize), @@ -62,7 +62,7 @@ impl RollingWindow { /// The rolling virtual storage node is useful for representing rolling licences. For example, a 30-day or 90-day /// licence on a water abstraction. /// -#[derive(serde::Deserialize, serde::Serialize, Clone, Default)] +#[derive(serde::Deserialize, serde::Serialize, Clone, Default, Debug)] pub struct RollingVirtualStorageNode { #[serde(flatten)] pub meta: NodeMeta, diff --git a/pywr-schema/src/nodes/virtual_storage.rs b/pywr-schema/src/nodes/virtual_storage.rs index e354fe62..2837ab0f 100644 --- a/pywr-schema/src/nodes/virtual_storage.rs +++ b/pywr-schema/src/nodes/virtual_storage.rs @@ -13,7 +13,7 @@ use pywr_core::virtual_storage::VirtualStorageReset; use pywr_v1_schema::nodes::VirtualStorageNode as VirtualStorageNodeV1; use std::path::Path; -#[derive(serde::Deserialize, serde::Serialize, Clone, Default)] +#[derive(serde::Deserialize, serde::Serialize, Clone, Default, Debug)] pub struct VirtualStorageNode { #[serde(flatten)] pub meta: NodeMeta, diff --git a/pywr-schema/src/nodes/water_treatment_works.rs b/pywr-schema/src/nodes/water_treatment_works.rs index db542a72..af081802 100644 --- a/pywr-schema/src/nodes/water_treatment_works.rs +++ b/pywr-schema/src/nodes/water_treatment_works.rs @@ -37,7 +37,7 @@ use std::path::Path; /// ``` /// )] -#[derive(serde::Deserialize, serde::Serialize, Clone, Default)] +#[derive(serde::Deserialize, serde::Serialize, Clone, Default, Debug)] pub struct WaterTreatmentWorks { /// Node metadata #[serde(flatten)] diff --git a/pywr-schema/src/parameters/mod.rs b/pywr-schema/src/parameters/mod.rs index 27e7cd8a..d34b63ca 100644 --- a/pywr-schema/src/parameters/mod.rs +++ b/pywr-schema/src/parameters/mod.rs @@ -50,14 +50,15 @@ use crate::nodes::NodeAttribute; use crate::parameters::core::DivisionParameter; pub use crate::parameters::data_frame::{DataFrameColumns, DataFrameParameter}; use crate::parameters::interpolated::InterpolatedParameter; -use crate::timeseries::{self, LoadedTimeseriesCollection}; +use crate::timeseries::LoadedTimeseriesCollection; pub use offset::OffsetParameter; use pywr_core::metric::Metric; use pywr_core::models::{ModelDomain, MultiNetworkTransferIndex}; use pywr_core::parameters::{IndexParameterIndex, IndexValue, ParameterType}; use pywr_v1_schema::parameters::{ - CoreParameter, ExternalDataRef as ExternalDataRefV1, Parameter as ParameterV1, ParameterMeta as ParameterMetaV1, - ParameterValue as ParameterValueV1, TableIndex as TableIndexV1, TableIndexEntry as TableIndexEntryV1, + CoreParameter, DataFrameParameter as DataFrameParameterV1, ExternalDataRef as ExternalDataRefV1, + Parameter as ParameterV1, ParameterMeta as ParameterMetaV1, ParameterValue as ParameterValueV1, ParameterVec, + TableIndex as TableIndexV1, TableIndexEntry as TableIndexEntryV1, }; use std::path::{Path, PathBuf}; @@ -425,7 +426,89 @@ impl Parameter { } } -impl TryFromV1Parameter for Parameter { +pub fn convert_parameter_v1_to_v2( + v1_parameters: ParameterVec, + unnamed_count: &mut usize, +) -> Result<(Vec, Vec), ConversionError> { + let param_or_ts: Vec = v1_parameters + .into_iter() + .map(|p| p.try_into_v2_parameter(None, unnamed_count)) + .collect::, _>>()?; + + let parameters = param_or_ts + .clone() + .into_iter() + .filter_map(|pot| match pot { + ParameterOrTimeseries::Parameter(p) => Some(p), + ParameterOrTimeseries::Timeseries(_) => None, + }) + .collect(); + + let timeseries = param_or_ts + .into_iter() + .filter_map(|pot| match pot { + ParameterOrTimeseries::Parameter(_) => None, + ParameterOrTimeseries::Timeseries(t) => Some(t), + }) + .collect(); + + Ok((parameters, timeseries)) +} + +#[derive(Clone)] +enum ParameterOrTimeseries { + Parameter(Parameter), + Timeseries(TimeseriesV1Data), +} + +#[derive(Clone, Debug)] +pub struct TimeseriesV1Data { + pub name: Option, + pub source: TimeseriesV1Source, + pub column: Option, + pub scenario: Option, +} + +impl From for TimeseriesV1Data { + fn from(p: DataFrameParameterV1) -> Self { + let source = if let Some(url) = p.url { + TimeseriesV1Source::Url(url) + } else if let Some(tbl) = p.table { + TimeseriesV1Source::Table(tbl) + } else { + panic!("DataFrameParameter must have a url or table attribute.") + }; + + let name = p.meta.and_then(|m| m.name); + + Self { + name, + source, + column: p.column, + scenario: p.scenario, + } + } +} + +#[derive(Clone, Debug)] +pub enum TimeseriesV1Source { + Url(PathBuf), + Table(String), +} + +impl From for ParameterOrTimeseries { + fn from(p: Parameter) -> Self { + Self::Parameter(p) + } +} + +impl From for ParameterOrTimeseries { + fn from(t: TimeseriesV1Data) -> Self { + Self::Timeseries(t) + } +} + +impl TryFromV1Parameter for ParameterOrTimeseries { type Error = ConversionError; fn try_from_v1_parameter( @@ -433,58 +516,66 @@ impl TryFromV1Parameter for Parameter { parent_node: Option<&str>, unnamed_count: &mut usize, ) -> Result { - let p = match v1 { + let p: ParameterOrTimeseries = match v1 { ParameterV1::Core(v1) => match v1 { CoreParameter::Aggregated(p) => { - Parameter::Aggregated(p.try_into_v2_parameter(parent_node, unnamed_count)?) + Parameter::Aggregated(p.try_into_v2_parameter(parent_node, unnamed_count)?).into() } CoreParameter::AggregatedIndex(p) => { - Parameter::AggregatedIndex(p.try_into_v2_parameter(parent_node, unnamed_count)?) + Parameter::AggregatedIndex(p.try_into_v2_parameter(parent_node, unnamed_count)?).into() } CoreParameter::AsymmetricSwitchIndex(p) => { - Parameter::AsymmetricSwitchIndex(p.try_into_v2_parameter(parent_node, unnamed_count)?) + Parameter::AsymmetricSwitchIndex(p.try_into_v2_parameter(parent_node, unnamed_count)?).into() + } + CoreParameter::Constant(p) => { + Parameter::Constant(p.try_into_v2_parameter(parent_node, unnamed_count)?).into() } - CoreParameter::Constant(p) => Parameter::Constant(p.try_into_v2_parameter(parent_node, unnamed_count)?), CoreParameter::ControlCurvePiecewiseInterpolated(p) => { Parameter::ControlCurvePiecewiseInterpolated(p.try_into_v2_parameter(parent_node, unnamed_count)?) + .into() } CoreParameter::ControlCurveInterpolated(p) => { - Parameter::ControlCurveInterpolated(p.try_into_v2_parameter(parent_node, unnamed_count)?) + Parameter::ControlCurveInterpolated(p.try_into_v2_parameter(parent_node, unnamed_count)?).into() } CoreParameter::ControlCurveIndex(p) => { - Parameter::ControlCurveIndex(p.try_into_v2_parameter(parent_node, unnamed_count)?) + Parameter::ControlCurveIndex(p.try_into_v2_parameter(parent_node, unnamed_count)?).into() } CoreParameter::ControlCurve(p) => match p.clone().try_into_v2_parameter(parent_node, unnamed_count) { - Ok(p) => Parameter::ControlCurve(p), - Err(_) => Parameter::ControlCurveIndex(p.try_into_v2_parameter(parent_node, unnamed_count)?), + Ok(p) => Parameter::ControlCurve(p).into(), + Err(_) => Parameter::ControlCurveIndex(p.try_into_v2_parameter(parent_node, unnamed_count)?).into(), }, CoreParameter::DailyProfile(p) => { - Parameter::DailyProfile(p.try_into_v2_parameter(parent_node, unnamed_count)?) + Parameter::DailyProfile(p.try_into_v2_parameter(parent_node, unnamed_count)?).into() } CoreParameter::IndexedArray(p) => { - Parameter::IndexedArray(p.try_into_v2_parameter(parent_node, unnamed_count)?) + Parameter::IndexedArray(p.try_into_v2_parameter(parent_node, unnamed_count)?).into() } CoreParameter::MonthlyProfile(p) => { - Parameter::MonthlyProfile(p.try_into_v2_parameter(parent_node, unnamed_count)?) + Parameter::MonthlyProfile(p.try_into_v2_parameter(parent_node, unnamed_count)?).into() } CoreParameter::UniformDrawdownProfile(p) => { - Parameter::UniformDrawdownProfile(p.try_into_v2_parameter(parent_node, unnamed_count)?) + Parameter::UniformDrawdownProfile(p.try_into_v2_parameter(parent_node, unnamed_count)?).into() + } + CoreParameter::Max(p) => Parameter::Max(p.try_into_v2_parameter(parent_node, unnamed_count)?).into(), + CoreParameter::Negative(p) => { + Parameter::Negative(p.try_into_v2_parameter(parent_node, unnamed_count)?).into() } - CoreParameter::Max(p) => Parameter::Max(p.try_into_v2_parameter(parent_node, unnamed_count)?), - CoreParameter::Negative(p) => Parameter::Negative(p.try_into_v2_parameter(parent_node, unnamed_count)?), CoreParameter::Polynomial1D(p) => { - Parameter::Polynomial1D(p.try_into_v2_parameter(parent_node, unnamed_count)?) + Parameter::Polynomial1D(p.try_into_v2_parameter(parent_node, unnamed_count)?).into() } CoreParameter::ParameterThreshold(p) => { - Parameter::ParameterThreshold(p.try_into_v2_parameter(parent_node, unnamed_count)?) + Parameter::ParameterThreshold(p.try_into_v2_parameter(parent_node, unnamed_count)?).into() } CoreParameter::TablesArray(p) => { - Parameter::TablesArray(p.try_into_v2_parameter(parent_node, unnamed_count)?) + Parameter::TablesArray(p.try_into_v2_parameter(parent_node, unnamed_count)?).into() + } + CoreParameter::Min(p) => Parameter::Min(p.try_into_v2_parameter(parent_node, unnamed_count)?).into(), + CoreParameter::Division(p) => { + Parameter::Division(p.try_into_v2_parameter(parent_node, unnamed_count)?).into() } - CoreParameter::Min(p) => Parameter::Min(p.try_into_v2_parameter(parent_node, unnamed_count)?), - CoreParameter::Division(p) => Parameter::Division(p.try_into_v2_parameter(parent_node, unnamed_count)?), CoreParameter::DataFrame(p) => { - Parameter::DataFrame(p.try_into_v2_parameter(parent_node, unnamed_count)?) + let ts_data: TimeseriesV1Data = p.into(); + ts_data.into() } CoreParameter::Deficit(p) => { return Err(ConversionError::DeprecatedParameter { @@ -494,19 +585,19 @@ impl TryFromV1Parameter for Parameter { }) } CoreParameter::DiscountFactor(p) => { - Parameter::DiscountFactor(p.try_into_v2_parameter(parent_node, unnamed_count)?) + Parameter::DiscountFactor(p.try_into_v2_parameter(parent_node, unnamed_count)?).into() } CoreParameter::InterpolatedVolume(p) => { - Parameter::Interpolated(p.try_into_v2_parameter(parent_node, unnamed_count)?) + Parameter::Interpolated(p.try_into_v2_parameter(parent_node, unnamed_count)?).into() } CoreParameter::InterpolatedFlow(p) => { - Parameter::Interpolated(p.try_into_v2_parameter(parent_node, unnamed_count)?) + Parameter::Interpolated(p.try_into_v2_parameter(parent_node, unnamed_count)?).into() } CoreParameter::NegativeMax(_) => todo!("Implement NegativeMaxParameter"), CoreParameter::NegativeMin(_) => todo!("Implement NegativeMinParameter"), CoreParameter::HydropowerTarget(_) => todo!("Implement HydropowerTargetParameter"), CoreParameter::WeeklyProfile(p) => { - Parameter::WeeklyProfile(p.try_into_v2_parameter(parent_node, unnamed_count)?) + Parameter::WeeklyProfile(p.try_into_v2_parameter(parent_node, unnamed_count)?).into() } CoreParameter::Storage(p) => { return Err(ConversionError::DeprecatedParameter { @@ -525,7 +616,7 @@ impl TryFromV1Parameter for Parameter { }) } CoreParameter::RbfProfile(p) => { - Parameter::RbfProfile(p.try_into_v2_parameter(parent_node, unnamed_count)?) + Parameter::RbfProfile(p.try_into_v2_parameter(parent_node, unnamed_count)?).into() } }, ParameterV1::Custom(p) => { @@ -545,6 +636,7 @@ impl TryFromV1Parameter for Parameter { }, value: ConstantValue::Literal(0.0), }) + .into() } }; @@ -672,13 +764,28 @@ impl MetricFloatReference { } } +#[derive(serde::Deserialize, serde::Serialize, Debug, Clone)] +pub struct TimeseriesReference { + #[serde(rename = "type")] + ty: String, + name: String, + columns: DataFrameColumns, +} + +impl TimeseriesReference { + pub fn new(name: String, columns: DataFrameColumns) -> Self { + let ty = "Timeseries".to_string(); + Self { ty, name, columns } + } +} + /// A floating-point(f64) value from a metric in the network. #[derive(serde::Deserialize, serde::Serialize, Debug, Clone)] #[serde(untagged)] pub enum MetricFloatValue { Reference(MetricFloatReference), InlineParameter { definition: Box }, - Timeseries { name: String, columns: DataFrameColumns }, + Timeseries(TimeseriesReference), } impl MetricFloatValue { @@ -726,10 +833,14 @@ impl MetricFloatValue { } } } - Self::Timeseries { name, columns } => { - let param_idx = match columns { - DataFrameColumns::Scenario(scenario) => timeseries.load_df(network, name, domain, scenario)?, - DataFrameColumns::Column(col) => timeseries.load_column(network, name, col)?, + Self::Timeseries(ts_ref) => { + let param_idx = match &ts_ref.columns { + DataFrameColumns::Scenario(scenario) => { + timeseries.load_df(network, ts_ref.name.as_ref(), domain, scenario.as_str())? + } + DataFrameColumns::Column(col) => { + timeseries.load_column(network, ts_ref.name.as_ref(), col.as_str())? + } }; Ok(Metric::ParameterValue(param_idx)) } @@ -846,9 +957,38 @@ impl TryFromV1Parameter for DynamicFloatValue { })) } ParameterValueV1::Table(tbl) => Self::Constant(ConstantValue::Table(tbl.try_into()?)), - ParameterValueV1::Inline(param) => Self::Dynamic(MetricFloatValue::InlineParameter { - definition: Box::new((*param).try_into_v2_parameter(parent_node, unnamed_count)?), - }), + ParameterValueV1::Inline(param) => { + let definition: ParameterOrTimeseries = (*param).try_into_v2_parameter(parent_node, unnamed_count)?; + match definition { + ParameterOrTimeseries::Parameter(p) => Self::Dynamic(MetricFloatValue::InlineParameter { + definition: Box::new(p), + }), + ParameterOrTimeseries::Timeseries(t) => { + let name = match t.name { + Some(n) => n, + None => { + match parent_node { + // TODO if the node has inline timeseries for more than 1 attribute then this name will not be unique! + // The attribute name might need passing into the function + Some(node_name) => format!("{}.timeseries", node_name), + None => format!("unnamed-timeseries-{}", *unnamed_count), + } + } + }; + + let cols = match (&t.column, &t.scenario) { + (Some(col), None) => DataFrameColumns::Column(col.clone()), + (None, Some(scenario)) => DataFrameColumns::Scenario(scenario.clone()), + (Some(_), Some(_)) => { + return Err(ConversionError::AmbiguousColumnAndScenario(name.clone())) + } + (None, None) => return Err(ConversionError::MissingColumnOrScenario(name.clone())), + }; + + Self::Dynamic(MetricFloatValue::Timeseries(TimeseriesReference::new(name, cols))) + } + } + } }; Ok(p) } @@ -911,9 +1051,16 @@ impl TryFromV1Parameter for DynamicIndexValue { ParameterValueV1::Constant(_) => return Err(ConversionError::FloatToIndex), ParameterValueV1::Reference(p_name) => Self::Dynamic(ParameterIndexValue::Reference(p_name)), ParameterValueV1::Table(tbl) => Self::Constant(ConstantValue::Table(tbl.try_into()?)), - ParameterValueV1::Inline(param) => Self::Dynamic(ParameterIndexValue::Inline(Box::new( - (*param).try_into_v2_parameter(parent_node, unnamed_count)?, - ))), + ParameterValueV1::Inline(param) => { + let definition: ParameterOrTimeseries = (*param).try_into_v2_parameter(parent_node, unnamed_count)?; + match definition { + ParameterOrTimeseries::Parameter(p) => Self::Dynamic(ParameterIndexValue::Inline(Box::new(p))), + ParameterOrTimeseries::Timeseries(_) => { + // TODO create an error for this + panic!("Timeseries do not support indexes yet") + } + } + } }; Ok(p) } diff --git a/pywr-schema/src/timeseries/align_and_resample.rs b/pywr-schema/src/timeseries/align_and_resample.rs index b7158b5c..52dbad6e 100644 --- a/pywr-schema/src/timeseries/align_and_resample.rs +++ b/pywr-schema/src/timeseries/align_and_resample.rs @@ -1,3 +1,4 @@ +use chrono::TimeDelta; use polars::{prelude::*, series::ops::NullBehavior}; use pywr_core::models::ModelDomain; use std::{cmp::Ordering, ops::Deref}; diff --git a/pywr-schema/src/timeseries/mod.rs b/pywr-schema/src/timeseries/mod.rs index f206fdfa..a4ad47c6 100644 --- a/pywr-schema/src/timeseries/mod.rs +++ b/pywr-schema/src/timeseries/mod.rs @@ -8,10 +8,12 @@ use polars::prelude::{DataFrame, Float64Type, IndexOrder}; use pywr_core::models::ModelDomain; use pywr_core::parameters::{Array1Parameter, Array2Parameter, ParameterIndex}; use pywr_core::PywrError; +use pywr_v1_schema::tables::TableVec; +use std::sync::Arc; use std::{collections::HashMap, path::Path}; use thiserror::Error; -use crate::parameters::ParameterMeta; +use crate::parameters::{ParameterMeta, TimeseriesV1Data, TimeseriesV1Source}; use self::polars_dataset::PolarsDataset; @@ -56,8 +58,13 @@ impl Timeseries { TimeseriesProvider::Pandas => todo!(), } } + + pub fn name(&self) -> &str { + &self.meta.name + } } +#[derive(Default)] pub struct LoadedTimeseriesCollection { timeseries: HashMap, } @@ -139,6 +146,52 @@ impl LoadedTimeseriesCollection { } } +pub fn convert_from_v1_data(df_data: &[TimeseriesV1Data], v1_tables: &Option) -> Vec { + let mut ts = HashMap::new(); + for data in df_data.iter() { + match &data.source { + TimeseriesV1Source::Table(name) => { + let tables = v1_tables.as_ref().unwrap(); + let table = tables.iter().find(|t| t.name == *name).unwrap(); + let name = table.name.clone(); + if ts.contains_key(&name) { + continue; + } + + let time_col = None; + let provider = PolarsDataset::new(time_col, table.url.clone()); + + ts.insert( + name.clone(), + Timeseries { + meta: ParameterMeta { name, comment: None }, + provider: TimeseriesProvider::Polars(provider), + }, + ); + } + TimeseriesV1Source::Url(url) => { + let name = data.name.as_ref().unwrap().clone(); + if ts.contains_key(&name) { + continue; + } + + // TODO: time_col should use the index_col from the v1 data? + let time_col = None; + let provider = PolarsDataset::new(time_col, url.clone()); + + ts.insert( + name.clone(), + Timeseries { + meta: ParameterMeta { name, comment: None }, + provider: TimeseriesProvider::Polars(provider), + }, + ); + } + } + } + ts.into_values().collect::>() +} + #[cfg(test)] mod tests { use std::path::PathBuf; diff --git a/pywr-schema/src/timeseries/polars_dataset.rs b/pywr-schema/src/timeseries/polars_dataset.rs index 1ed9ba56..cf9c984f 100644 --- a/pywr-schema/src/timeseries/polars_dataset.rs +++ b/pywr-schema/src/timeseries/polars_dataset.rs @@ -9,11 +9,15 @@ use super::align_and_resample::align_and_resample; #[derive(serde::Deserialize, serde::Serialize, Debug, Clone)] pub struct PolarsDataset { - time_col: String, + time_col: Option, url: PathBuf, } impl PolarsDataset { + pub fn new(time_col: Option, url: PathBuf) -> Self { + Self { time_col, url } + } + pub fn load( &self, name: &str, @@ -62,7 +66,14 @@ impl PolarsDataset { } }; - df = align_and_resample(name, df, self.time_col.as_str(), domain)?; + df = match self.time_col { + Some(ref col) => align_and_resample(name, df, col, domain)?, + None => { + // If a time col has not been provided assume it is the first column + let first_col = df.get_column_names()[0].to_string(); + align_and_resample(name, df, first_col.as_str(), domain)? + } + }; Ok(df) } From 1300e772b10de710a9fbd5ce3ccc1f36cf72a4a2 Mon Sep 17 00:00:00 2001 From: James Batchelor Date: Thu, 7 Mar 2024 21:20:26 +0000 Subject: [PATCH 06/12] update timeseries url in test model + remove dbg statement --- pywr-schema/src/test_models/timeseries.json | 2 +- pywr-schema/src/timeseries/mod.rs | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/pywr-schema/src/test_models/timeseries.json b/pywr-schema/src/test_models/timeseries.json index 76845396..130e43c0 100644 --- a/pywr-schema/src/test_models/timeseries.json +++ b/pywr-schema/src/test_models/timeseries.json @@ -74,7 +74,7 @@ "provider": { "type": "Polars", "time_col": "date", - "url": "../test_models/inflow.csv" + "url": "inflow.csv" } } ] diff --git a/pywr-schema/src/timeseries/mod.rs b/pywr-schema/src/timeseries/mod.rs index a4ad47c6..562bfc30 100644 --- a/pywr-schema/src/timeseries/mod.rs +++ b/pywr-schema/src/timeseries/mod.rs @@ -212,8 +212,6 @@ mod tests { let model_dir = PathBuf::from(cargo_manifest_dir).join("src/test_models"); - dbg!(&model_dir); - let data = model_str(); let schema: PywrModel = serde_json::from_str(data).unwrap(); let mut model = schema.build_model(Some(model_dir.as_path()), None).unwrap(); From 6b2978045bd244ff261a1064ca02b1a8c4e5c142 Mon Sep 17 00:00:00 2001 From: James Tomlinson Date: Mon, 11 Mar 2024 13:53:56 +0000 Subject: [PATCH 07/12] feat: Implement Parameter for PyParameter. This supports using PyParameter as an index parameter with updated schema to define the return type of the Python method. --- pywr-core/src/parameters/py.rs | 142 +++++++++++------- pywr-schema-macros/src/lib.rs | 8 +- pywr-schema/src/parameters/python.rs | 88 +++++++---- .../src/test_models/test_parameters.py | 20 +++ 4 files changed, 167 insertions(+), 91 deletions(-) create mode 100644 pywr-schema/src/test_models/test_parameters.py diff --git a/pywr-core/src/parameters/py.rs b/pywr-core/src/parameters/py.rs index ba88286f..e9047aa3 100644 --- a/pywr-core/src/parameters/py.rs +++ b/pywr-core/src/parameters/py.rs @@ -67,21 +67,8 @@ impl PyParameter { Ok(index_values.into_py_dict(py)) } -} - -impl Parameter for PyParameter { - fn as_any_mut(&mut self) -> &mut dyn Any { - self - } - fn meta(&self) -> &ParameterMeta { - &self.meta - } - fn setup( - &self, - _timesteps: &[Timestep], - _scenario_index: &ScenarioIndex, - ) -> Result>, PywrError> { + fn setup(&self) -> Result>, PywrError> { pyo3::prepare_freethreaded_python(); let user_obj: PyObject = Python::with_gil(|py| -> PyResult { @@ -96,26 +83,20 @@ impl Parameter for PyParameter { Ok(Some(internal.into_boxed_any())) } - // fn before(&self, internal_state: &mut Option>) -> Result<(), PywrError> { - // let internal = downcast_internal_state::(internal_state); - // - // Python::with_gil(|py| internal.user_obj.call_method0(py, "before")) - // .map_err(|e| PywrError::PythonError(e.to_string()))?; - // - // Ok(()) - // } - - fn compute( + fn compute( &self, timestep: &Timestep, scenario_index: &ScenarioIndex, model: &Network, state: &State, internal_state: &mut Option>, - ) -> Result { + ) -> Result + where + T: for<'a> FromPyObject<'a>, + { let internal = downcast_internal_state_mut::(internal_state); - let value: f64 = Python::with_gil(|py| { + let value: T = Python::with_gil(|py| { let date = timestep.date.into_py(py); let si = scenario_index.index.into_py(py); @@ -164,11 +145,49 @@ impl Parameter for PyParameter { } } -impl Parameter for PyParameter { +impl Parameter for PyParameter { fn as_any_mut(&mut self) -> &mut dyn Any { self } + fn meta(&self) -> &ParameterMeta { + &self.meta + } + + fn setup( + &self, + _timesteps: &[Timestep], + _scenario_index: &ScenarioIndex, + ) -> Result>, PywrError> { + self.setup() + } + + fn compute( + &self, + timestep: &Timestep, + scenario_index: &ScenarioIndex, + model: &Network, + state: &State, + internal_state: &mut Option>, + ) -> Result { + self.compute(timestep, scenario_index, model, state, internal_state) + } + fn after( + &self, + timestep: &Timestep, + scenario_index: &ScenarioIndex, + model: &Network, + state: &State, + internal_state: &mut Option>, + ) -> Result<(), PywrError> { + self.after(timestep, scenario_index, model, state, internal_state) + } +} + +impl Parameter for PyParameter { + fn as_any_mut(&mut self) -> &mut dyn Any { + self + } fn meta(&self) -> &ParameterMeta { &self.meta } @@ -178,18 +197,47 @@ impl Parameter for PyParameter { _timesteps: &[Timestep], _scenario_index: &ScenarioIndex, ) -> Result>, PywrError> { - pyo3::prepare_freethreaded_python(); + self.setup() + } - let user_obj: PyObject = Python::with_gil(|py| -> PyResult { - let args = self.args.as_ref(py); - let kwargs = self.kwargs.as_ref(py); - self.object.call(py, args, Some(kwargs)) - }) - .unwrap(); + fn compute( + &self, + timestep: &Timestep, + scenario_index: &ScenarioIndex, + model: &Network, + state: &State, + internal_state: &mut Option>, + ) -> Result { + self.compute(timestep, scenario_index, model, state, internal_state) + } - let internal = Internal { user_obj }; + fn after( + &self, + timestep: &Timestep, + scenario_index: &ScenarioIndex, + model: &Network, + state: &State, + internal_state: &mut Option>, + ) -> Result<(), PywrError> { + self.after(timestep, scenario_index, model, state, internal_state) + } +} - Ok(Some(internal.into_boxed_any())) +impl Parameter for PyParameter { + fn as_any_mut(&mut self) -> &mut dyn Any { + self + } + + fn meta(&self) -> &ParameterMeta { + &self.meta + } + + fn setup( + &self, + _timesteps: &[Timestep], + _scenario_index: &ScenarioIndex, + ) -> Result>, PywrError> { + self.setup() } // fn before(&self, internal_state: &mut Option>) -> Result<(), PywrError> { @@ -265,27 +313,7 @@ impl Parameter for PyParameter { state: &State, internal_state: &mut Option>, ) -> Result<(), PywrError> { - let internal = downcast_internal_state_mut::(internal_state); - - Python::with_gil(|py| { - // Only do this if the object has an "after" method defined. - if internal.user_obj.getattr(py, "after").is_ok() { - let date = timestep.date.into_py(py); - - let si = scenario_index.index.into_py(py); - - let metric_dict = self.get_metrics_dict(model, state, py)?; - let index_dict = self.get_indices_dict(state, py)?; - - let args = PyTuple::new(py, [date.as_ref(py), si.as_ref(py), metric_dict, index_dict]); - - internal.user_obj.call_method1(py, "after", args)?; - } - Ok(()) - }) - .map_err(|e: PyErr| PywrError::PythonError(e.to_string()))?; - - Ok(()) + self.after(timestep, scenario_index, model, state, internal_state) } } diff --git a/pywr-schema-macros/src/lib.rs b/pywr-schema-macros/src/lib.rs index e5de8f38..827560d8 100644 --- a/pywr-schema-macros/src/lib.rs +++ b/pywr-schema-macros/src/lib.rs @@ -17,7 +17,7 @@ enum PywrField { /// Generates a [`TokenStream`] containing the implementation of two methods, `parameters` /// and `parameters_mut`, for the given struct. /// -/// Both method returns a [`HashMap`] of parameter names to [`DynamicFloatValue`]. This +/// Both method returns a [`HashMap`] of parameter names to [`DynamicFloatValue`]. This /// is intended to be used for nodes and parameter structs in the Pywr schema. fn impl_parameter_references_derive(ast: &syn::DeriveInput) -> TokenStream { // Name of the node type @@ -157,9 +157,7 @@ fn type_to_ident(ty: &syn::Type) -> Option { // Match on path types that are no self types. let arg_type_path = match arg_ty { Some(ty) => match ty { - syn::Type::Path(type_path) if type_path.qself.is_none() => { - Some(type_path) - } + syn::Type::Path(type_path) if type_path.qself.is_none() => Some(type_path), _ => None, }, None => None, @@ -195,4 +193,4 @@ fn is_parameter_ident(ident: &syn::Ident) -> bool { // TODO this currenty omits more complex attributes, such as `factors` for AggregatedNode // and steps for PiecewiseLinks, that can internally contain `DynamicFloatValue` fields ident == "DynamicFloatValue" -} \ No newline at end of file +} diff --git a/pywr-schema/src/parameters/python.rs b/pywr-schema/src/parameters/python.rs index 52b62f61..0a76ac97 100644 --- a/pywr-schema/src/parameters/python.rs +++ b/pywr-schema/src/parameters/python.rs @@ -18,6 +18,16 @@ pub enum PythonModule { Path(PathBuf), } +/// The expected return type of the Python parameter. +#[derive(serde::Deserialize, serde::Serialize, Debug, Clone, Default)] +#[serde(rename_all = "lowercase")] +pub enum PythonReturnType { + #[default] + Float, + Int, + Dict, +} + /// A Parameter that uses a Python object for its calculations. /// /// This struct defines a schema for loading a [`crate::parameters::PyParameter`] from external @@ -67,10 +77,10 @@ pub struct PythonParameter { pub module: PythonModule, /// The name of Python object from the module to use. pub object: String, - /// Is this a multi-valued parameter or not. If true then the calculation method should - /// return a dictionary with string keys and either floats or ints as values. + /// The return type of the Python calculation. This is used to convert the Python return value + /// to the appropriate type for the Parameter. #[serde(default)] - pub multi: bool, + pub return_type: PythonReturnType, /// Position arguments to pass to the object during setup. pub args: Vec, /// Keyword arguments to pass to the object during setup. @@ -193,10 +203,11 @@ impl PythonParameter { }; let p = PyParameter::new(&self.meta.name, object, args, kwargs, &metrics, &indices); - let pt = if self.multi { - ParameterType::Multi(network.add_multi_value_parameter(Box::new(p))?) - } else { - ParameterType::Parameter(network.add_parameter(Box::new(p))?) + + let pt = match self.return_type { + PythonReturnType::Float => ParameterType::Parameter(network.add_parameter(Box::new(p))?), + PythonReturnType::Int => ParameterType::Index(network.add_index_parameter(Box::new(p))?), + PythonReturnType::Dict => ParameterType::Multi(network.add_multi_value_parameter(Box::new(p))?), }; Ok(pt) @@ -212,42 +223,59 @@ mod tests { use pywr_core::network::Network; use pywr_core::test_utils::default_time_domain; use serde_json::json; - use std::fs::File; - use std::io::Write; - use tempfile::tempdir; + use std::path::PathBuf; #[test] - fn test_python_parameter() { - let dir = tempdir().unwrap(); - - let file_path = dir.path().join("my_parameter.py"); + fn test_python_float_parameter() { + let mut py_fn = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + py_fn.push("src/test_models/test_parameters.py"); let data = json!( { - "name": "my-custom-calculation", + "name": "my-float-parameter", "type": "Python", - "path": file_path, - "object": "MyParameter", + "path": py_fn, + "object": "FloatParameter", "args": [0, ], "kwargs": {}, } ) .to_string(); - let mut file = File::create(file_path).unwrap(); - write!( - file, - r#" -class MyParameter: - def __init__(self, count, *args, **kwargs): - self.count = 0 + // Init Python + pyo3::prepare_freethreaded_python(); + // Load the schema ... + let param: PythonParameter = serde_json::from_str(data.as_str()).unwrap(); + // ... add it to an empty network + // this should trigger loading the module and extracting the class + let domain: ModelDomain = default_time_domain().into(); + let schema = PywrNetwork::default(); + let mut network = Network::default(); + let tables = LoadedTableCollection::from_schema(None, None).unwrap(); + param + .add_to_model(&mut network, &schema, &domain, &tables, None, &[]) + .unwrap(); + + assert!(network.get_parameter_by_name("my-float-parameter").is_ok()); + } + + #[test] + fn test_python_int_parameter() { + let mut py_fn = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + py_fn.push("src/test_models/test_parameters.py"); - def calc(self, ts, si, p_values): - self.count += si - return float(self.count + ts.day) -"# + let data = json!( + { + "name": "my-int-parameter", + "type": "Python", + "path": py_fn, + "return_type": "int", + "object": "FloatParameter", + "args": [0, ], + "kwargs": {}, + } ) - .unwrap(); + .to_string(); // Init Python pyo3::prepare_freethreaded_python(); @@ -262,5 +290,7 @@ class MyParameter: param .add_to_model(&mut network, &schema, &domain, &tables, None, &[]) .unwrap(); + + assert!(network.get_index_parameter_by_name("my-int-parameter").is_ok()); } } diff --git a/pywr-schema/src/test_models/test_parameters.py b/pywr-schema/src/test_models/test_parameters.py new file mode 100644 index 00000000..47794588 --- /dev/null +++ b/pywr-schema/src/test_models/test_parameters.py @@ -0,0 +1,20 @@ +class FloatParameter: + """A simple float parameter""" + + def __init__(self, count, *args, **kwargs): + self.count = 0 + + def calc(self, ts, si, p_values) -> float: + self.count += si + return float(self.count + ts.day) + + +class IntParameter: + """A simple int parameter""" + + def __init__(self, count, *args, **kwargs): + self.count = 0 + + def calc(self, ts, si, p_values) -> int: + self.count += si + return self.count + ts.day From 483b17e0cc2e213507c449b0a5bfb57598d20688 Mon Sep 17 00:00:00 2001 From: James Tomlinson Date: Tue, 12 Mar 2024 15:26:06 +0000 Subject: [PATCH 08/12] fix: Fix and upgrade highs_sys to v1.6.2 Restores some removed functions required for the highs solver. Adds highs feature to CI. --- .github/workflows/linux.yml | 32 ++++++++++++------------- pywr-core/Cargo.toml | 9 ++++--- pywr-core/src/solvers/builder.rs | 12 ++++++++++ pywr-core/src/solvers/highs/settings.rs | 11 ++++----- 4 files changed, 36 insertions(+), 28 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 1d5bc7d9..963748d1 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -14,21 +14,21 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - with: - submodules: true + - uses: actions/checkout@v4 + with: + submodules: true - - name: Install latest mdbook - run: | - tag=$(curl 'https://api.github.com/repos/rust-lang/mdbook/releases/latest' | jq -r '.tag_name') - url="https://github.com/rust-lang/mdbook/releases/download/${tag}/mdbook-${tag}-x86_64-unknown-linux-gnu.tar.gz" - mkdir bin - curl -sSL $url | tar -xz --directory=bin - echo "$(pwd)/bin" >> $GITHUB_PATH + - name: Install latest mdbook + run: | + tag=$(curl 'https://api.github.com/repos/rust-lang/mdbook/releases/latest' | jq -r '.tag_name') + url="https://github.com/rust-lang/mdbook/releases/download/${tag}/mdbook-${tag}-x86_64-unknown-linux-gnu.tar.gz" + mkdir bin + curl -sSL $url | tar -xz --directory=bin + echo "$(pwd)/bin" >> $GITHUB_PATH - - name: Build - run: cargo build --verbose --no-default-features - - name: Run tests - run: cargo test --no-default-features - - name: Run mdbook tests - run: mdbook test ./pywr-book + - name: Build + run: cargo build --verbose --no-default-features --features highs + - name: Run tests + run: cargo test --no-default-features --features highs + - name: Run mdbook tests + run: mdbook test ./pywr-book diff --git a/pywr-core/Cargo.toml b/pywr-core/Cargo.toml index 5bd95f1c..0fdc225f 100644 --- a/pywr-core/Cargo.toml +++ b/pywr-core/Cargo.toml @@ -18,20 +18,19 @@ libc = "0.2.97" thiserror = { workspace = true } ndarray = { workspace = true } num = { workspace = true } -float-cmp = { workspace = true } +float-cmp = { workspace = true } hdf5 = { workspace = true } csv = { workspace = true } clp-sys = { path = "../clp-sys", version = "0.1.0" } ipm-ocl = { path = "../ipm-ocl", optional = true } ipm-simd = { path = "../ipm-simd", optional = true } -tracing = { workspace = true } -highs-sys = { git = "https://github.com/jetuk/highs-sys", branch="fix-build-libz-linking", optional = true } -# highs-sys = { path = "../../highs-sys" } +tracing = { workspace = true } +highs-sys = { version = "1.6.2", optional = true } nalgebra = "0.32.3" chrono = { workspace = true } polars = { workspace = true } -pyo3 = { workspace = true, features = ["chrono"] } +pyo3 = { workspace = true, features = ["chrono"] } rayon = "1.6.1" diff --git a/pywr-core/src/solvers/builder.rs b/pywr-core/src/solvers/builder.rs index e644076e..2bdc2715 100644 --- a/pywr-core/src/solvers/builder.rs +++ b/pywr-core/src/solvers/builder.rs @@ -293,6 +293,14 @@ where I::from(self.builder.col_upper.len()).unwrap() } + pub fn num_rows(&self) -> I { + I::from(self.builder.row_upper.len()).unwrap() + } + + pub fn num_non_zero(&self) -> I { + I::from(self.builder.elements.len()).unwrap() + } + pub fn col_lower(&self) -> &[f64] { &self.builder.col_lower } @@ -313,6 +321,10 @@ where &self.builder.row_upper } + pub fn row_mask(&self) -> &[I] { + &self.builder.row_mask + } + pub fn row_starts(&self) -> &[I] { &self.builder.row_starts } diff --git a/pywr-core/src/solvers/highs/settings.rs b/pywr-core/src/solvers/highs/settings.rs index 939bc26f..6f3e01aa 100644 --- a/pywr-core/src/solvers/highs/settings.rs +++ b/pywr-core/src/solvers/highs/settings.rs @@ -39,14 +39,11 @@ impl HighsSolverSettings { /// /// ``` /// use std::num::NonZeroUsize; -/// use pywr::solvers::ClpSolverSettingsBuilder; +/// use pywr_core::solvers::HighsSolverSettingsBuilder; /// // Settings with parallel enabled and 4 threads. -/// let settings = ClpSolverSettingsBuilder::default().parallel().threads(4).build(); -/// -/// let mut builder = ClpSolverSettingsBuilder::default(); -/// builder.chunk_size(NonZeroUsize::new(1024).unwrap()); -/// let settings = builder.build(); +/// let settings = HighsSolverSettingsBuilder::default().parallel().threads(4).build(); /// +/// let mut builder = HighsSolverSettingsBuilder::default(); /// builder.parallel(); /// let settings = builder.build(); /// @@ -97,6 +94,6 @@ mod tests { }; let settings_from_builder = HighsSolverSettingsBuilder::default().parallel().build(); - assert_eq!(settings_from_builder, settings_from_builder); + assert_eq!(settings_from_builder, settings); } } From 4040da263f9235dc0307427bc6397b965305369a Mon Sep 17 00:00:00 2001 From: James Batchelor Date: Thu, 21 Mar 2024 00:06:54 +0000 Subject: [PATCH 09/12] fix: conversion of inline dataframe parameter to timeseries --- pywr-core/src/parameters/py.rs | 2 - pywr-schema/src/model.rs | 34 +-- .../src/nodes/annual_virtual_storage.rs | 1 - pywr-schema/src/nodes/mod.rs | 225 ++++++++++++++++-- pywr-schema/src/parameters/mod.rs | 14 +- pywr-schema/src/parameters/python.rs | 6 +- 6 files changed, 222 insertions(+), 60 deletions(-) diff --git a/pywr-core/src/parameters/py.rs b/pywr-core/src/parameters/py.rs index 9289ce39..5c9b97a1 100644 --- a/pywr-core/src/parameters/py.rs +++ b/pywr-core/src/parameters/py.rs @@ -145,7 +145,6 @@ impl PyParameter { } impl Parameter for PyParameter { - fn meta(&self) -> &ParameterMeta { &self.meta } @@ -182,7 +181,6 @@ impl Parameter for PyParameter { } impl Parameter for PyParameter { - fn meta(&self) -> &ParameterMeta { &self.meta } diff --git a/pywr-schema/src/model.rs b/pywr-schema/src/model.rs index dd12d6f5..eec601d9 100644 --- a/pywr-schema/src/model.rs +++ b/pywr-schema/src/model.rs @@ -426,7 +426,7 @@ impl TryFrom for PywrModel { .flatten() .collect::>(); - let mut nodes = nodes_and_ts.into_iter().map(|n| n.node).collect::>(); + let nodes = nodes_and_ts.into_iter().map(|n| n.node).collect::>(); let edges = v1.edges.into_iter().map(|e| e.into()).collect(); @@ -442,38 +442,6 @@ impl TryFrom for PywrModel { let timeseries = if !ts_data.is_empty() { let ts = convert_from_v1_data(&ts_data, &v1.tables); - let ts_names: HashSet<&str> = ts.iter().map(|t| t.name()).collect(); - - // Update node param references - for n in nodes.iter_mut() { - let mut params = n.parameters_mut(); - for param in params.values_mut() { - if let DynamicFloatValue::Dynamic(MetricFloatValue::Reference(MetricFloatReference::Parameter { - name, - key: _, - })) = param - { - // If the parameter name matches one of the timeseries names, assume parameter reference needs updating to a timeseries reference. - // This should be fine as the sources of all the names is the v1 parameter list which should have unique names. - if ts_names.contains(name.as_str()) { - if let Some(data) = ts_data.iter().find(|t| t.name.as_ref().unwrap() == name) { - let col = match (&data.column, &data.scenario) { - (Some(col), None) => DataFrameColumns::Column(col.clone()), - (None, Some(scenario)) => DataFrameColumns::Scenario(scenario.clone()), - (Some(_), Some(_)) => { - return Err(ConversionError::AmbiguousColumnAndScenario(name.clone())) - } - (None, None) => return Err(ConversionError::MissingColumnOrScenario(name.clone())), - }; - let ts_ref = DynamicFloatValue::Dynamic(MetricFloatValue::Timeseries( - TimeseriesReference::new(name.clone(), col), - )); - **param = ts_ref; - } - } - } - } - } Some(ts) } else { None diff --git a/pywr-schema/src/nodes/annual_virtual_storage.rs b/pywr-schema/src/nodes/annual_virtual_storage.rs index 40948d3c..cedd42a6 100644 --- a/pywr-schema/src/nodes/annual_virtual_storage.rs +++ b/pywr-schema/src/nodes/annual_virtual_storage.rs @@ -32,7 +32,6 @@ impl Default for AnnualReset { } } - #[derive(serde::Deserialize, serde::Serialize, Clone, Default, Debug, PywrNode)] pub struct AnnualVirtualStorageNode { #[serde(flatten)] diff --git a/pywr-schema/src/nodes/mod.rs b/pywr-schema/src/nodes/mod.rs index 1ebf59b2..32b20bfa 100644 --- a/pywr-schema/src/nodes/mod.rs +++ b/pywr-schema/src/nodes/mod.rs @@ -34,7 +34,7 @@ use pywr_v1_schema::nodes::{ CoreNode as CoreNodeV1, Node as NodeV1, NodeMeta as NodeMetaV1, NodePosition as NodePositionV1, }; use pywr_v1_schema::parameters::{ - CoreParameter as CoreParameterV1, Parameter as ParameterV1, ParameterValue as ParameterValueV1, + CoreParameter as CoreParameterV1, Parameter as ParameterV1, ParameterValue as ParameterValueV1, ParameterValueType, }; pub use river_gauge::RiverGaugeNode; pub use river_split_with_gauge::RiverSplitWithGaugeNode; @@ -688,29 +688,220 @@ impl TryFrom for NodeAndTimeseries { fn try_from(v1: NodeV1) -> Result { let mut ts_vec = Vec::new(); + let mut unnamed_count: usize = 0; for param_value in v1.parameters().values() { - match param_value { - pywr_v1_schema::parameters::ParameterValueType::Single(param) => { - if let ParameterValueV1::Inline(p) = param { - if let ParameterV1::Core(CoreParameterV1::DataFrame(df_param)) = p.as_ref() { - let mut ts_data: TimeseriesV1Data = df_param.clone().into(); - - if ts_data.name.is_none() { - let name = format!("{}.timeseries", v1.name()); - ts_data.name = Some(name); - } - - ts_vec.push(ts_data); + ts_vec.extend(extract_timeseries(param_value, v1.name(), &mut unnamed_count)); + } + + let timeseries = if ts_vec.is_empty() { None } else { Some(ts_vec) }; + let node = Node::try_from(v1)?; + Ok(Self { node, timeseries }) + } +} + +/// Extract timeseries data from a parameter value. +/// +/// If the parameter value is a DataFrame, then convert it to timeseries data. If it is another type then recursively +/// call the function on any inline parameters this parameter may contain to check for other dataframe parameters. +fn extract_timeseries( + param_value: &ParameterValueType, + name: &str, + unnamed_count: &mut usize, +) -> Vec { + let mut ts_vec = Vec::new(); + match param_value { + ParameterValueType::Single(param) => { + if let ParameterValueV1::Inline(p) = param { + if let ParameterV1::Core(CoreParameterV1::DataFrame(df_param)) = p.as_ref() { + let mut ts_data: TimeseriesV1Data = df_param.clone().into(); + if ts_data.name.is_none() { + let name = format!("{}-p{}.timeseries", name, unnamed_count); + *unnamed_count += 1; + ts_data.name = Some(name); + } + ts_vec.push(ts_data); + } else { + // Not a dataframe parameter but the parameter might have child dataframe parameters. + // Update the name and call the function recursively on all child parameters. + let name = if p.name().is_none() { + let n = format!("{}-p{}", name, unnamed_count); + *unnamed_count += 1; + n + } else { + p.name().unwrap().to_string() + }; + for nested_param in p.parameters().values() { + ts_vec.extend(extract_timeseries(nested_param, &name, unnamed_count)); + } + } + } + } + ParameterValueType::List(params) => { + for param in params.iter() { + if let ParameterValueV1::Inline(p) = param { + if let ParameterV1::Core(CoreParameterV1::DataFrame(df_param)) = p.as_ref() { + let mut ts_data: TimeseriesV1Data = df_param.clone().into(); + if ts_data.name.is_none() { + let name = format!("{}-p{}.timeseries", name, unnamed_count); + *unnamed_count += 1; + ts_data.name = Some(name); + } + ts_vec.push(ts_data); + } else { + // Not a dataframe parameter but the parameter might have child dataframe parameters. + // Update the name and call the function recursively on all child parameters. + let name = if p.name().is_none() { + let n = format!("{}-p{}", name, unnamed_count); + *unnamed_count += 1; + n + } else { + p.name().unwrap().to_string() + }; + for nested_param in p.parameters().values() { + ts_vec.extend(extract_timeseries(nested_param, &name, unnamed_count)); } } } - pywr_v1_schema::parameters::ParameterValueType::List(_) => todo!(), } } + }; + ts_vec +} - let timeseries = if ts_vec.is_empty() { None } else { Some(ts_vec) }; +#[cfg(test)] +mod tests { + use pywr_v1_schema::nodes::Node as NodeV1; + + use crate::{ + nodes::{Node, NodeAndTimeseries}, + parameters::{DynamicFloatValue, MetricFloatValue, Parameter}, + }; + + #[test] + fn test_ts_inline() { + let node_data = r#" + { + "name": "catchment1", + "type": "Input", + "max_flow": { + "type": "dataframe", + "url" : "timeseries1.csv", + "parse_dates": true, + "dayfirst": true, + "index_col": 0, + "column": "Data" + } + } + "#; - let node = Node::try_from(v1)?; - Ok(Self { node, timeseries }) + let v1_node: NodeV1 = serde_json::from_str(node_data).unwrap(); + + let node_ts: NodeAndTimeseries = v1_node.try_into().unwrap(); + + let input_node = match node_ts.node { + Node::Input(n) => n, + _ => panic!("Expected InputNode"), + }; + + let expected_name = String::from("catchment1-p0.timeseries"); + + match input_node.max_flow { + Some(DynamicFloatValue::Dynamic(MetricFloatValue::Timeseries(ts))) => { + assert_eq!(ts.name(), &expected_name) + } + _ => panic!("Expected Timeseries"), + }; + + match node_ts.timeseries { + Some(ts) => { + assert_eq!(ts.len(), 1); + assert_eq!(ts.first().unwrap().name.as_ref().unwrap().as_str(), &expected_name); + } + None => panic!("Expected timeseries data"), + }; + } + + #[test] + fn test_ts_inline_nested() { + let node_data = r#" + { + "name": "catchment1", + "type": "Input", + "max_flow": { + "type": "aggregated", + "agg_func": "product", + "parameters": [ + { + "type": "constant", + "value": 0.9 + }, + { + "type": "dataframe", + "url" : "timeseries1.csv", + "parse_dates": true, + "dayfirst": true, + "index_col": 0, + "column": "Data" + }, + { + "type": "constant", + "value": 0.9 + }, + { + "type": "dataframe", + "url" : "timeseries2.csv", + "parse_dates": true, + "dayfirst": true, + "index_col": 0, + "column": "Data" + } + ] + } + } + "#; + + let v1_node: NodeV1 = serde_json::from_str(node_data).unwrap(); + + let node_ts: NodeAndTimeseries = v1_node.try_into().unwrap(); + + let input_node = match node_ts.node { + Node::Input(n) => n, + _ => panic!("Expected InputNode"), + }; + + let expected_name1 = String::from("catchment1-p0-p2.timeseries"); + let expected_name2 = String::from("catchment1-p0-p4.timeseries"); + + match input_node.max_flow { + Some(DynamicFloatValue::Dynamic(MetricFloatValue::InlineParameter { definition })) => match *definition { + Parameter::Aggregated(param) => { + assert_eq!(param.metrics.len(), 4); + match ¶m.metrics[1] { + DynamicFloatValue::Dynamic(MetricFloatValue::Timeseries(ts)) => { + assert_eq!(ts.name(), &expected_name1) + } + _ => panic!("Expected Timeseries"), + } + + match ¶m.metrics[3] { + DynamicFloatValue::Dynamic(MetricFloatValue::Timeseries(ts)) => { + assert_eq!(ts.name(), &expected_name2) + } + _ => panic!("Expected Timeseries"), + } + } + _ => panic!("Expected Aggregated parameter"), + }, + _ => panic!("Expected Timeseries"), + }; + + match node_ts.timeseries { + Some(ts) => { + assert_eq!(ts.len(), 2); + assert_eq!(ts[0].name.as_ref().unwrap().as_str(), &expected_name1); + assert_eq!(ts[1].name.as_ref().unwrap().as_str(), &expected_name2); + } + None => panic!("Expected timeseries data"), + }; } } diff --git a/pywr-schema/src/parameters/mod.rs b/pywr-schema/src/parameters/mod.rs index bd02a660..ad9329c9 100644 --- a/pywr-schema/src/parameters/mod.rs +++ b/pywr-schema/src/parameters/mod.rs @@ -777,6 +777,10 @@ impl TimeseriesReference { let ty = "Timeseries".to_string(); Self { ty, name, columns } } + + pub fn name(&self) -> &str { + self.name.as_str() + } } /// A floating-point(f64) value from a metric in the network. @@ -967,12 +971,12 @@ impl TryFromV1Parameter for DynamicFloatValue { let name = match t.name { Some(n) => n, None => { - match parent_node { - // TODO if the node has inline timeseries for more than 1 attribute then this name will not be unique! - // The attribute name might need passing into the function - Some(node_name) => format!("{}.timeseries", node_name), + let n = match parent_node { + Some(node_name) => format!("{}-p{}.timeseries", node_name, *unnamed_count), None => format!("unnamed-timeseries-{}", *unnamed_count), - } + }; + *unnamed_count += 1; + n } }; diff --git a/pywr-schema/src/parameters/python.rs b/pywr-schema/src/parameters/python.rs index 9964684e..e7597085 100644 --- a/pywr-schema/src/parameters/python.rs +++ b/pywr-schema/src/parameters/python.rs @@ -271,8 +271,9 @@ mod tests { let schema = PywrNetwork::default(); let mut network = Network::default(); let tables = LoadedTableCollection::from_schema(None, None).unwrap(); + let ts = LoadedTimeseriesCollection::default(); param - .add_to_model(&mut network, &schema, &domain, &tables, None, &[]) + .add_to_model(&mut network, &schema, &domain, &tables, None, &[], &ts) .unwrap(); assert!(network.get_parameter_by_name("my-float-parameter").is_ok()); @@ -306,8 +307,9 @@ mod tests { let schema = PywrNetwork::default(); let mut network = Network::default(); let tables = LoadedTableCollection::from_schema(None, None).unwrap(); + let ts = LoadedTimeseriesCollection::default(); param - .add_to_model(&mut network, &schema, &domain, &tables, None, &[]) + .add_to_model(&mut network, &schema, &domain, &tables, None, &[], &ts) .unwrap(); assert!(network.get_index_parameter_by_name("my-int-parameter").is_ok()); From 38f16d4d80e49c041843ebad25c3dc7ba5811405 Mon Sep 17 00:00:00 2001 From: James Batchelor Date: Sat, 23 Mar 2024 11:20:05 +0000 Subject: [PATCH 10/12] response to review comments --- pywr-core/src/timestep.rs | 3 ++- pywr-schema/src/error.rs | 1 - pywr-schema/src/model.rs | 22 +++++++------------ pywr-schema/src/nodes/mod.rs | 13 ++++++++++- pywr-schema/src/parameters/control_curves.rs | 2 +- pywr-schema/src/parameters/mod.rs | 8 ++++++- pywr-schema/src/parameters/offset.rs | 2 +- .../src/timeseries/align_and_resample.rs | 14 ++++++------ pywr-schema/src/timeseries/mod.rs | 11 ++++++++-- 9 files changed, 47 insertions(+), 29 deletions(-) diff --git a/pywr-core/src/timestep.rs b/pywr-core/src/timestep.rs index 48c71e38..55adf34d 100644 --- a/pywr-core/src/timestep.rs +++ b/pywr-core/src/timestep.rs @@ -54,11 +54,12 @@ impl PywrDuration { } } - // Returns the fractional number of days in the duration. + /// Returns the fractional number of days in the duration. pub fn fractional_days(&self) -> f64 { self.0.num_seconds() as f64 / SECS_IN_DAY as f64 } + /// Returns the number of nanoseconds in the duration. pub fn whole_nanoseconds(&self) -> Option { self.0.num_nanoseconds() } diff --git a/pywr-schema/src/error.rs b/pywr-schema/src/error.rs index 06c146ab..ad0d216c 100644 --- a/pywr-schema/src/error.rs +++ b/pywr-schema/src/error.rs @@ -1,7 +1,6 @@ use crate::data_tables::TableError; use crate::nodes::NodeAttribute; use crate::timeseries::TimeseriesError; -use polars::error::PolarsError; use pyo3::exceptions::PyRuntimeError; use pyo3::PyErr; use thiserror::Error; diff --git a/pywr-schema/src/model.rs b/pywr-schema/src/model.rs index c32fc3da..7f6c6af6 100644 --- a/pywr-schema/src/model.rs +++ b/pywr-schema/src/model.rs @@ -6,19 +6,12 @@ use crate::error::{ConversionError, SchemaError}; use crate::metric_sets::MetricSet; use crate::nodes::NodeAndTimeseries; use crate::outputs::Output; -use crate::parameters::{ - convert_parameter_v1_to_v2, DataFrameColumns, DynamicFloatValue, MetricFloatReference, MetricFloatValue, - TimeseriesReference, TryIntoV2Parameter, -}; +use crate::parameters::{convert_parameter_v1_to_v2, MetricFloatReference}; use crate::timeseries::{convert_from_v1_data, LoadedTimeseriesCollection, Timeseries}; use chrono::{NaiveDate, NaiveDateTime, NaiveTime}; -use polars::frame::DataFrame; use pywr_core::models::ModelDomain; use pywr_core::timestep::TimestepDuration; use pywr_core::PywrError; -use serde::de; -use std::collections::HashSet; -use std::hash::Hash; use std::path::{Path, PathBuf}; use std::str::FromStr; @@ -445,6 +438,7 @@ impl PywrModel { Timestepper::default() }); + // Extract nodes and any timeseries data from the v1 nodes let nodes_and_ts: Vec = v1 .nodes .clone() @@ -468,16 +462,16 @@ impl PywrModel { let edges = v1.edges.into_iter().map(|e| e.into()).collect(); - let (parameters, param_ts_data) = if let Some(v1_parameters) = v1.parameters { + let parameters = if let Some(v1_parameters) = v1.parameters { let mut unnamed_count: usize = 0; - let (parameters, param_ts_data) = convert_parameter_v1_to_v2(v1_parameters, &mut unnamed_count, &mut errors); - (Some(parameters), Some(param_ts_data)) + let (parameters, param_ts_data) = + convert_parameter_v1_to_v2(v1_parameters, &mut unnamed_count, &mut errors); + ts_data.extend(param_ts_data); + Some(parameters) } else { - (None, None) + None }; - ts_data.extend(param_ts_data.into_iter().flatten()); - let timeseries = if !ts_data.is_empty() { let ts = convert_from_v1_data(&ts_data, &v1.tables); Some(ts) diff --git a/pywr-schema/src/nodes/mod.rs b/pywr-schema/src/nodes/mod.rs index 32b20bfa..40927843 100644 --- a/pywr-schema/src/nodes/mod.rs +++ b/pywr-schema/src/nodes/mod.rs @@ -677,6 +677,10 @@ impl TryFrom> for Node { } } +/// struct that acts as a container for a node and any associated timeseries data. +/// +/// v1 nodes may contain inline DataFrame parameters from which data needs to be extract +/// to created timeseries entries in the schema. #[derive(Debug)] pub struct NodeAndTimeseries { pub node: Node, @@ -689,11 +693,14 @@ impl TryFrom for NodeAndTimeseries { fn try_from(v1: NodeV1) -> Result { let mut ts_vec = Vec::new(); let mut unnamed_count: usize = 0; + + // extract timeseries data for all inline DataFame parameters included in the node. for param_value in v1.parameters().values() { ts_vec.extend(extract_timeseries(param_value, v1.name(), &mut unnamed_count)); } - let timeseries = if ts_vec.is_empty() { None } else { Some(ts_vec) }; + + // Now convert the node to the v2 schema representation let node = Node::try_from(v1)?; Ok(Self { node, timeseries }) } @@ -715,6 +722,8 @@ fn extract_timeseries( if let ParameterV1::Core(CoreParameterV1::DataFrame(df_param)) = p.as_ref() { let mut ts_data: TimeseriesV1Data = df_param.clone().into(); if ts_data.name.is_none() { + // Because the parameter could contain multiple inline DataFrame parameters use the unnamed_count + // to create a unique name. let name = format!("{}-p{}.timeseries", name, unnamed_count); *unnamed_count += 1; ts_data.name = Some(name); @@ -742,6 +751,8 @@ fn extract_timeseries( if let ParameterV1::Core(CoreParameterV1::DataFrame(df_param)) = p.as_ref() { let mut ts_data: TimeseriesV1Data = df_param.clone().into(); if ts_data.name.is_none() { + // Because the parameter could contain multiple inline DataFrame parameters use the unnamed_count + // to create a unique name. let name = format!("{}-p{}.timeseries", name, unnamed_count); *unnamed_count += 1; ts_data.name = Some(name); diff --git a/pywr-schema/src/parameters/control_curves.rs b/pywr-schema/src/parameters/control_curves.rs index a0869b91..09a2c1ec 100644 --- a/pywr-schema/src/parameters/control_curves.rs +++ b/pywr-schema/src/parameters/control_curves.rs @@ -5,7 +5,7 @@ use crate::nodes::NodeAttribute; use crate::parameters::{ DynamicFloatValue, IntoV2Parameter, NodeReference, ParameterMeta, TryFromV1Parameter, TryIntoV2Parameter, }; -use crate::timeseries::{self, LoadedTimeseriesCollection}; +use crate::timeseries::LoadedTimeseriesCollection; use pywr_core::models::ModelDomain; use pywr_core::parameters::ParameterIndex; use pywr_v1_schema::parameters::{ diff --git a/pywr-schema/src/parameters/mod.rs b/pywr-schema/src/parameters/mod.rs index 65158663..e3b143db 100644 --- a/pywr-schema/src/parameters/mod.rs +++ b/pywr-schema/src/parameters/mod.rs @@ -433,7 +433,7 @@ pub fn convert_parameter_v1_to_v2( ) -> (Vec, Vec) { let param_or_ts: Vec = v1_parameters .into_iter() - .filter_map(|p| match p.try_into_v2_parameter(None, unnamed_count){ + .filter_map(|p| match p.try_into_v2_parameter(None, unnamed_count) { Ok(pt) => Some(pt), Err(e) => { errors.push(e); @@ -472,6 +472,7 @@ enum ParameterOrTimeseries { pub struct TimeseriesV1Data { pub name: Option, pub source: TimeseriesV1Source, + pub time_col: Option, pub column: Option, pub scenario: Option, } @@ -487,10 +488,15 @@ impl From for TimeseriesV1Data { }; let name = p.meta.and_then(|m| m.name); + let time_col = match p.pandas_kwargs.get("index_col") { + Some(v) => v.as_str().map(|s| s.to_string()), + None => None, + }; Self { name, source, + time_col, column: p.column, scenario: p.scenario, } diff --git a/pywr-schema/src/parameters/offset.rs b/pywr-schema/src/parameters/offset.rs index c49c8711..4e74dfee 100644 --- a/pywr-schema/src/parameters/offset.rs +++ b/pywr-schema/src/parameters/offset.rs @@ -1,5 +1,5 @@ use crate::data_tables::LoadedTableCollection; -use crate::parameters::{ConstantValue, DynamicFloatValue, DynamicFloatValueType, ParameterMeta, VariableSettings}; +use crate::parameters::{ConstantValue, DynamicFloatValue, DynamicFloatValueType, ParameterMeta}; use crate::timeseries::LoadedTimeseriesCollection; use pywr_core::parameters::ParameterIndex; diff --git a/pywr-schema/src/timeseries/align_and_resample.rs b/pywr-schema/src/timeseries/align_and_resample.rs index 52dbad6e..17411dcb 100644 --- a/pywr-schema/src/timeseries/align_and_resample.rs +++ b/pywr-schema/src/timeseries/align_and_resample.rs @@ -1,4 +1,3 @@ -use chrono::TimeDelta; use polars::{prelude::*, series::ops::NullBehavior}; use pywr_core::models::ModelDomain; use std::{cmp::Ordering, ops::Deref}; @@ -31,8 +30,7 @@ pub fn align_and_resample( let durations = durations.column("duration")?.duration()?.deref(); if durations.len() > 1 { - // Non-uniform timestep are not yet supported - todo!(); + todo!("Non-uniform timestep are not yet supported"); } let timeseries_duration = match durations.get(0) { @@ -44,7 +42,7 @@ pub fn align_and_resample( .time() .step_duration() .whole_nanoseconds() - .expect("Nano seconds could not be extracted from model step duration"); + .ok_or(TimeseriesError::NoDurationNanoSeconds)?; let df = match model_duration.cmp(×eries_duration) { Ordering::Greater => { @@ -78,7 +76,10 @@ pub fn align_and_resample( let df = slice_end(df, time_col, domain)?; - // TODO check df length equals number of model timesteps + if df.height() != domain.time().timesteps().len() { + return Err(TimeseriesError::DataFrameTimestepMismatch(name.to_string())); + } + Ok(df) } @@ -96,7 +97,6 @@ fn slice_end(df: DataFrame, time_col: &str, domain: &ModelDomain) -> Result = vec![1.0; 31]; + let values: Vec = (1..32).map(|x| x as f64).collect(); let mut df = df!( "time" => time, diff --git a/pywr-schema/src/timeseries/mod.rs b/pywr-schema/src/timeseries/mod.rs index 02a82433..878eee4c 100644 --- a/pywr-schema/src/timeseries/mod.rs +++ b/pywr-schema/src/timeseries/mod.rs @@ -9,7 +9,6 @@ use pywr_core::models::ModelDomain; use pywr_core::parameters::{Array1Parameter, Array2Parameter, ParameterIndex}; use pywr_core::PywrError; use pywr_v1_schema::tables::TableVec; -use std::sync::Arc; use std::{collections::HashMap, path::Path}; use thiserror::Error; @@ -31,6 +30,12 @@ pub enum TimeseriesError { TimeseriesUnparsableFileFormat { provider: String, path: String }, #[error("A scenario group with name '{0}' was not found")] ScenarioGroupNotFound(String), + #[error("Duration could not be represented as nanoseconds")] + NoDurationNanoSeconds, + #[error("The length of the resampled timeseries dataframe '{0}' does not match the number of model timesteps.")] + DataFrameTimestepMismatch(String), + #[error("A timeseries dataframe with the name '{0}' already exists.")] + TimeseriesDataframeAlreadyExists(String), #[error("Polars error: {0}")] PolarsError(#[from] PolarsError), #[error("Pywr core error: {0}")] @@ -79,7 +84,9 @@ impl LoadedTimeseriesCollection { if let Some(timeseries_defs) = timeseries_defs { for ts in timeseries_defs { let df = ts.load(domain, data_path)?; - // TODO error if key already exists + if timeseries.contains_key(&ts.meta.name) { + return Err(TimeseriesError::TimeseriesDataframeAlreadyExists(ts.meta.name.clone())); + } timeseries.insert(ts.meta.name.clone(), df); } } From a021b2b49865a56eaec1c5e6433a669fc0e8ff67 Mon Sep 17 00:00:00 2001 From: James Batchelor Date: Sun, 24 Mar 2024 00:02:59 +0000 Subject: [PATCH 11/12] fix: use index_col for time_col when converting v1 df param to timeseries --- pywr-schema/src/error.rs | 2 ++ pywr-schema/src/model.rs | 2 +- pywr-schema/src/timeseries/mod.rs | 23 ++++++++++++++++------- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/pywr-schema/src/error.rs b/pywr-schema/src/error.rs index ad0d216c..a458878d 100644 --- a/pywr-schema/src/error.rs +++ b/pywr-schema/src/error.rs @@ -110,4 +110,6 @@ pub enum ConversionError { AmbiguousColumnAndScenario(String), #[error("The dataframe parameters '{0}' defines both a column and a scenario. Only 1 is allowed.")] MissingColumnOrScenario(String), + #[error("Unable to create a timeseries for file: '{0}'. No name was found.")] + MissingTimeseriesName(String), } diff --git a/pywr-schema/src/model.rs b/pywr-schema/src/model.rs index 7f6c6af6..2b2c9435 100644 --- a/pywr-schema/src/model.rs +++ b/pywr-schema/src/model.rs @@ -473,7 +473,7 @@ impl PywrModel { }; let timeseries = if !ts_data.is_empty() { - let ts = convert_from_v1_data(&ts_data, &v1.tables); + let ts = convert_from_v1_data(ts_data, &v1.tables, &mut errors); Some(ts) } else { None diff --git a/pywr-schema/src/timeseries/mod.rs b/pywr-schema/src/timeseries/mod.rs index 878eee4c..99083153 100644 --- a/pywr-schema/src/timeseries/mod.rs +++ b/pywr-schema/src/timeseries/mod.rs @@ -13,6 +13,7 @@ use std::{collections::HashMap, path::Path}; use thiserror::Error; use crate::parameters::{ParameterMeta, TimeseriesV1Data, TimeseriesV1Source}; +use crate::ConversionError; use self::polars_dataset::PolarsDataset; @@ -153,10 +154,14 @@ impl LoadedTimeseriesCollection { } } -pub fn convert_from_v1_data(df_data: &[TimeseriesV1Data], v1_tables: &Option) -> Vec { +pub fn convert_from_v1_data( + df_data: Vec, + v1_tables: &Option, + errors: &mut Vec, +) -> Vec { let mut ts = HashMap::new(); - for data in df_data.iter() { - match &data.source { + for data in df_data.into_iter() { + match data.source { TimeseriesV1Source::Table(name) => { let tables = v1_tables.as_ref().unwrap(); let table = tables.iter().find(|t| t.name == *name).unwrap(); @@ -177,14 +182,18 @@ pub fn convert_from_v1_data(df_data: &[TimeseriesV1Data], v1_tables: &Option { - let name = data.name.as_ref().unwrap().clone(); + let name = match data.name { + Some(name) => name, + None => { + errors.push(ConversionError::MissingTimeseriesName(url.to_str().unwrap_or("").to_string())); + continue; + } + }; if ts.contains_key(&name) { continue; } - // TODO: time_col should use the index_col from the v1 data? - let time_col = None; - let provider = PolarsDataset::new(time_col, url.clone()); + let provider = PolarsDataset::new(data.time_col, url); ts.insert( name.clone(), From 4150d546adb0aea0e6b5ef46c6d4fad932a5d62b Mon Sep 17 00:00:00 2001 From: James Batchelor Date: Mon, 25 Mar 2024 16:08:18 +0000 Subject: [PATCH 12/12] remove dataframe parameter from schema and update test models --- .../tests/models/aggregated-node1/model.json | 23 +- .../tests/models/piecewise-link1/model.json | 31 +- .../models/simple-custom-parameter/model.json | 23 +- .../tests/models/simple-timeseries/model.json | 23 +- .../tests/models/simple-wasm/model.json | 17 +- pywr-schema/src/parameters/data_frame.rs | 205 ----- pywr-schema/src/parameters/mod.rs | 25 +- pywr-schema/src/test_models/inflow.csv | 732 +++++++++--------- pywr-schema/src/test_models/timeseries.json | 25 +- pywr-schema/src/timeseries/mod.rs | 4 +- 10 files changed, 470 insertions(+), 638 deletions(-) delete mode 100644 pywr-schema/src/parameters/data_frame.rs diff --git a/pywr-python/tests/models/aggregated-node1/model.json b/pywr-python/tests/models/aggregated-node1/model.json index 4c69a449..1ddd2c59 100644 --- a/pywr-python/tests/models/aggregated-node1/model.json +++ b/pywr-python/tests/models/aggregated-node1/model.json @@ -13,8 +13,12 @@ "name": "input1", "type": "Input", "max_flow": { - "type": "Parameter", - "name": "inflow" + "type": "Timeseries", + "name": "inflow", + "columns": { + "type": "Column", + "name": "inflow" + } } }, { @@ -69,17 +73,14 @@ "name": "demand", "type": "Constant", "value": 10.0 - }, + } + ], + "timeseries": [ { "name": "inflow", - "type": "DataFrame", - "url": "inflow.csv", - "pandas_kwargs": { - "index_col": 0 - }, - "columns": { - "type": "Column", - "name": "inflow" + "provider": { + "type": "Polars", + "url": "inflow.csv" } } ], diff --git a/pywr-python/tests/models/piecewise-link1/model.json b/pywr-python/tests/models/piecewise-link1/model.json index 8b067e54..94f0d359 100644 --- a/pywr-python/tests/models/piecewise-link1/model.json +++ b/pywr-python/tests/models/piecewise-link1/model.json @@ -13,12 +13,20 @@ "name": "input1", "type": "Input", "max_flow": { - "type": "Parameter", - "name": "inflow" + "type": "Timeseries", + "name": "inflow", + "columns": { + "type": "Column", + "name": "inflow" + } }, "min_flow": { - "type": "Parameter", - "name": "inflow" + "type": "Timeseries", + "name": "inflow", + "columns": { + "type": "Column", + "name": "inflow" + } } }, { @@ -74,17 +82,14 @@ "name": "demand", "type": "Constant", "value": 10.0 - }, + } + ], + "timeseries": [ { "name": "inflow", - "type": "DataFrame", - "url": "inflow.csv", - "pandas_kwargs": { - "index_col": 0 - }, - "columns": { - "type": "Column", - "name": "inflow" + "provider": { + "type": "Polars", + "url": "inflow.csv" } } ], diff --git a/pywr-python/tests/models/simple-custom-parameter/model.json b/pywr-python/tests/models/simple-custom-parameter/model.json index 5b709a1c..c2da5aa0 100644 --- a/pywr-python/tests/models/simple-custom-parameter/model.json +++ b/pywr-python/tests/models/simple-custom-parameter/model.json @@ -13,8 +13,12 @@ "name": "input1", "type": "Input", "max_flow": { - "type": "Parameter", - "name": "inflow" + "type": "Timeseries", + "name": "inflow", + "columns": { + "type": "Column", + "name": "inflow" + } } }, { @@ -53,17 +57,14 @@ "kwargs": { "multiplier": 2.0 } - }, + } + ], + "timeseries": [ { "name": "inflow", - "type": "DataFrame", - "url": "inflow.csv", - "pandas_kwargs": { - "index_col": 0 - }, - "columns": { - "type": "Column", - "name": "inflow" + "provider": { + "type": "Polars", + "url": "inflow.csv" } } ], diff --git a/pywr-python/tests/models/simple-timeseries/model.json b/pywr-python/tests/models/simple-timeseries/model.json index 0a992c1e..90f7e9cc 100644 --- a/pywr-python/tests/models/simple-timeseries/model.json +++ b/pywr-python/tests/models/simple-timeseries/model.json @@ -13,8 +13,12 @@ "name": "input1", "type": "Input", "max_flow": { - "type": "Parameter", - "name": "inflow" + "type": "Timeseries", + "name": "inflow", + "columns": { + "type": "Column", + "name": "inflow" + } } }, { @@ -46,17 +50,14 @@ "name": "demand", "type": "Constant", "value": 10.0 - }, + } + ], + "timeseries": [ { "name": "inflow", - "type": "DataFrame", - "url": "inflow.csv", - "pandas_kwargs": { - "index_col": 0 - }, - "columns": { - "type": "Column", - "name": "inflow" + "provider": { + "type": "Polars", + "url": "inflow.csv" } } ], diff --git a/pywr-python/tests/models/simple-wasm/model.json b/pywr-python/tests/models/simple-wasm/model.json index cdb1bbe7..99f105d7 100644 --- a/pywr-python/tests/models/simple-wasm/model.json +++ b/pywr-python/tests/models/simple-wasm/model.json @@ -10,8 +10,12 @@ "name": "input1", "type": "input", "max_flow": { - "type": "Parameter", - "name": "inflow" + "type": "Timeseries", + "name": "inflow", + "columns": { + "type": "Column", + "name": "inflow" + } } }, { @@ -64,6 +68,15 @@ "url": "inflow.csv.gz", "column": "inflow" } + ], + "timeseries": [ + { + "name": "inflow", + "provider": { + "type": "Polars", + "url": "inflow.csv.gz" + } + } ] } } diff --git a/pywr-schema/src/parameters/data_frame.rs b/pywr-schema/src/parameters/data_frame.rs deleted file mode 100644 index d0ce6a9e..00000000 --- a/pywr-schema/src/parameters/data_frame.rs +++ /dev/null @@ -1,205 +0,0 @@ -use crate::error::SchemaError; -use crate::parameters::python::try_json_value_into_py; -use crate::parameters::{DynamicFloatValueType, IntoV2Parameter, ParameterMeta, TryFromV1Parameter}; -use crate::ConversionError; -use ndarray::Array2; -use polars::prelude::DataType::Float64; -use polars::prelude::{DataFrame, Float64Type, IndexOrder}; -use pyo3::prelude::PyModule; -use pyo3::types::{PyDict, PyTuple}; -use pyo3::{IntoPy, PyErr, PyObject, Python, ToPyObject}; -use pyo3_polars::PyDataFrame; -use pywr_core::models::ModelDomain; -use pywr_core::parameters::{Array1Parameter, Array2Parameter, ParameterIndex}; -use pywr_v1_schema::parameters::DataFrameParameter as DataFrameParameterV1; -use std::collections::HashMap; -use std::path::{Path, PathBuf}; - -#[derive(serde::Deserialize, serde::Serialize, Debug, Clone)] -#[serde(tag = "type", content = "name")] -pub enum DataFrameColumns { - Scenario(String), - Column(String), -} - -enum FileFormat { - Csv, - Hdf, - Excel, -} - -impl FileFormat { - /// Determine file format from a path's extension. - fn from_path(path: &Path) -> Option { - match path.extension() { - None => None, // No extension; unknown format - Some(ext) => match ext.to_str() { - None => None, - Some(ext) => match ext.to_lowercase().as_str() { - "h5" | "hdf5" | "hdf" => Some(FileFormat::Hdf), - "csv" => Some(FileFormat::Csv), - "xlsx" | "xlsm" => Some(FileFormat::Excel), - "gz" => FileFormat::from_path(&path.with_extension("")), - _ => None, - }, - }, - } - } -} - -/// A parameter that reads its data into a Pandas DataFrame object. -/// -/// Upon loading this parameter will attempt to read its data using the Python library -/// `pandas`. It expects to load a timeseries DataFrame which is then sliced and aligned -/// to the -#[derive(serde::Deserialize, serde::Serialize, Debug, Clone)] -pub struct DataFrameParameter { - #[serde(flatten)] - pub meta: ParameterMeta, - pub url: PathBuf, - pub columns: DataFrameColumns, - pub timestep_offset: Option, - pub pandas_kwargs: HashMap, -} - -impl DataFrameParameter { - pub fn node_references(&self) -> HashMap<&str, &str> { - HashMap::new() - } - pub fn parameters(&self) -> HashMap<&str, DynamicFloatValueType> { - HashMap::new() - } - - pub fn add_to_model( - &self, - network: &mut pywr_core::network::Network, - domain: &ModelDomain, - data_path: Option<&Path>, - ) -> Result, SchemaError> { - // Handle the case of an optional data path with a relative url. - let pth = if let Some(dp) = data_path { - if self.url.is_relative() { - dp.join(&self.url) - } else { - self.url.clone() - } - } else { - self.url.clone() - }; - - let format = FileFormat::from_path(&pth).ok_or(SchemaError::UnsupportedFileFormat)?; - - // 1. Call Python & Pandas to read the data and return an array - let df: DataFrame = Python::with_gil(|py| { - // Import pandas and appropriate read function depending on file extension - let pandas = PyModule::import(py, "pandas")?; - // Determine pandas read function from file format. - let read_func = match format { - FileFormat::Csv => pandas.getattr("read_csv"), - FileFormat::Hdf => pandas.getattr("read_hdf"), - FileFormat::Excel => pandas.getattr("read_excel"), - }?; - - // Import polars and get a reference to the DataFrame initialisation method - let polars = PyModule::import(py, "polars")?; - let polars_data_frame_init = polars.getattr("DataFrame")?; - - // Create arguments for pandas - let args = (pth.into_py(py),); - let seq = PyTuple::new( - py, - self.pandas_kwargs - .iter() - .map(|(k, v)| (k.into_py(py), try_json_value_into_py(py, v).unwrap())), - ); - let kwargs = PyDict::from_sequence(py, seq.to_object(py))?; - // Read pandas DataFrame from relevant function - let py_pandas_df: PyObject = read_func.call(args, Some(kwargs))?.extract()?; - // Convert to polars DataFrame using the Python library - let py_polars_df: PyDataFrame = polars_data_frame_init.call1((py_pandas_df,))?.extract()?; - - Ok(py_polars_df.into()) - }) - .map_err(|e: PyErr| SchemaError::PythonError(e.to_string()))?; - - // 2. TODO Validate the shape of the data array. I.e. check number of columns matches scenario - // and number of rows matches time-steps. - - // 3. Create an ArrayParameter using the loaded array. - match &self.columns { - DataFrameColumns::Scenario(scenario) => { - let scenario_group_index = domain - .scenarios() - .group_index(scenario) - .ok_or(SchemaError::ScenarioGroupNotFound(scenario.to_string()))?; - - let array: Array2 = df.to_ndarray::(IndexOrder::default()).unwrap(); - let p = Array2Parameter::new(&self.meta.name, array, scenario_group_index, self.timestep_offset); - Ok(network.add_parameter(Box::new(p))?) - } - DataFrameColumns::Column(column) => { - let series = df.column(column).unwrap(); - let array = series - .cast(&Float64) - .unwrap() - .f64() - .unwrap() - .to_ndarray() - .unwrap() - .to_owned(); - - let p = Array1Parameter::new(&self.meta.name, array, self.timestep_offset); - Ok(network.add_parameter(Box::new(p))?) - } - } - } -} - -impl TryFromV1Parameter for DataFrameParameter { - type Error = ConversionError; - - fn try_from_v1_parameter( - v1: DataFrameParameterV1, - parent_node: Option<&str>, - unnamed_count: &mut usize, - ) -> Result { - let meta: ParameterMeta = v1.meta.into_v2_parameter(parent_node, unnamed_count); - let url = v1.url.ok_or(ConversionError::MissingAttribute { - attrs: vec!["url".to_string()], - name: meta.name.clone(), - })?; - - // Here we can only handle a specific column or assume the columns map to a scenario group. - let columns = match (v1.column, v1.scenario) { - (None, None) => { - return Err(ConversionError::MissingAttribute { - attrs: vec!["column".to_string(), "scenario".to_string()], - name: meta.name.clone(), - }) - } - (Some(_), Some(_)) => { - return Err(ConversionError::UnexpectedAttribute { - attrs: vec!["column".to_string(), "scenario".to_string()], - name: meta.name.clone(), - }) - } - (Some(c), None) => DataFrameColumns::Column(c), - (None, Some(s)) => DataFrameColumns::Scenario(s), - }; - - if v1.index.is_some() || v1.indexes.is_some() || v1.table.is_some() { - return Err(ConversionError::UnsupportedAttribute { - attrs: vec!["index".to_string(), "indexes".to_string(), "table".to_string()], - name: meta.name.clone(), - }); - } - - Ok(Self { - meta, - url, - columns, - timestep_offset: v1.timestep_offset, - pandas_kwargs: v1.pandas_kwargs, - }) - } -} diff --git a/pywr-schema/src/parameters/mod.rs b/pywr-schema/src/parameters/mod.rs index 6776af0a..2c0a8b49 100644 --- a/pywr-schema/src/parameters/mod.rs +++ b/pywr-schema/src/parameters/mod.rs @@ -11,7 +11,6 @@ mod aggregated; mod asymmetric_switch; mod control_curves; mod core; -mod data_frame; mod delay; mod discount_factor; mod indexed_array; @@ -48,7 +47,6 @@ use crate::error::{ConversionError, SchemaError}; use crate::model::PywrMultiNetworkTransfer; use crate::nodes::NodeAttribute; use crate::parameters::core::DivisionParameter; -pub use crate::parameters::data_frame::{DataFrameColumns, DataFrameParameter}; use crate::parameters::interpolated::InterpolatedParameter; use crate::timeseries::LoadedTimeseriesCollection; pub use offset::OffsetParameter; @@ -164,7 +162,6 @@ pub enum Parameter { ParameterThreshold(ParameterThresholdParameter), TablesArray(TablesArrayParameter), Python(PythonParameter), - DataFrame(DataFrameParameter), Delay(DelayParameter), Division(DivisionParameter), Offset(OffsetParameter), @@ -196,7 +193,6 @@ impl Parameter { Self::ParameterThreshold(p) => p.meta.name.as_str(), Self::TablesArray(p) => p.meta.name.as_str(), Self::Python(p) => p.meta.name.as_str(), - Self::DataFrame(p) => p.meta.name.as_str(), Self::Division(p) => p.meta.name.as_str(), Self::Delay(p) => p.meta.name.as_str(), Self::Offset(p) => p.meta.name.as_str(), @@ -228,7 +224,6 @@ impl Parameter { Self::ParameterThreshold(_) => "ParameterThreshold", Self::TablesArray(_) => "TablesArray", Self::Python(_) => "Python", - Self::DataFrame(_) => "DataFrame", Self::Delay(_) => "Delay", Self::Division(_) => "Division", Self::Offset(_) => "Offset", @@ -373,7 +368,6 @@ impl Parameter { inter_network_transfers, timeseries, )?, - Self::DataFrame(p) => ParameterType::Parameter(p.add_to_model(network, domain, data_path)?), Self::Delay(p) => ParameterType::Parameter(p.add_to_model( network, schema, @@ -777,16 +771,23 @@ impl MetricFloatReference { } } +#[derive(serde::Deserialize, serde::Serialize, Debug, Clone)] +#[serde(tag = "type", content = "name")] +pub enum TimeseriesColumns { + Scenario(String), + Column(String), +} + #[derive(serde::Deserialize, serde::Serialize, Debug, Clone)] pub struct TimeseriesReference { #[serde(rename = "type")] ty: String, name: String, - columns: DataFrameColumns, + columns: TimeseriesColumns, } impl TimeseriesReference { - pub fn new(name: String, columns: DataFrameColumns) -> Self { + pub fn new(name: String, columns: TimeseriesColumns) -> Self { let ty = "Timeseries".to_string(); Self { ty, name, columns } } @@ -852,10 +853,10 @@ impl MetricFloatValue { } Self::Timeseries(ts_ref) => { let param_idx = match &ts_ref.columns { - DataFrameColumns::Scenario(scenario) => { + TimeseriesColumns::Scenario(scenario) => { timeseries.load_df(network, ts_ref.name.as_ref(), domain, scenario.as_str())? } - DataFrameColumns::Column(col) => { + TimeseriesColumns::Column(col) => { timeseries.load_column(network, ts_ref.name.as_ref(), col.as_str())? } }; @@ -994,8 +995,8 @@ impl TryFromV1Parameter for DynamicFloatValue { }; let cols = match (&t.column, &t.scenario) { - (Some(col), None) => DataFrameColumns::Column(col.clone()), - (None, Some(scenario)) => DataFrameColumns::Scenario(scenario.clone()), + (Some(col), None) => TimeseriesColumns::Column(col.clone()), + (None, Some(scenario)) => TimeseriesColumns::Scenario(scenario.clone()), (Some(_), Some(_)) => { return Err(ConversionError::AmbiguousColumnAndScenario(name.clone())) } diff --git a/pywr-schema/src/test_models/inflow.csv b/pywr-schema/src/test_models/inflow.csv index 40df8051..07f79268 100644 --- a/pywr-schema/src/test_models/inflow.csv +++ b/pywr-schema/src/test_models/inflow.csv @@ -1,366 +1,366 @@ -date,inflow1,inflow2 -01/01/2021,1,2 -02/01/2021,2,2 -03/01/2021,3,2 -04/01/2021,4,2 -05/01/2021,5,2 -06/01/2021,6,2 -07/01/2021,7,2 -08/01/2021,8,2 -09/01/2021,9,2 -10/01/2021,10,2 -11/01/2021,11,2 -12/01/2021,12,2 -13/01/2021,13,2 -14/01/2021,14,2 -15/01/2021,15,2 -16/01/2021,16,2 -17/01/2021,17,2 -18/01/2021,18,2 -19/01/2021,19,2 -20/01/2021,20,2 -21/01/2021,21,2 -22/01/2021,22,2 -23/01/2021,23,2 -24/01/2021,24,2 -25/01/2021,25,2 -26/01/2021,26,2 -27/01/2021,27,2 -28/01/2021,28,2 -29/01/2021,29,2 -30/01/2021,30,2 -31/01/2021,31,2 -01/02/2021,1,2 -02/02/2021,2,2 -03/02/2021,3,2 -04/02/2021,4,2 -05/02/2021,5,2 -06/02/2021,6,2 -07/02/2021,7,2 -08/02/2021,8,2 -09/02/2021,9,2 -10/02/2021,10,2 -11/02/2021,11,2 -12/02/2021,12,2 -13/02/2021,13,2 -14/02/2021,14,2 -15/02/2021,15,2 -16/02/2021,16,2 -17/02/2021,17,2 -18/02/2021,18,2 -19/02/2021,19,2 -20/02/2021,20,2 -21/02/2021,21,2 -22/02/2021,22,2 -23/02/2021,23,2 -24/02/2021,24,2 -25/02/2021,25,2 -26/02/2021,26,2 -27/02/2021,27,2 -28/02/2021,28,2 -01/03/2021,1,2 -02/03/2021,2,2 -03/03/2021,3,2 -04/03/2021,4,2 -05/03/2021,5,2 -06/03/2021,6,2 -07/03/2021,7,2 -08/03/2021,8,2 -09/03/2021,9,2 -10/03/2021,10,2 -11/03/2021,11,2 -12/03/2021,12,2 -13/03/2021,13,2 -14/03/2021,14,2 -15/03/2021,15,2 -16/03/2021,16,2 -17/03/2021,17,2 -18/03/2021,18,2 -19/03/2021,19,2 -20/03/2021,20,2 -21/03/2021,21,2 -22/03/2021,22,2 -23/03/2021,23,2 -24/03/2021,24,2 -25/03/2021,25,2 -26/03/2021,26,2 -27/03/2021,27,2 -28/03/2021,28,2 -29/03/2021,29,2 -30/03/2021,30,2 -31/03/2021,31,2 -01/04/2021,1,2 -02/04/2021,2,2 -03/04/2021,3,2 -04/04/2021,4,2 -05/04/2021,5,2 -06/04/2021,6,2 -07/04/2021,7,2 -08/04/2021,8,2 -09/04/2021,9,2 -10/04/2021,10,2 -11/04/2021,11,2 -12/04/2021,12,2 -13/04/2021,13,2 -14/04/2021,14,2 -15/04/2021,15,2 -16/04/2021,16,2 -17/04/2021,17,2 -18/04/2021,18,2 -19/04/2021,19,2 -20/04/2021,20,2 -21/04/2021,21,2 -22/04/2021,22,2 -23/04/2021,23,2 -24/04/2021,24,2 -25/04/2021,25,2 -26/04/2021,26,2 -27/04/2021,27,2 -28/04/2021,28,2 -29/04/2021,29,2 -30/04/2021,30,2 -01/05/2021,1,2 -02/05/2021,2,2 -03/05/2021,3,2 -04/05/2021,4,2 -05/05/2021,5,2 -06/05/2021,6,2 -07/05/2021,7,2 -08/05/2021,8,2 -09/05/2021,9,2 -10/05/2021,10,2 -11/05/2021,11,2 -12/05/2021,12,2 -13/05/2021,13,2 -14/05/2021,14,2 -15/05/2021,15,2 -16/05/2021,16,2 -17/05/2021,17,2 -18/05/2021,18,2 -19/05/2021,19,2 -20/05/2021,20,2 -21/05/2021,21,2 -22/05/2021,22,2 -23/05/2021,23,2 -24/05/2021,24,2 -25/05/2021,25,2 -26/05/2021,26,2 -27/05/2021,27,2 -28/05/2021,28,2 -29/05/2021,29,2 -30/05/2021,30,2 -31/05/2021,31,2 -01/06/2021,1,2 -02/06/2021,2,2 -03/06/2021,3,2 -04/06/2021,4,2 -05/06/2021,5,2 -06/06/2021,6,2 -07/06/2021,7,2 -08/06/2021,8,2 -09/06/2021,9,2 -10/06/2021,10,2 -11/06/2021,11,2 -12/06/2021,12,2 -13/06/2021,13,2 -14/06/2021,14,2 -15/06/2021,15,2 -16/06/2021,16,2 -17/06/2021,17,2 -18/06/2021,18,2 -19/06/2021,19,2 -20/06/2021,20,2 -21/06/2021,21,2 -22/06/2021,22,2 -23/06/2021,23,2 -24/06/2021,24,2 -25/06/2021,25,2 -26/06/2021,26,2 -27/06/2021,27,2 -28/06/2021,28,2 -29/06/2021,29,2 -30/06/2021,30,2 -01/07/2021,1,2 -02/07/2021,2,2 -03/07/2021,3,2 -04/07/2021,4,2 -05/07/2021,5,2 -06/07/2021,6,2 -07/07/2021,7,2 -08/07/2021,8,2 -09/07/2021,9,2 -10/07/2021,10,2 -11/07/2021,11,2 -12/07/2021,12,2 -13/07/2021,13,2 -14/07/2021,14,2 -15/07/2021,15,2 -16/07/2021,16,2 -17/07/2021,17,2 -18/07/2021,18,2 -19/07/2021,19,2 -20/07/2021,20,2 -21/07/2021,21,2 -22/07/2021,22,2 -23/07/2021,23,2 -24/07/2021,24,2 -25/07/2021,25,2 -26/07/2021,26,2 -27/07/2021,27,2 -28/07/2021,28,2 -29/07/2021,29,2 -30/07/2021,30,2 -31/07/2021,31,2 -01/08/2021,1,2 -02/08/2021,2,2 -03/08/2021,3,2 -04/08/2021,4,2 -05/08/2021,5,2 -06/08/2021,6,2 -07/08/2021,7,2 -08/08/2021,8,2 -09/08/2021,9,2 -10/08/2021,10,2 -11/08/2021,11,2 -12/08/2021,12,2 -13/08/2021,13,2 -14/08/2021,14,2 -15/08/2021,15,2 -16/08/2021,16,2 -17/08/2021,17,2 -18/08/2021,18,2 -19/08/2021,19,2 -20/08/2021,20,2 -21/08/2021,21,2 -22/08/2021,22,2 -23/08/2021,23,2 -24/08/2021,24,2 -25/08/2021,25,2 -26/08/2021,26,2 -27/08/2021,27,2 -28/08/2021,28,2 -29/08/2021,29,2 -30/08/2021,30,2 -31/08/2021,31,2 -01/09/2021,1,2 -02/09/2021,2,2 -03/09/2021,3,2 -04/09/2021,4,2 -05/09/2021,5,2 -06/09/2021,6,2 -07/09/2021,7,2 -08/09/2021,8,2 -09/09/2021,9,2 -10/09/2021,10,2 -11/09/2021,11,2 -12/09/2021,12,2 -13/09/2021,13,2 -14/09/2021,14,2 -15/09/2021,15,2 -16/09/2021,16,2 -17/09/2021,17,2 -18/09/2021,18,2 -19/09/2021,19,2 -20/09/2021,20,2 -21/09/2021,21,2 -22/09/2021,22,2 -23/09/2021,23,2 -24/09/2021,24,2 -25/09/2021,25,2 -26/09/2021,26,2 -27/09/2021,27,2 -28/09/2021,28,2 -29/09/2021,29,2 -30/09/2021,30,2 -01/10/2021,1,2 -02/10/2021,2,2 -03/10/2021,3,2 -04/10/2021,4,2 -05/10/2021,5,2 -06/10/2021,6,2 -07/10/2021,7,2 -08/10/2021,8,2 -09/10/2021,9,2 -10/10/2021,10,2 -11/10/2021,11,2 -12/10/2021,12,2 -13/10/2021,13,2 -14/10/2021,14,2 -15/10/2021,15,2 -16/10/2021,16,2 -17/10/2021,17,2 -18/10/2021,18,2 -19/10/2021,19,2 -20/10/2021,20,2 -21/10/2021,21,2 -22/10/2021,22,2 -23/10/2021,23,2 -24/10/2021,24,2 -25/10/2021,25,2 -26/10/2021,26,2 -27/10/2021,27,2 -28/10/2021,28,2 -29/10/2021,29,2 -30/10/2021,30,2 -31/10/2021,31,2 -01/11/2021,1,2 -02/11/2021,2,2 -03/11/2021,3,2 -04/11/2021,4,2 -05/11/2021,5,2 -06/11/2021,6,2 -07/11/2021,7,2 -08/11/2021,8,2 -09/11/2021,9,2 -10/11/2021,10,2 -11/11/2021,11,2 -12/11/2021,12,2 -13/11/2021,13,2 -14/11/2021,14,2 -15/11/2021,15,2 -16/11/2021,16,2 -17/11/2021,17,2 -18/11/2021,18,2 -19/11/2021,19,2 -20/11/2021,20,2 -21/11/2021,21,2 -22/11/2021,22,2 -23/11/2021,23,2 -24/11/2021,24,2 -25/11/2021,25,2 -26/11/2021,26,2 -27/11/2021,27,2 -28/11/2021,28,2 -29/11/2021,29,2 -30/11/2021,30,2 -01/12/2021,1,2 -02/12/2021,2,2 -03/12/2021,3,2 -04/12/2021,4,2 -05/12/2021,5,2 -06/12/2021,6,2 -07/12/2021,7,2 -08/12/2021,8,2 -09/12/2021,9,2 -10/12/2021,10,2 -11/12/2021,11,2 -12/12/2021,12,2 -13/12/2021,13,2 -14/12/2021,14,2 -15/12/2021,15,2 -16/12/2021,16,2 -17/12/2021,17,2 -18/12/2021,18,2 -19/12/2021,19,2 -20/12/2021,20,2 -21/12/2021,21,2 -22/12/2021,22,2 -23/12/2021,23,2 -24/12/2021,24,2 -25/12/2021,25,2 -26/12/2021,26,2 -27/12/2021,27,2 -28/12/2021,28,2 -29/12/2021,29,2 -30/12/2021,30,2 -31/12/2021,31,2 +date,inflow1 +01/01/2021,1 +02/01/2021,2 +03/01/2021,3 +04/01/2021,4 +05/01/2021,5 +06/01/2021,6 +07/01/2021,7 +08/01/2021,8 +09/01/2021,9 +10/01/2021,10 +11/01/2021,11 +12/01/2021,12 +13/01/2021,13 +14/01/2021,14 +15/01/2021,15 +16/01/2021,16 +17/01/2021,17 +18/01/2021,18 +19/01/2021,19 +20/01/2021,20 +21/01/2021,21 +22/01/2021,22 +23/01/2021,23 +24/01/2021,24 +25/01/2021,25 +26/01/2021,26 +27/01/2021,27 +28/01/2021,28 +29/01/2021,29 +30/01/2021,30 +31/01/2021,31 +01/02/2021,1 +02/02/2021,2 +03/02/2021,3 +04/02/2021,4 +05/02/2021,5 +06/02/2021,6 +07/02/2021,7 +08/02/2021,8 +09/02/2021,9 +10/02/2021,10 +11/02/2021,11 +12/02/2021,12 +13/02/2021,13 +14/02/2021,14 +15/02/2021,15 +16/02/2021,16 +17/02/2021,17 +18/02/2021,18 +19/02/2021,19 +20/02/2021,20 +21/02/2021,21 +22/02/2021,22 +23/02/2021,23 +24/02/2021,24 +25/02/2021,25 +26/02/2021,26 +27/02/2021,27 +28/02/2021,28 +01/03/2021,1 +02/03/2021,2 +03/03/2021,3 +04/03/2021,4 +05/03/2021,5 +06/03/2021,6 +07/03/2021,7 +08/03/2021,8 +09/03/2021,9 +10/03/2021,10 +11/03/2021,11 +12/03/2021,12 +13/03/2021,13 +14/03/2021,14 +15/03/2021,15 +16/03/2021,16 +17/03/2021,17 +18/03/2021,18 +19/03/2021,19 +20/03/2021,20 +21/03/2021,21 +22/03/2021,22 +23/03/2021,23 +24/03/2021,24 +25/03/2021,25 +26/03/2021,26 +27/03/2021,27 +28/03/2021,28 +29/03/2021,29 +30/03/2021,30 +31/03/2021,31 +01/04/2021,1 +02/04/2021,2 +03/04/2021,3 +04/04/2021,4 +05/04/2021,5 +06/04/2021,6 +07/04/2021,7 +08/04/2021,8 +09/04/2021,9 +10/04/2021,10 +11/04/2021,11 +12/04/2021,12 +13/04/2021,13 +14/04/2021,14 +15/04/2021,15 +16/04/2021,16 +17/04/2021,17 +18/04/2021,18 +19/04/2021,19 +20/04/2021,20 +21/04/2021,21 +22/04/2021,22 +23/04/2021,23 +24/04/2021,24 +25/04/2021,25 +26/04/2021,26 +27/04/2021,27 +28/04/2021,28 +29/04/2021,29 +30/04/2021,30 +01/05/2021,1 +02/05/2021,2 +03/05/2021,3 +04/05/2021,4 +05/05/2021,5 +06/05/2021,6 +07/05/2021,7 +08/05/2021,8 +09/05/2021,9 +10/05/2021,10 +11/05/2021,11 +12/05/2021,12 +13/05/2021,13 +14/05/2021,14 +15/05/2021,15 +16/05/2021,16 +17/05/2021,17 +18/05/2021,18 +19/05/2021,19 +20/05/2021,20 +21/05/2021,21 +22/05/2021,22 +23/05/2021,23 +24/05/2021,24 +25/05/2021,25 +26/05/2021,26 +27/05/2021,27 +28/05/2021,28 +29/05/2021,29 +30/05/2021,30 +31/05/2021,31 +01/06/2021,1 +02/06/2021,2 +03/06/2021,3 +04/06/2021,4 +05/06/2021,5 +06/06/2021,6 +07/06/2021,7 +08/06/2021,8 +09/06/2021,9 +10/06/2021,10 +11/06/2021,11 +12/06/2021,12 +13/06/2021,13 +14/06/2021,14 +15/06/2021,15 +16/06/2021,16 +17/06/2021,17 +18/06/2021,18 +19/06/2021,19 +20/06/2021,20 +21/06/2021,21 +22/06/2021,22 +23/06/2021,23 +24/06/2021,24 +25/06/2021,25 +26/06/2021,26 +27/06/2021,27 +28/06/2021,28 +29/06/2021,29 +30/06/2021,30 +01/07/2021,1 +02/07/2021,2 +03/07/2021,3 +04/07/2021,4 +05/07/2021,5 +06/07/2021,6 +07/07/2021,7 +08/07/2021,8 +09/07/2021,9 +10/07/2021,10 +11/07/2021,11 +12/07/2021,12 +13/07/2021,13 +14/07/2021,14 +15/07/2021,15 +16/07/2021,16 +17/07/2021,17 +18/07/2021,18 +19/07/2021,19 +20/07/2021,20 +21/07/2021,21 +22/07/2021,22 +23/07/2021,23 +24/07/2021,24 +25/07/2021,25 +26/07/2021,26 +27/07/2021,27 +28/07/2021,28 +29/07/2021,29 +30/07/2021,30 +31/07/2021,31 +01/08/2021,1 +02/08/2021,2 +03/08/2021,3 +04/08/2021,4 +05/08/2021,5 +06/08/2021,6 +07/08/2021,7 +08/08/2021,8 +09/08/2021,9 +10/08/2021,10 +11/08/2021,11 +12/08/2021,12 +13/08/2021,13 +14/08/2021,14 +15/08/2021,15 +16/08/2021,16 +17/08/2021,17 +18/08/2021,18 +19/08/2021,19 +20/08/2021,20 +21/08/2021,21 +22/08/2021,22 +23/08/2021,23 +24/08/2021,24 +25/08/2021,25 +26/08/2021,26 +27/08/2021,27 +28/08/2021,28 +29/08/2021,29 +30/08/2021,30 +31/08/2021,31 +01/09/2021,1 +02/09/2021,2 +03/09/2021,3 +04/09/2021,4 +05/09/2021,5 +06/09/2021,6 +07/09/2021,7 +08/09/2021,8 +09/09/2021,9 +10/09/2021,10 +11/09/2021,11 +12/09/2021,12 +13/09/2021,13 +14/09/2021,14 +15/09/2021,15 +16/09/2021,16 +17/09/2021,17 +18/09/2021,18 +19/09/2021,19 +20/09/2021,20 +21/09/2021,21 +22/09/2021,22 +23/09/2021,23 +24/09/2021,24 +25/09/2021,25 +26/09/2021,26 +27/09/2021,27 +28/09/2021,28 +29/09/2021,29 +30/09/2021,30 +01/10/2021,1 +02/10/2021,2 +03/10/2021,3 +04/10/2021,4 +05/10/2021,5 +06/10/2021,6 +07/10/2021,7 +08/10/2021,8 +09/10/2021,9 +10/10/2021,10 +11/10/2021,11 +12/10/2021,12 +13/10/2021,13 +14/10/2021,14 +15/10/2021,15 +16/10/2021,16 +17/10/2021,17 +18/10/2021,18 +19/10/2021,19 +20/10/2021,20 +21/10/2021,21 +22/10/2021,22 +23/10/2021,23 +24/10/2021,24 +25/10/2021,25 +26/10/2021,26 +27/10/2021,27 +28/10/2021,28 +29/10/2021,29 +30/10/2021,30 +31/10/2021,31 +01/11/2021,1 +02/11/2021,2 +03/11/2021,3 +04/11/2021,4 +05/11/2021,5 +06/11/2021,6 +07/11/2021,7 +08/11/2021,8 +09/11/2021,9 +10/11/2021,10 +11/11/2021,11 +12/11/2021,12 +13/11/2021,13 +14/11/2021,14 +15/11/2021,15 +16/11/2021,16 +17/11/2021,17 +18/11/2021,18 +19/11/2021,19 +20/11/2021,20 +21/11/2021,21 +22/11/2021,22 +23/11/2021,23 +24/11/2021,24 +25/11/2021,25 +26/11/2021,26 +27/11/2021,27 +28/11/2021,28 +29/11/2021,29 +30/11/2021,30 +01/12/2021,1 +02/12/2021,2 +03/12/2021,3 +04/12/2021,4 +05/12/2021,5 +06/12/2021,6 +07/12/2021,7 +08/12/2021,8 +09/12/2021,9 +10/12/2021,10 +11/12/2021,11 +12/12/2021,12 +13/12/2021,13 +14/12/2021,14 +15/12/2021,15 +16/12/2021,16 +17/12/2021,17 +18/12/2021,18 +19/12/2021,19 +20/12/2021,20 +21/12/2021,21 +22/12/2021,22 +23/12/2021,23 +24/12/2021,24 +25/12/2021,25 +26/12/2021,26 +27/12/2021,27 +28/12/2021,28 +29/12/2021,29 +30/12/2021,30 +31/12/2021,31 diff --git a/pywr-schema/src/test_models/timeseries.json b/pywr-schema/src/test_models/timeseries.json index 130e43c0..6f876a12 100644 --- a/pywr-schema/src/test_models/timeseries.json +++ b/pywr-schema/src/test_models/timeseries.json @@ -25,12 +25,8 @@ "name": "input2", "type": "Input", "max_flow": { - "type": "Timeseries", - "name": "inflow", - "columns": { - "type": "Column", - "name": "inflow2" - } + "type": "Parameter", + "name": "factored_flow" } }, { @@ -66,7 +62,24 @@ "name": "demand", "type": "Constant", "value": 100.0 + }, + { + "name": "factored_flow", + "type": "Aggregated", + "agg_func": "product", + "metrics": [ + { + "type": "Timeseries", + "name": "inflow", + "columns": { + "type": "Column", + "name": "inflow1" + } + }, + 0.5 + ] } + ], "timeseries": [ { diff --git a/pywr-schema/src/timeseries/mod.rs b/pywr-schema/src/timeseries/mod.rs index 2a0b44f7..74edd4fa 100644 --- a/pywr-schema/src/timeseries/mod.rs +++ b/pywr-schema/src/timeseries/mod.rs @@ -235,8 +235,10 @@ mod tests { let mut model = schema.build_model(Some(model_dir.as_path()), None).unwrap(); let expected = Array::from_shape_fn((365, 1), |(x, _)| { - (NaiveDate::from_yo_opt(2021, (x + 1) as u32).unwrap().day() + 2) as f64 + let month_day = NaiveDate::from_yo_opt(2021, (x + 1) as u32).unwrap().day() as f64; + month_day + month_day * 0.5 }); + let idx = model.network().get_node_by_name("output1", None).unwrap().index(); let recorder = AssertionRecorder::new("output-flow", MetricF64::NodeInFlow(idx), expected.clone(), None, None);