Merge branch 'main' into feature/add-turbine-components

pywr · Mar 20, 2024 · f80c77f · f80c77f
2 parents 807f0e4 + a0324a4
commit f80c77f
Show file tree

Hide file tree

Showing 77 changed files with 852 additions and 660 deletions.
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
@@ -14,21 +14,21 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
 
-    - name: Install latest mdbook
-      run: |
-        tag=$(curl 'https://api.github.com/repos/rust-lang/mdbook/releases/latest' | jq -r '.tag_name')
-        url="https://github.com/rust-lang/mdbook/releases/download/${tag}/mdbook-${tag}-x86_64-unknown-linux-gnu.tar.gz"
-        mkdir bin
-        curl -sSL $url | tar -xz --directory=bin
-        echo "$(pwd)/bin" >> $GITHUB_PATH
+      - name: Install latest mdbook
+        run: |
+          tag=$(curl 'https://api.github.com/repos/rust-lang/mdbook/releases/latest' | jq -r '.tag_name')
+          url="https://github.com/rust-lang/mdbook/releases/download/${tag}/mdbook-${tag}-x86_64-unknown-linux-gnu.tar.gz"
+          mkdir bin
+          curl -sSL $url | tar -xz --directory=bin
+          echo "$(pwd)/bin" >> $GITHUB_PATH
 
-    - name: Build
-      run: cargo build --verbose --no-default-features
-    - name: Run tests
-      run: cargo test --no-default-features
-    - name: Run mdbook tests
-      run: mdbook test ./pywr-book
+      - name: Build
+        run: cargo build --verbose --no-default-features --features highs
+      - name: Run tests
+        run: cargo test --no-default-features --features highs
+      - name: Run mdbook tests
+        run: mdbook test ./pywr-book
diff --git a/Cargo.toml b/Cargo.toml
@@ -9,6 +9,7 @@ members = [
     "pywr-schema",
     "pywr-cli",
     "pywr-python",
+    "pywr-schema-macros",
 ]
 exclude = [
     "tests/models/simple-wasm/simple-wasm-parameter"
@@ -38,8 +39,8 @@ thiserror = "1.0.25"
 num = "0.4.0"
 float-cmp = "0.9.0"
 ndarray = "0.15.3"
-polars = { version = "0.37.0", features = ["lazy", "rows", "ndarray"] }
-pyo3-polars = "0.11.1"
+polars = { version = "0.38.1", features = ["lazy", "rows", "ndarray"] }
+pyo3-polars = "0.12.0"
 pyo3 = { version = "0.20.2", default-features = false }
 pyo3-log = "0.9.0"
 tracing = { version = "0.1", features = ["log"] }

diff --git a/pywr-cli/src/main.rs b/pywr-cli/src/main.rs
@@ -1,7 +1,8 @@
 mod tracing;
 
 use crate::tracing::setup_tracing;
-use anyhow::{Context, Result};
+use ::tracing::info;
+use anyhow::Result;
 use clap::{Parser, Subcommand, ValueEnum};
 #[cfg(feature = "ipm-ocl")]
 use pywr_core::solvers::{ClIpmF32Solver, ClIpmF64Solver, ClIpmSolverSettings};
@@ -12,7 +13,6 @@ use pywr_core::solvers::{HighsSolver, HighsSolverSettings};
 use pywr_core::solvers::{SimdIpmF64Solver, SimdIpmSolverSettings};
 use pywr_core::test_utils::make_random_model;
 use pywr_schema::model::{PywrModel, PywrMultiNetworkModel};
-use pywr_schema::ConversionError;
 use rand::SeedableRng;
 use rand_chacha::ChaCha8Rng;
 use std::fmt::{Display, Formatter};
@@ -50,16 +50,9 @@ impl Display for Solver {
 #[derive(Parser)]
 #[command(author, version, about, long_about = None)]
 struct Cli {
-    // /// Optional name to operate on
-    // name: Option<String>,
-    //
-    // /// Sets a custom config file
-    // #[arg(short, long, value_name = "FILE")]
-    // config: Option<PathBuf>,
-    //
-    // /// Turn debugging information on
-    // #[arg(short, long, action = clap::ArgAction::Count)]
-    // debug: u8,
+    /// Turn debugging information on
+    #[arg(long, default_value_t = false)]
+    debug: bool,
     #[command(subcommand)]
     command: Option<Commands>,
 }
@@ -69,6 +62,9 @@ enum Commands {
     Convert {
         /// Path to Pywr v1.x JSON.
         model: PathBuf,
+        /// Stop if there is an error converting the model.
+        #[arg(short, long, default_value_t = false)]
+        stop_on_error: bool,
     },
 
     Run {
@@ -87,8 +83,6 @@ enum Commands {
         /// The number of threads to use in parallel simulation.
         #[arg(short, long, default_value_t = 1)]
         threads: usize,
-        #[arg(long, default_value_t = false)]
-        debug: bool,
     },
     RunMulti {
         /// Path to Pywr model JSON.
@@ -106,8 +100,6 @@ enum Commands {
         /// The number of threads to use in parallel simulation.
         #[arg(short, long, default_value_t = 1)]
         threads: usize,
-        #[arg(long, default_value_t = false)]
-        debug: bool,
     },
     RunRandom {
         num_systems: usize,
@@ -121,28 +113,27 @@ enum Commands {
 
 fn main() -> Result<()> {
     let cli = Cli::parse();
+    setup_tracing(cli.debug).unwrap();
 
     match &cli.command {
         Some(command) => match command {
-            Commands::Convert { model } => convert(model)?,
+            Commands::Convert { model, stop_on_error } => convert(model, *stop_on_error),
             Commands::Run {
                 model,
                 solver,
                 data_path,
                 output_path,
                 parallel: _,
                 threads: _,
-                debug,
-            } => run(model, solver, data_path.as_deref(), output_path.as_deref(), *debug),
+            } => run(model, solver, data_path.as_deref(), output_path.as_deref()),
             Commands::RunMulti {
                 model,
                 solver,
                 data_path,
                 output_path,
                 parallel: _,
                 threads: _,
-                debug,
-            } => run_multi(model, solver, data_path.as_deref(), output_path.as_deref(), *debug),
+            } => run_multi(model, solver, data_path.as_deref(), output_path.as_deref()),
             Commands::RunRandom {
                 num_systems,
                 density,
@@ -156,30 +147,42 @@ fn main() -> Result<()> {
     Ok(())
 }
 
-fn convert(path: &Path) -> Result<()> {
+fn convert(path: &Path, stop_on_error: bool) {
     if path.is_dir() {
         for entry in path.read_dir().expect("read_dir call failed").flatten() {
             let path = entry.path();
             if path.is_file()
                 && (path.extension().unwrap() == "json")
                 && (!path.file_stem().unwrap().to_str().unwrap().contains("_v2"))
             {
-                v1_to_v2(&path).with_context(|| format!("Could not convert model: `{:?}`", &path))?;
+                v1_to_v2(&path, stop_on_error);
             }
         }
     } else {
-        v1_to_v2(path).with_context(|| format!("Could not convert model: `{:?}`", path))?;
+        v1_to_v2(path, stop_on_error);
     }
-
-    Ok(())
 }
 
-fn v1_to_v2(path: &Path) -> std::result::Result<(), ConversionError> {
-    println!("Model: {}", path.display());
+fn v1_to_v2(path: &Path, stop_on_error: bool) {
+    info!("Model: {}", path.display());
 
     let data = std::fs::read_to_string(path).unwrap();
+    // Load the v1 schema
     let schema: pywr_v1_schema::PywrModel = serde_json::from_str(data.as_str()).unwrap();
-    let schema_v2: PywrModel = schema.try_into()?;
+    // Convert to v2 schema and collect any errors
+    let (schema_v2, errors) = PywrModel::from_v1(schema);
+
+    if !errors.is_empty() {
+        info!("Model converted with {} errors:", errors.len());
+        for error in errors {
+            info!("  {}", error);
+        }
+        if stop_on_error {
+            return;
+        }
+    } else {
+        info!("Model converted with zero errors!");
+    }
 
     // There must be a better way to do this!!
     let mut new_file_name = path.file_stem().unwrap().to_os_string();
@@ -189,13 +192,9 @@ fn v1_to_v2(path: &Path) -> std::result::Result<(), ConversionError> {
     let new_file_pth = path.parent().unwrap().join(new_file_name);
 
     std::fs::write(new_file_pth, serde_json::to_string_pretty(&schema_v2).unwrap()).unwrap();
-
-    Ok(())
 }
 
-fn run(path: &Path, solver: &Solver, data_path: Option<&Path>, output_path: Option<&Path>, debug: bool) {
-    setup_tracing(debug).unwrap();
-
+fn run(path: &Path, solver: &Solver, data_path: Option<&Path>, output_path: Option<&Path>) {
     let data = std::fs::read_to_string(path).unwrap();
     let data_path = data_path.or_else(|| path.parent());
     let schema_v2: PywrModel = serde_json::from_str(data.as_str()).unwrap();
@@ -216,9 +215,7 @@ fn run(path: &Path, solver: &Solver, data_path: Option<&Path>, output_path: Opti
     .unwrap();
 }
 
-fn run_multi(path: &Path, solver: &Solver, data_path: Option<&Path>, output_path: Option<&Path>, debug: bool) {
-    setup_tracing(debug).unwrap();
-
+fn run_multi(path: &Path, solver: &Solver, data_path: Option<&Path>, output_path: Option<&Path>) {
     let data = std::fs::read_to_string(path).unwrap();
     let data_path = data_path.or_else(|| path.parent());
 

diff --git a/pywr-core/Cargo.toml b/pywr-core/Cargo.toml
@@ -18,20 +18,19 @@ libc = "0.2.97"
 thiserror = { workspace = true }
 ndarray = { workspace = true }
 num = { workspace = true }
-float-cmp = {  workspace = true }
+float-cmp = { workspace = true }
 hdf5 = { workspace = true }
 csv = { workspace = true }
 clp-sys = { path = "../clp-sys", version = "0.1.0" }
 ipm-ocl = { path = "../ipm-ocl", optional = true }
 ipm-simd = { path = "../ipm-simd", optional = true }
-tracing = {  workspace = true }
-highs-sys = {  git = "https://github.com/jetuk/highs-sys", branch="fix-build-libz-linking", optional = true }
-# highs-sys = { path = "../../highs-sys" }
+tracing = { workspace = true }
+highs-sys = { version = "1.6.2", optional = true }
 nalgebra = "0.32.3"
 chrono = { workspace = true }
 polars = { workspace = true }
 
-pyo3 = {  workspace = true, features = ["chrono"] }
+pyo3 = { workspace = true, features = ["chrono"] }
 
 
 rayon = "1.6.1"

diff --git a/pywr-core/src/lib.rs b/pywr-core/src/lib.rs
@@ -5,8 +5,9 @@ extern crate core;
 use crate::derived_metric::DerivedMetricIndex;
 use crate::models::MultiNetworkTransferIndex;
 use crate::node::NodeIndex;
-use crate::parameters::{IndexParameterIndex, InterpolationError, MultiValueParameterIndex, ParameterIndex};
+use crate::parameters::{InterpolationError, ParameterIndex};
 use crate::recorders::{AggregationError, MetricSetIndex, RecorderIndex};
+use crate::state::MultiValue;
 use crate::virtual_storage::VirtualStorageIndex;
 use pyo3::exceptions::{PyException, PyRuntimeError};
 use pyo3::{create_exception, PyErr};
@@ -45,11 +46,11 @@ pub enum PywrError {
     #[error("virtual storage index {0} not found")]
     VirtualStorageIndexNotFound(VirtualStorageIndex),
     #[error("parameter index {0} not found")]
-    ParameterIndexNotFound(ParameterIndex),
+    ParameterIndexNotFound(ParameterIndex<f64>),
     #[error("index parameter index {0} not found")]
-    IndexParameterIndexNotFound(IndexParameterIndex),
+    IndexParameterIndexNotFound(ParameterIndex<usize>),
     #[error("multi1 value parameter index {0} not found")]
-    MultiValueParameterIndexNotFound(MultiValueParameterIndex),
+    MultiValueParameterIndexNotFound(ParameterIndex<MultiValue>),
     #[error("multi1 value parameter key {0} not found")]
     MultiValueParameterKeyNotFound(String),
     #[error("inter-network parameter state not initialised")]
@@ -73,9 +74,9 @@ pub enum PywrError {
     #[error("node name `{0}` already exists")]
     NodeNameAlreadyExists(String),
     #[error("parameter name `{0}` already exists at index {1}")]
-    ParameterNameAlreadyExists(String, ParameterIndex),
+    ParameterNameAlreadyExists(String, ParameterIndex<f64>),
     #[error("index parameter name `{0}` already exists at index {1}")]
-    IndexParameterNameAlreadyExists(String, IndexParameterIndex),
+    IndexParameterNameAlreadyExists(String, ParameterIndex<usize>),
     #[error("metric set name `{0}` already exists")]
     MetricSetNameAlreadyExists(String),
     #[error("recorder name `{0}` already exists at index {1}")]
@@ -159,7 +160,7 @@ pub enum PywrError {
     #[error("parameters do not provide an initial value")]
     ParameterNoInitialValue,
     #[error("parameter state not found for parameter index {0}")]
-    ParameterStateNotFound(ParameterIndex),
+    ParameterStateNotFound(ParameterIndex<f64>),
     #[error("Could not create timestep range due to following error: {0}")]
     TimestepRangeGenerationError(String),
     #[error("Could not create timesteps for frequency '{0}'")]

diff --git a/pywr-core/src/metric.rs b/pywr-core/src/metric.rs
@@ -5,8 +5,8 @@ use crate::edge::EdgeIndex;
 use crate::models::MultiNetworkTransferIndex;
 use crate::network::Network;
 use crate::node::NodeIndex;
-use crate::parameters::{IndexParameterIndex, MultiValueParameterIndex, ParameterIndex};
-use crate::state::State;
+use crate::parameters::ParameterIndex;
+use crate::state::{MultiValue, State};
 use crate::virtual_storage::VirtualStorageIndex;
 use crate::PywrError;
 #[derive(Clone, Debug, PartialEq)]
@@ -18,8 +18,8 @@ pub enum Metric {
     AggregatedNodeOutFlow(AggregatedNodeIndex),
     AggregatedNodeVolume(AggregatedStorageNodeIndex),
     EdgeFlow(EdgeIndex),
-    ParameterValue(ParameterIndex),
-    MultiParameterValue((MultiValueParameterIndex, String)),
+    ParameterValue(ParameterIndex<f64>),
+    MultiParameterValue((ParameterIndex<MultiValue>, String)),
     VirtualStorageVolume(VirtualStorageIndex),
     MultiNodeInFlow { indices: Vec<NodeIndex>, name: String },
     MultiNodeOutFlow { indices: Vec<NodeIndex>, name: String },
@@ -87,7 +87,7 @@ impl Metric {
 
 #[derive(Clone, Debug, PartialEq)]
 pub enum IndexMetric {
-    IndexParameterValue(IndexParameterIndex),
+    IndexParameterValue(ParameterIndex<usize>),
     Constant(usize),
 }