Merge branch 'main' into rbf-parameter

.github/workflows/rust.yml → .github/workflows/linux.yml

@@ -1,4 +1,4 @@
-name: Rust
+name: Rust (Linux)
 
 on:
   push:
@@ -17,10 +17,6 @@ jobs:
     - uses: actions/checkout@v4
       with:
         submodules: true
-    - name: Install HDF5
-      run: |
-        sudo apt-get update
-        sudo apt-get install libhdf5-dev ocl-icd-opencl-dev zlib1g-dev
     - name: Build
       run: cargo build --verbose --no-default-features
     - name: Run tests

.github/workflows/python.yml

@@ -1,4 +1,4 @@
-name: Python
+name: Python (Linux)
 
 on:
   push:

.github/workflows/windows.yml

@@ -0,0 +1,23 @@
+name: Rust (Windows)
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+
+    runs-on: windows-latest
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - name: Build
+      run: cargo build --verbose --no-default-features
+    - name: Run tests
+      run: cargo test --no-default-features

Cargo.toml

@@ -1,93 +1,27 @@
-[package]
-name = "pywr"
-version = "2.0.0-dev"
-authors = ["James Tomlinson <[email protected]>"]
-edition = "2021"
-rust-version = "1.60"
-description = "A generalised water resource allocation model."
-readme = "README.md"
-repository = "https://github.com/pywr/pywr-next/"
-license = "MIT OR Apache-2.0"
-license-file = "LICENSE"
-keywords = ["water", "modelling"]
-categories = ["science", "simulation"]
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-libc = "0.2.97"
-thiserror = "1.0.25"
-ndarray = "0.15.3"
-numpy = "0.19.0"
-num = "0.4.0"
-float-cmp = "0.9.0"
-hdf5 = { version="0.8.1" }
-csv = "1.1"
-clp-sys = { path = "./clp-sys" }
-ipm-ocl = { path = "./ipm-ocl", optional = true }
-ipm-simd = { path = "./ipm-simd", optional = true }
-wasmer = "4.0.0"
-serde = { version = "1", features = ["derive"] }
-serde_json = "1.0"
-time = { version = "0.3", features = ["serde", "serde-well-known", "serde-human-readable", "macros"] }
-svgbobdoc = { version = "0.3.0", features = ["enable"]  }
-tracing = "0.1"
-tracing-subscriber = { version ="0.3.17", features=["env-filter"] }
-pyo3-log = "0.8.0"
-indicatif = "0.17.2"
-highs-sys = {  git = "https://github.com/jetuk/highs-sys", branch="fix-build-libz-linking", optional = true }
-# highs-sys = { path = "../../highs-sys" }
-pyo3 = { version = "0.19.0" }
-rayon = "1.6.1"
-polars = { version = "0.33.2", features = ["lazy", "rows", "ndarray"] }
-pyo3-polars = "0.7.0"
-pywr-schema = { git = "https://github.com/pywr/pywr-schema/", tag="v0.7.0" }
-rhai = { version="1.12.0", features=["sync"] }
-nalgebra = "0.32.3"
-
-# OpenCL
-ocl = { version = "0.19", optional = true }
-
-# Binary dependencies
-clap =  { version="4.0", features=["derive"] }
-anyhow = "1.0.69"
-
-rand = "0.8.5"
-rand_distr = "0.4.3"
-rand_chacha = "0.3.1"
-
-[dev-dependencies]
-tempfile = "3.3.0"
-criterion = "0.5"
-
-[lib]
-name = "pywr"
-path = "src/lib.rs"
-crate-type = ["cdylib", "rlib"]
-
-
-[features]
-extension-module = ["pyo3/extension-module"]
-default = ["extension-module"]
-highs = ["dep:highs-sys"]
-ipm-ocl = ["dep:ipm-ocl", "dep:ocl"]
-ipm-simd = ["dep:ipm-simd"]
-
-[[bin]]
-name = "pywr-cli"
-path = "src/main.rs"
-
-
 [workspace]
+resolver = "2"
 members = [
     "ipm-common",
     "ipm-ocl",
     "ipm-simd",
     "clp-sys",
+    "pywr-core",
+    "pywr-schema",
+    "pywr-cli",
+    "pywr-python",
 ]
 exclude = [
     "tests/models/simple-wasm/simple-wasm-parameter"
 ]
+# IPM packages are not default because they require nightly (portable_simd).
+default-members = [
+    "clp-sys",
+    "pywr-core",
+    "pywr-schema",
+    "pywr-cli",
+    "pywr-python",
+]
+
 
 [profile.release]
 opt-level = 3 # fast and small wasm
@@ -97,6 +31,16 @@ opt-level = 3 # fast and small wasm
 # debug = true
 
 
-[[bench]]
-name = "random_models"
-harness = false
+[workspace.dependencies]
+serde = { version = "1", features = ["derive"] }
+serde_json = "1.0"
+thiserror = "1.0.25"
+time = { version = "0.3", features = ["serde", "serde-well-known", "serde-human-readable", "macros"] }
+num = "0.4.0"
+ndarray = "0.15.3"
+pyo3 = { version = "0.19.0" }
+tracing = "0.1"
+csv = "1.1"
+hdf5 = { version="0.8.1" }
+hdf5-sys = { version="0.8.1", features=["static"] }
+pywr-v1-schema = { git = "https://github.com/pywr/pywr-schema/", tag="v0.7.0", package = "pywr-schema" }

README.md

@@ -117,10 +117,11 @@ git submodule init
 git submodule update
 ```
 
-Rust is required for installation. To create a development installation requires first compiling the
-Rust library and then installing the Python package in editable model.
+Rust is required for installation of the Python extension. To create a Python development installation 
+requires first compiling the Rust library and then installing the Python package in editable model.
 
 ```bash
+cd pywr-python
 maturin develop
 pip install -e .
 ```
@@ -143,19 +144,19 @@ python -m pywr
 
 ### Rust CLI
 
-A basic command line interface is included such that you can use this version of Pywr without Python. Running this
-currently requires disabling the default features that create the Python extension.
+A basic command line interface is included such that you can use this version of Pywr without Python.
+This CLI is in the `pywr-cli` crate. 
 
 To see the CLI commands available run the following:
 
 ```bash
-cargo run --no-default-features -- --help
+cargo run -p pywr-cli -- --help
 ```
 
 To run a Pywr v2 model use the following:
 
 ```bash
-cargo run --no-default-features -- run tests/models/simple1.json
+cargo run -p pywr-cli -- run tests/models/simple1.json
 ```
 
 ### Python CLI
@@ -191,6 +192,14 @@ Feedback on porting models is very welcome, so please open an issue with any que
 
 <p align="right">(<a href="#readme-top">back to top</a>)</p>
 
+<!-- CRATES -->
+## Crates
+This repository contains the following crates:
+
+- `pywr-core`: A low-level Rust library for constructing network models. This crate interfaces with linear program solvers.
+- `pywr-schema`: A Rust library for validating Pywr JSON files against a schema, and then building a model from the schema using `pywr-core`.
+- `pywr-cli`: A command line interface for running Pywr models.
+- `pywr-python`: A Python package (and extension) for constructing and running Pywr models.
 
 
 <!-- ROADMAP -->

clp-sys/build.rs

@@ -1,12 +1,44 @@
+use std::env;
+
+fn make_builder() -> cc::Build {
+    let target = env::var("TARGET").expect("Could not find TARGET in environment.");
+    let mut builder = cc::Build::new()
+        .cpp(true)
+        .warnings(false)
+        .extra_warnings(false)
+        .define("NDEBUG", None)
+        .define("HAVE_STDIO_H", None)
+        .define("HAVE_STDLIB_H", None)
+        .define("HAVE_STRING_H", None)
+        .define("HAVE_INTTYPES_H", None)
+        .define("HAVE_STDINT_H", None)
+        .define("HAVE_STRINGS_H", None)
+        .define("HAVE_SYS_TYPES_H", None)
+        .define("HAVE_SYS_STAT_H", None)
+        .define("HAVE_UNISTD_H", None)
+        .define("HAVE_CMATH", None)
+        .define("HAVE_CFLOAT", None)
+        // .define("HAVE_DLFCN_H", None)
+        .define("HAVE_MEMORY_H", None)
+        .to_owned();
+
+    if target.contains("msvc") {
+        builder.flag("-EHsc").flag_if_supported("-std:c++11");
+    } else {
+        builder.flag("-std=c++11").flag("-w");
+    }
+
+    builder
+}
+
 fn main() {
-    const COIN_UTILS_SRC: &str = "vendor/CoinUtils/CoinUtils/src";
-    const COIN_CLP_SRC: &str = "vendor/Clp/Clp/src";
+    const COIN_UTILS_SRC: &str = "vendor/CoinUtils/src";
+    const COIN_CLP_SRC: &str = "vendor/Clp/src";
+
     // Compile CoinUtils
-    cc::Build::new()
-        .cpp(true)
-        .flag("-w")
-        .flag("-DNDEBUG")
-        .flag("-DHAVE_CFLOAT")
+    let mut builder = make_builder();
+
+    builder
         .flag(&*format!("-I{}", COIN_UTILS_SRC))
         .file(format!("{}/CoinAlloc.cpp", COIN_UTILS_SRC))
         .file(format!("{}/CoinBuild.cpp", COIN_UTILS_SRC))
@@ -69,14 +101,11 @@ fn main() {
 
     // Compile CoinUtils
 
-    cc::Build::new()
-        .cpp(true)
-        .flag("-w")
+    let mut builder = make_builder();
+
+    builder
         .flag(&*format!("-I{}", COIN_UTILS_SRC))
         .flag(&*format!("-I{}", COIN_CLP_SRC))
-        .flag("-DNDEBUG")
-        .flag("-DHAVE_CFLOAT")
-        .flag("-DHAVE_CMATH")
         .file(format!("{}/ClpCholeskyBase.cpp", COIN_CLP_SRC))
         .file(format!("{}/ClpCholeskyDense.cpp", COIN_CLP_SRC))
         .file(format!("{}/ClpCholeskyPardiso.cpp", COIN_CLP_SRC))
@@ -132,7 +161,7 @@ fn main() {
         .file(format!("{}/ClpSimplexOther.cpp", COIN_CLP_SRC))
         .file(format!("{}/ClpSimplexPrimal.cpp", COIN_CLP_SRC))
         .file(format!("{}/ClpSolve.cpp", COIN_CLP_SRC))
-        .file(format!("{}/ClpSolver.cpp", COIN_CLP_SRC))
+        // .file(format!("{}/ClpSolver.cpp", COIN_CLP_SRC))
         // .file(format!("{}/CoinAbcBaseFactorization1.cpp", COIN_CLP_SRC))
         // .file(format!("{}/CoinAbcBaseFactorization2.cpp", COIN_CLP_SRC))
         // .file(format!("{}/CoinAbcBaseFactorization3.cpp", COIN_CLP_SRC))

pywr-cli/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "pywr-cli"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+clap =  { version="4.0", features=["derive"] }
+anyhow = "1.0.69"
+
+rand = "0.8.5"
+rand_chacha = "0.3.1"
+time = {  workspace = true, features = ["serde", "serde-well-known", "serde-human-readable", "macros"] }
+serde = { workspace = true }
+serde_json = { workspace = true }
+pywr-v1-schema = { workspace = true }
+
+pywr-core = { path = "../pywr-core" }
+pywr-schema = { path = "../pywr-schema" }

src/main.rs → pywr-cli/src/main.rs

@@ -1,19 +1,19 @@
 use anyhow::{Context, Result};
 use clap::{Parser, Subcommand, ValueEnum};
-use pywr::model::Model;
-use pywr::schema::model::PywrModel;
-use pywr::schema::ConversionError;
+use pywr_core::model::Model;
 #[cfg(feature = "ipm-ocl")]
-use pywr::solvers::{ClIpmF32Solver, ClIpmF64Solver, ClIpmSolverSettings};
-use pywr::solvers::{ClpSolver, ClpSolverSettings};
+use pywr_core::solvers::{ClIpmF32Solver, ClIpmF64Solver, ClIpmSolverSettings};
+use pywr_core::solvers::{ClpSolver, ClpSolverSettings};
 #[cfg(feature = "highs")]
-use pywr::solvers::{HighsSolver, HighsSolverSettings};
+use pywr_core::solvers::{HighsSolver, HighsSolverSettings};
 #[cfg(feature = "ipm-simd")]
-use pywr::solvers::{SimdIpmF64Solver, SimdIpmSolverSettings};
-use pywr::test_utils::make_random_model;
-use pywr::timestep::Timestepper;
-use pywr::tracing::setup_tracing;
-use pywr::PywrError;
+use pywr_core::solvers::{SimdIpmF64Solver, SimdIpmSolverSettings};
+use pywr_core::test_utils::make_random_model;
+use pywr_core::timestep::Timestepper;
+use pywr_core::tracing::setup_tracing;
+use pywr_core::PywrError;
+use pywr_schema::model::PywrModel;
+use pywr_schema::ConversionError;
 use rand::SeedableRng;
 use rand_chacha::ChaCha8Rng;
 use std::fmt::{Display, Formatter};
@@ -138,15 +138,11 @@ fn convert(path: &Path) -> Result<()> {
                 && (path.extension().unwrap() == "json")
                 && (!path.file_stem().unwrap().to_str().unwrap().contains("_v2"))
             {
-                v1_to_v2(&path)
-                    .map_err(PywrError::Conversion)
-                    .with_context(|| format!("Could not convert model: `{:?}`", &path))?;
+                v1_to_v2(&path).with_context(|| format!("Could not convert model: `{:?}`", &path))?;
             }
         }
     } else {
-        v1_to_v2(path)
-            .map_err(PywrError::Conversion)
-            .with_context(|| format!("Could not convert model: `{:?}`", path))?;
+        v1_to_v2(path).with_context(|| format!("Could not convert model: `{:?}`", path))?;
     }
 
     Ok(())
@@ -156,7 +152,7 @@ fn v1_to_v2(path: &Path) -> std::result::Result<(), ConversionError> {
     println!("Model: {}", path.display());
 
     let data = std::fs::read_to_string(path).unwrap();
-    let schema: pywr_schema::PywrModel = serde_json::from_str(data.as_str()).unwrap();
+    let schema: pywr_v1_schema::PywrModel = serde_json::from_str(data.as_str()).unwrap();
     let schema_v2: PywrModel = schema.try_into()?;
 
     // There must be a better way to do this!!