feat: Improve conversion error handling & add migration guide.

Cargo.toml

@@ -49,7 +49,7 @@ float-cmp = "0.10"
 ndarray = "0.15"
 polars = { version = "0.44", features = ["lazy", "rows", "ndarray"] }
 pyo3-polars = "0.18"
-pyo3 = { version = "0.21", default-features = false }
+pyo3 = { version = "0.21" }
 pyo3-log = "0.11"
 tracing = { version = "0.1", features = ["log"] }
 csv = "1.3"

pywr-book/py-listings/model-conversion/convert.py

@@ -0,0 +1,93 @@
+import json
+from pathlib import Path
+
+# ANCHOR: convert
+from pywr import (
+    convert_model_from_v1_json_string,
+    ComponentConversionError,
+    ConversionError,
+    Schema,
+)
+
+
+def convert(v1_path: Path):
+    with open(v1_path) as fh:
+        v1_model_str = fh.read()
+    # 1. Convert the v1 model to a v2 schema
+    schema, errors = convert_model_from_v1_json_string(v1_model_str)
+
+    schema_data = json.loads(schema.to_json_string())
+    # 2. Handle any conversion errors
+    for error in errors:
+        handle_conversion_error(error, schema_data)
+
+    # 3. Apply any other manual changes to the converted JSON.
+    patch_model(schema_data)
+
+    schema_data_str = json.dumps(schema_data, indent=4)
+    # 4. Save the converted JSON as a new file (uncomment to save)
+    # with open(v1_path.parent / "v2-model.json", "w") as fh:
+    #     fh.write(schema_data_str)
+    print("Conversion complete; running model...")
+    # 5. Load and run the new JSON file in Pywr v2.x.
+    schema = Schema.from_json_string(schema_data_str)
+    model = schema.build(Path(__file__).parent, None)
+    model.run("clp")
+    print("Model run complete 🎉")
+
+
+# ANCHOR_END: convert
+# ANCHOR: handle_conversion_error
+def handle_conversion_error(error: ComponentConversionError, schema_data):
+    """Handle a schema conversion error.
+
+    Raises a `RuntimeError` if an unhandled error case is found.
+    """
+    match error:
+        case ComponentConversionError.Parameter():
+            match error.error:
+                case ConversionError.UnrecognisedType() as e:
+                    print(
+                        f"Patching custom parameter of type {e.ty} with name {error.name}"
+                    )
+                    handle_custom_parameters(schema_data, error.name, e.ty)
+                case _:
+                    raise RuntimeError(f"Other parameter conversion error: {error}")
+        case ComponentConversionError.Node():
+            raise RuntimeError(f"Failed to convert node `{error.name}`: {error.error}")
+        case _:
+            raise RuntimeError(f"Unexpected conversion error: {error}")
+
+
+def handle_custom_parameters(schema_data, name: str, p_type: str):
+    """Patch the v2 schema to add the custom parameter with `name` and `p_type`."""
+
+    # Ensure the network parameters is a list
+    if schema_data["network"]["parameters"] is None:
+        schema_data["network"]["parameters"] = []
+
+    schema_data["network"]["parameters"].append(
+        {
+            "meta": {"name": name},
+            "type": "Python",
+            "source": {"path": "v2_custom_parameter.py"},
+            "object": p_type,  # Use the same class name in v1 & v2
+            "args": [],
+            "kwargs": {},
+        }
+    )
+
+
+# ANCHOR_END: handle_conversion_error
+# ANCHOR: patch_model
+def patch_model(schema_data):
+    """Patch the v2 schema to add any additional changes."""
+    # Add any additional patches here
+    schema_data["metadata"]["description"] = "Converted from v1 model"
+
+
+# ANCHOR_END: patch_model
+
+if __name__ == "__main__":
+    pth = Path(__file__).parent / "v1-model.json"
+    convert(pth)

pywr-book/py-listings/model-conversion/v1-model.json

@@ -0,0 +1,45 @@
+{
+  "metadata": {
+    "title": "Python include",
+    "description": "An example of including a Python file to define a custom parameter.",
+    "minimum_version": "0.1"
+  },
+  "timestepper": {
+    "start": "2015-01-01",
+    "end": "2015-12-31",
+    "timestep": 1
+  },
+  "nodes": [
+    {
+      "name": "supply1",
+      "type": "Input",
+      "max_flow": "supply1_max_flow"
+    },
+    {
+      "name": "link1",
+      "type": "Link"
+    },
+    {
+      "name": "demand1",
+      "type": "Output",
+      "max_flow": 10,
+      "cost": -10
+    }
+  ],
+  "edges": [
+    [
+      "supply1",
+      "link1"
+    ],
+    [
+      "link1",
+      "demand1"
+    ]
+  ],
+  "parameters": {
+    "supply1_max_flow": {
+      "type": "MyParameter",
+      "value": 15
+    }
+  }
+}

pywr-book/py-listings/model-conversion/v1_custom_parameter.py

@@ -0,0 +1,9 @@
+from pywr.parameters import ConstantParameter
+
+
+class MyParameter(ConstantParameter):
+    def value(self, *args, **kwargs):
+        return 42
+
+
+MyParameter.register()

pywr-book/py-listings/model-conversion/v2_custom_parameter.py

@@ -0,0 +1,3 @@
+class MyParameter:
+    def calc(self, *args, **kwargs):
+        return 42

pywr-book/src/getting_started.md

@@ -9,25 +9,29 @@ TBC
 ## Python
 
 Pywr requires Python 3.9 or later.
-It is currently not available on PyPI, but wheels are available from the GitHub [actions](https://github.com/pywr/pywr-next/actions) page.
+It is currently not available on PyPI, but wheels are available from the
+GitHub [actions](https://github.com/pywr/pywr-next/actions) page.
 Navigate to the latest successful build, and download the archive and extract the wheel for your platform.
 
 ```bash
 pip install pywr-2.0.0b0-cp312-none-win_amd64.whl
 ```
+
 > **Note**: That current Pywr v2.x is in pre-release and may not be suitable for production use.
 > If you require Pywr v1.x please use `pip install pywr<2`.
 
-
 # Running a model
 
 Pywr is a modelling system for simulating water resources systems.
 Models are defined using a JSON schema, and can be run using the `pywr` command line tool.
 Below is an example of a simple model definition `simple1.json`:
 
+[//]: # (@formatter:off)
+
 ```json
-{{#include ../../pywr-schema/src/test_models/simple1.json}}
+{{#include ../../pywr-schema/tests/simple1.json}}
 ```
+[//]: # (@formatter:on)
 
 To run the model, use the `pywr` command line tool:
 

pywr-book/src/migration_guide.md

@@ -0,0 +1,136 @@
+# Migrating from Pywr v1.x
+
+This guide is intended to help users of Pywr v1.x migrate to Pywr v2.x. Pywr v2.x is a complete rewrite of Pywr with a
+new API and new features. This guide will help you update your models to this new version.
+
+## Overview of the process
+
+Pywr v2.x includes a more strict schema for defining models. This schema, along with the
+[pywr-v1-schema](https://crates.io/crates/pywr-v1-schema) crate, provide a way to convert models from v1.x to v2.x.
+However, this process is not perfect and will more than likely require manual intervention to complete the migration.
+
+The overall process will follow these steps:
+
+1. Convert the JSON from v1.x to v2.x using the provided conversion tool.
+2. Handle any errors or warnings from the conversion tool.
+3. Apply any other manual changes to the converted JSON.
+4. (Optional) Save the converted JSON as a new file.
+5. Load and run the new JSON file in Pywr v2.x.
+6. Compare model outputs to ensure it behaves as expected. If necessary, make further changes to the above process and
+   repeat.
+
+## Converting a model
+
+The example below is a basic script that demonstrates how to convert a v1.x model to v2.x. This process converts
+the model at runtime, and does not replace the existing v1.x model with a v2.x definition.
+
+> **Note**: This example is meant to be a starting point for users to build their own conversion process;
+> it is not a complete generic solution.
+
+The function in the listing below is an example of the overall conversion process.
+The function takes a path to a JSON file containing a v1 Pywr model.
+The function reads the JSON, and applies the conversion function (`convert_model_from_v1_json_string`).
+The conversion function that takes a JSON string and returns a tuple of the converted JSON string and a list of errors.
+The function then handles these errors using the `handle_conversion_error` function.
+After the errors are handled other arbitrary changes are applied using the `patch_model` function.
+Finally, the converted JSON can be saved to a new file and run using Pywr v2.x.
+
+[//]: # (@formatter:off)
+
+```python,ignore
+{{ #include ../py-listings/model-conversion/convert.py:convert}}
+```
+
+[//]: # (@formatter:on)
+
+### Handling conversion errors
+
+The `convert_model_from_v1_json_string` function returns a list of errors that occurred during the conversion process.
+These errors can be handled in a variety of ways, such as modifying the model definition, raising exceptions, or
+ignoring them.
+It is suggested to implement a function that can handle these errors in a way that is appropriate for your use case.
+Begin by matching a few types of errors and then expand the matching as needed. By raising exceptions
+for unhandled errors, you can ensure that all errors are eventually accounted, and that new errors are not missed.
+
+The example handles the `ComponentConversionError` by matching on the error subclass (either `Parameter()` or `Node()`),
+and then handling each case separately.
+These two classes will contain the name of the component and optionally the attribute that caused the error.
+In addition, these types contain an inner error (`ConversionError`) that can be used to provide more detailed
+information.
+In the example, the `UnrecognisedType()` class is handled for `Parameter()` errors by applying the
+`handle_custom_parameters` function.
+
+This second function adds a Pywr v2.x compatible custom parameter to the model definition using the same name
+and type (class name) as the original parameter.
+
+[//]: # (@formatter:off)
+
+```python,ignore
+{{ #include ../py-listings/model-conversion/convert.py:handle_conversion_error}}
+```
+
+[//]: # (@formatter:on)
+
+### Other changes
+
+The upgrade to v2.x may require other changes to the model.
+For example, the conversion process does not currently handle recorders and other model outputs.
+These will need to be manually added to the model definition.
+Such manual changes can be applied using, for example a `patch_model` function.
+This function will make arbitrary changes to the model definition.
+The example, below updates the metadata of the model to modify the description.
+
+[//]: # (@formatter:off)
+
+```python,ignore
+{{ #include ../py-listings/model-conversion/convert.py:patch_model}}
+```
+
+[//]: # (@formatter:on)
+
+### Full example
+
+The complete example below demonstrates the conversion process for a v1.x model to v2.x:
+
+[//]: # (@formatter:off)
+
+```python,ignore
+{{ #include ../py-listings/model-conversion/convert.py}}
+```
+
+[//]: # (@formatter:on)
+
+## Converting custom parameters
+
+The main changes to custom parameters in Pywr v2.x are as follows:
+
+1. Custom parameters are no longer required to be a subclass of `Parameter`. They instead can be simple Python classes
+   that implement a `calc` method.
+2. Users are no longer required to handle scenarios within custom parameters. Instead an instance of the custom
+   parameter is created for each scenario in the simulation. This simplifies writing parameters and removes the risk of
+   accidentally contaminating state between scenarios.
+3. Custom parameters are now added to the model using the "Python" parameter type. I.e. the "type" field in the
+   parameter definition should be set to "Python" (not the class name of the custom parameter). This parameter type
+   requires that the user explicitly define which metrics the custom parameter requires.
+
+### Simple example
+
+v1.x custom parameter:
+
+[//]: # (@formatter:off)
+
+```python,ignore
+{{ #include ../py-listings/model-conversion/v1_custom_parameter.py}}
+```
+
+[//]: # (@formatter:on)
+
+v2.x custom parameter:
+
+[//]: # (@formatter:off)
+
+```python,ignore
+{{ #include ../py-listings/model-conversion/v2_custom_parameter.py}}
+```
+
+[//]: # (@formatter:on)

pywr-cli/src/main.rs

@@ -13,7 +13,7 @@ use pywr_core::solvers::{HighsSolver, HighsSolverSettings};
 use pywr_core::solvers::{SimdIpmF64Solver, SimdIpmSolverSettings};
 use pywr_core::test_utils::make_random_model;
 use pywr_schema::model::{PywrModel, PywrMultiNetworkModel, PywrNetwork};
-use pywr_schema::ConversionError;
+use pywr_schema::ComponentConversionError;
 use rand::SeedableRng;
 use rand_chacha::ChaCha8Rng;
 use schemars::schema_for;
@@ -236,7 +236,7 @@ fn v1_to_v2(in_path: &Path, out_path: &Path, stop_on_error: bool, network_only:
     Ok(())
 }
 
-fn handle_conversion_errors(errors: &[ConversionError], stop_on_error: bool) -> Result<()> {
+fn handle_conversion_errors(errors: &[ComponentConversionError], stop_on_error: bool) -> Result<()> {
     if !errors.is_empty() {
         info!("File converted with {} errors:", errors.len());
         for error in errors {

pywr-python/pywr/__init__.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 from typing import Optional
 
-from .pywr import Schema, Model, convert_model_from_v1_json_string, convert_metric_from_v1_json_string  # type: ignore
+from .pywr import *  # type: ignore
 
 
 def run_from_path(

pywr-python/src/lib.rs

@@ -16,7 +16,7 @@ use pywr_core::solvers::{ClpSolver, ClpSolverSettings, ClpSolverSettingsBuilder}
 #[cfg(feature = "highs")]
 use pywr_core::solvers::{HighsSolver, HighsSolverSettings, HighsSolverSettingsBuilder};
 use pywr_schema::model::DateType;
-use pywr_schema::{ConversionData, ConversionError, TryIntoV2};
+use pywr_schema::{ComponentConversionError, ConversionData, ConversionError, TryIntoV2};
 use std::fmt;
 use std::path::PathBuf;
 use std::str::FromStr;
@@ -112,7 +112,7 @@ fn convert_model_from_v1_json_string(py: Python, data: &str) -> PyResult<Py<PyTu
 
     // Create a new schema object
     let py_schema = Schema { schema };
-    let py_errors = errors.into_iter().map(|e| e.to_string()).collect::<Vec<_>>();
+    let py_errors = errors.into_iter().map(|e| e.into_py(py)).collect::<Vec<_>>();
 
     let result = PyTuple::new_bound(py, &[py_schema.into_py(py), py_errors.into_py(py)]).into();
     Ok(result)
@@ -253,5 +253,9 @@ fn pywr(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<Model>()?;
     m.add_class::<Metric>()?;
 
+    // Error classes
+    m.add_class::<ComponentConversionError>()?;
+    m.add_class::<ConversionError>()?;
+
     Ok(())
 }