Added DVC, teamcity-messages and Julia TC integration (#1915)

Fixes #1879 

This does three things
- Adds teamcity-messages as a Python test dependency, enabling TC to
keep track of pytest tests (number of tests, number of failures, linking
to specific logs), making debugging much easier.
<img width="652" alt="Screenshot 2024-10-21 at 22 35 54"
src="https://github.com/user-attachments/assets/96f99290-bab3-4c78-a8ca-e1df6a929258">

- Adds teamcity specific logging in Julia (much like teamcity-messages)
to keep track of the integration test time and differences with a
previous model. These parameters are tracked by TC (and can be failed
on), and can be plotted over time.

<img width="732" alt="Screenshot 2024-10-21 at 22 38 31"
src="https://github.com/user-attachments/assets/7e136e2b-284e-4546-ba1d-fe85cac13a2e">

- Finally, and biggest, this adds [DVC](https://dvc.org/) to our
project, with the minio repo tracked, and the integration test as stage.
It tracks a new default toml for that model (models/integration.toml),
and like TC in 2, the time and differences. Once someone changes either
the hws model, or the core code, or the integration.toml, the results
are outdated and `dvc repro` will rerun the test. We can save these
experiments over time, and compare results with the VS Code plugin:
<img width="1346" alt="Screenshot 2024-10-21 at 22 09 27"
src="https://github.com/user-attachments/assets/f313a159-fa96-4d20-a616-0b808f23210e">


This disables the threshold test in the integration test, and opts for
monitoring results (could be done automatically) instead. I've also
opted for keeping track of results and settings threshold _outside_ of
Julia.

This is a backwards compatible setup PR. Changes/additions for benchmark
tuning can be made in another PR.

---------

Co-authored-by: Martijn Visser <[email protected]>

.dvc/.gitignore

@@ -0,0 +1,3 @@
+/config.local
+/tmp
+/cache

.dvc/config

@@ -0,0 +1,8 @@
+[core]
+    remote = minio
+['remote "minio"']  # for tracking artifacts with unreadable names
+    url = s3://ribasim/dvc
+    endpointurl = https://s3.deltares.nl
+['remote "minio_readonly"']  # for readable named models in the top-dir
+    url = s3://ribasim
+    endpointurl = https://s3.deltares.nl

.dvcignore

@@ -0,0 +1,4 @@
+# Add patterns of files dvc should ignore, which could improve
+# the performance. Learn more at
+# https://dvc.org/doc/user-guide/dvcignore
+docs

.gitignore

@@ -148,7 +148,8 @@ python/ribasim_api/tests/temp/
 report.xml
 
 # Designated working dir for working on Ribasim models
-models/
+models/*
+!models/*.dvc
 playground/output
 
 # Ruff

Manifest.toml

@@ -1398,7 +1398,7 @@ uuid = "295af30f-e4ad-537b-8983-00126c2a3abe"
 version = "3.6.2"
 
 [[deps.Ribasim]]
-deps = ["Accessors", "Arrow", "BasicModelInterface", "CodecZstd", "ComponentArrays", "Configurations", "DBInterface", "DataInterpolations", "DataStructures", "Dates", "DiffEqBase", "DiffEqCallbacks", "EnumX", "FiniteDiff", "Graphs", "HiGHS", "IterTools", "JuMP", "Legolas", "LineSearches", "LinearAlgebra", "LinearSolve", "Logging", "LoggingExtras", "MetaGraphsNext", "OrdinaryDiffEqBDF", "OrdinaryDiffEqCore", "OrdinaryDiffEqLowOrderRK", "OrdinaryDiffEqNonlinearSolve", "OrdinaryDiffEqRosenbrock", "OrdinaryDiffEqSDIRK", "OrdinaryDiffEqTsit5", "PreallocationTools", "SQLite", "SciMLBase", "SparseArrays", "SparseConnectivityTracer", "StructArrays", "Tables", "TerminalLoggers", "TranscodingStreams"]
+deps = ["Accessors", "Arrow", "BasicModelInterface", "CodecZstd", "ComponentArrays", "Configurations", "DBInterface", "DataInterpolations", "DataStructures", "Dates", "DiffEqBase", "DiffEqCallbacks", "EnumX", "FiniteDiff", "Graphs", "HiGHS", "IterTools", "JuMP", "Legolas", "LineSearches", "LinearAlgebra", "LinearSolve", "Logging", "LoggingExtras", "MetaGraphsNext", "OrdinaryDiffEqBDF", "OrdinaryDiffEqCore", "OrdinaryDiffEqLowOrderRK", "OrdinaryDiffEqNonlinearSolve", "OrdinaryDiffEqRosenbrock", "OrdinaryDiffEqSDIRK", "OrdinaryDiffEqTsit5", "PreallocationTools", "SQLite", "SciMLBase", "SparseArrays", "SparseConnectivityTracer", "Statistics", "StructArrays", "Tables", "TerminalLoggers", "TranscodingStreams"]
 path = "core"
 uuid = "aac5e3d9-0b8f-4d4f-8241-b1a7a9632635"
 version = "2024.11.0"

core/Project.toml

@@ -78,8 +78,8 @@ IOCapture = "0.2"
 IterTools = "1.4"
 JuMP = "1.15"
 Legolas = "0.5"
-LinearAlgebra = "1"
 LineSearches = "7"
+LinearAlgebra = "1"
 LinearSolve = "2.24"
 Logging = "1"
 LoggingExtras = "1"
@@ -97,6 +97,7 @@ SQLite = "1.5.1"
 SciMLBase = "2.36"
 SparseArrays = "1"
 SparseConnectivityTracer = "0.6.8"
+Statistics = "1"
 StructArrays = "0.6.13"
 TOML = "1"
 Tables = "1"
@@ -112,6 +113,7 @@ CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 IOCapture = "b5f81e59-6552-4d32-b1f0-c071b021bf89"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 TerminalLoggers = "5d786b92-1e48-4d6f-9151-6b4477ca9bed"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
@@ -121,4 +123,4 @@ TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"
 LoadMKL_JLL = false
 
 [targets]
-test = ["Aqua", "CSV", "DataFrames", "IOCapture", "Logging", "TerminalLoggers", "Test", "TOML", "TestItemRunner"]
+test = ["Aqua", "CSV", "DataFrames", "IOCapture", "Logging", "Statistics", "TerminalLoggers", "Test", "TOML", "TestItemRunner"]

core/integration_test/hws_integration_test.jl

@@ -1,8 +1,12 @@
 @testitem "HWS model integration test" begin
     using SciMLBase: successful_retcode
+    using Dates
+    using Statistics
     using Arrow
+    using TOML
+    include(joinpath(@__DIR__, "../test/utils.jl"))
 
-    toml_path = normpath(@__DIR__, "../../models/hws_2024_7_0/hws.toml")
+    toml_path = normpath(@__DIR__, "../../models/integration.toml")
     @test ispath(toml_path)
     model = Ribasim.run(toml_path)
     @test model isa Ribasim.Model
@@ -12,26 +16,45 @@
         read(normpath(@__DIR__, "../../models/hws_2024_7_0/benchmark/basin_state.arrow"))
     basin_bench = Arrow.Table(basin_bytes_bench)
 
-    basin_bytes = read(normpath(dirname(toml_path), "results/basin_state.arrow"))
+    basin_bytes =
+        read(normpath(dirname(toml_path), model.config.results_dir, "basin_state.arrow"))
     basin = Arrow.Table(basin_bytes)
 
     @testset "Results values" begin
         @test basin.node_id == basin_bench.node_id
         @test all(q -> abs(q) < 0.2, basin.level - basin_bench.level)
     end
 
+    diff = basin.level - basin_bench.level
+
     timed = @timed Ribasim.run(toml_path)
+    dt = Millisecond(round(Int, timed.time * 1000)) + Time(0)
+
+    @tcstatistic "time" timed.time
+    @tcstatistic "min_diff" minimum(diff)
+    @tcstatistic "max_diff" maximum(diff)
+    @tcstatistic "med_diff" median(diff)
+
+    data = Dict(
+        "time" => timed.time,
+        "min_diff" => minimum(diff),
+        "max_diff" => maximum(diff),
+        "med_diff" => median(diff),
+    )
+    open(joinpath(@__DIR__, "../../data/integration.toml"), "w") do io
+        TOML.print(io, data)
+    end
 
     # current benchmark in seconds, TeamCity is up to 4x slower than local
     benchmark_runtime = 32
     performance_diff =
         round((timed.time - benchmark_runtime) / benchmark_runtime * 100; digits = 2)
     if performance_diff < 0.0
         performance_diff = abs(performance_diff)
-        @info "Runtime is $(timed.time) and it is $performance_diff % faster than benchmark"
+        @tcstatus "Runtime is $(dt) and it is $performance_diff % faster than benchmark"
     elseif performance_diff > 0.0 && performance_diff < 0.2
-        @info "Runtime is $(timed.time) and it is $performance_diff % slower than benchmark"
+        @tcstatus "Runtime is $(dt) and it is $performance_diff % slower than benchmark"
     else
-        @warn "Runtime is $(timed.time) and it is $performance_diff % slower than benchmark, close to fail the benchmark"
+        @tcstatus "Runtime is $(dt) and it is $performance_diff % slower than benchmark, close to fail the benchmark"
     end
 end

core/regression_test/regression_test.jl

@@ -55,6 +55,8 @@ end
     using SciMLBase: successful_retcode
     import Arrow
     using Ribasim
+    using Statistics
+    include(joinpath(@__DIR__, "../test/utils.jl"))
 
     toml_path = normpath(@__DIR__, "../../generated_testmodels/basic/ribasim.toml")
     @test ispath(toml_path)
@@ -102,6 +104,14 @@ end
                 # Testbench for basin.arrow
                 @test basin.time == basin_bench.time
                 @test basin.node_id == basin_bench.node_id
+
+                # The storage seems to failing the most, so let's report it for now
+                sdiff = basin.storage - basin_bench.storage
+                key = "basic.$solver.$sparse_on_off.$autodiff_on_off"
+                @tcstatistic "$key.min_diff" minimum(sdiff)
+                @tcstatistic "$key.max_diff" maximum(sdiff)
+                @tcstatistic "$key.med_diff" median(sdiff)
+
                 @test all(q -> abs(q) < 1.0, basin.storage - basin_bench.storage)
                 @test all(q -> abs(q) < 0.5, basin.level - basin_bench.level)
                 @test all(q -> abs(q) < 1e-3, basin.balance_error)

core/src/callback.jl

@@ -234,6 +234,10 @@ function update_cumulative_flows!(u, t, integrator)::Nothing
                 elseif to_node.type == NodeType.UserDemand
                     basin.mass[from_node.idx, :] .-=
                         user_demand.concentration[to_node.idx, :] .* flow
+                elseif to_node.type == NodeType.Terminal && to_node.value == 0
+                    # UserDemand inflow is discoupled from its outflow,
+                    # and the unset flow edge defaults to Terminal #0
+                    nothing
                 else
                     @warn "Unsupported outflow from $(to_node.type) #$(to_node.value) to $(from_node.type) #$(from_node.value) with flow $flow"
                 end

core/test/utils.jl

@@ -0,0 +1,23 @@
+macro tcstatus(message)
+    if haskey(ENV, "TEAMCITY_VERSION")
+        return esc(:(println("##teamcity[buildStatus text='", $message, "']")))
+    else
+        return esc(:(println($message)))
+    end
+end
+
+macro tcstatistic(key, value)
+    if haskey(ENV, "TEAMCITY_VERSION")
+        return esc(
+            :(println(
+                "##teamcity[buildStatisticValue key='",
+                $key,
+                "' value='",
+                $value,
+                "']",
+            )),
+        )
+    else
+        return esc(:(println($key, '=', $value)))
+    end
+end

data/.gitignore

@@ -0,0 +1,3 @@
+/predict.dat
+/integration.csv
+/integration.toml

dvc.lock

@@ -0,0 +1,26 @@
+schema: '2.0'
+stages:
+  integration:
+    cmd: pixi run model-integration-test
+    deps:
+    - path: core/
+      hash: md5
+      md5: 6228f56acef895eeb5d4fdcd479cd1ee.dir
+      size: 474129
+      nfiles: 45
+    - path: models/hws_2024_7_0
+      hash: md5
+      md5: e88123c69a0f7e3590b94c55dbe0062e.dir
+      size: 45421022
+      nfiles: 10
+    params:
+      models/integration.toml:
+        solver.abstol: 1e-07
+        solver.algorithm: QNDF
+        solver.autodiff: false
+        solver.reltol: 1e-07
+    outs:
+    - path: data/integration.toml
+      hash: md5
+      md5: aa6ff2820a6eda91df1073d3bc41755e
+      size: 109

dvc.yaml

@@ -0,0 +1,16 @@
+stages:
+  integration:
+    cmd: pixi run model-integration-test
+    deps:
+      - core/
+      - models/hws_2024_7_0
+    params:
+      - models/integration.toml:
+          - solver.algorithm
+          - solver.abstol
+          - solver.reltol
+          - solver.autodiff
+    outs:
+      - data/integration.toml
+metrics:
+  - data/integration.toml

models/benchmark.dvc

@@ -0,0 +1,14 @@
+md5: 2fac758278d29071bb3e4632d69cbb0c
+frozen: true
+deps:
+- md5: 8620613cb3fc380a20bcbbb2bd36b28c.dir
+  size: 187932
+  nfiles: 6
+  hash: md5
+  path: remote://minio_readonly/benchmark
+outs:
+- md5: ceb5e5e81c402510497da43d29dce099.dir
+  size: 187932
+  nfiles: 6
+  hash: md5
+  path: benchmark

models/hws_2024_7_0.dvc

@@ -0,0 +1,14 @@
+md5: aa976a0658b5e52b211b9714c93b6a34
+frozen: true
+deps:
+- md5: 24169f5a25b4c78f1868b8401a367117.dir
+  size: 43269986
+  nfiles: 4
+  hash: md5
+  path: remote://minio_readonly/hws_2024_7_0
+outs:
+- md5: bc02da0ad6e562f31ae8b61d34884a11.dir
+  size: 43269986
+  nfiles: 4
+  hash: md5
+  path: hws_2024_7_0

models/hws_migration_test.dvc

@@ -0,0 +1,14 @@
+md5: bda2339cc165379b0f6da21aa67039d9
+frozen: true
+deps:
+- md5: ab64e9bee55f9a7524f36ca317ade8fb.dir
+  size: 43307158
+  nfiles: 2
+  hash: md5
+  path: remote://minio_readonly/hws_migration_test
+outs:
+- md5: 6ca6218661d0f021bc53bf468000a62b.dir
+  size: 43307158
+  nfiles: 2
+  hash: md5
+  path: hws_migration_test

models/integration.toml

@@ -0,0 +1,12 @@
+starttime = 2023-01-20 00:00:00
+endtime = 2023-07-01 00:00:00
+crs = "EPSG:28992"
+input_dir = "hws_2024_7_0"
+results_dir = "hws_2024_7_0/results"
+ribasim_version = "2024.10.0"
+
+[solver]
+algorithm = "QNDF" # optional, default "QNDF"
+abstol = 1e-7      # optional, default 1e-7
+reltol = 1e-7      # optional, default 1e-7
+autodiff = false   # optional, default false