Merge branch 'main' of https://github.com/olivierbonte/HydroModels.jl

.gitignore

@@ -18,11 +18,15 @@ docs/build/
 docs/site/
 docs/tutorials
 docs/start
+docs/code
 docs/.vscode
 
 # Files generated by invoking Julia with --code-coverage
 *.jl.cov
 *.jl.*.cov
 
 # Files generated by invoking Julia with --track-allocation
-*.jl.mem
+*.jl.mem
+
+
+log

Project.toml

@@ -1,7 +1,7 @@
 name = "HydroModels"
 uuid = "7e3cde01-c141-467b-bff6-5350a0af19fc"
 authors = ["jingx <[email protected]>"]
-version = "0.1.1"
+version = "0.1.2"
 
 [deps]
 Accessors = "7d9f7c33-5ae7-4f3b-8dc6-eff91059b697"

docs/Manifest.toml

@@ -2,7 +2,7 @@
 
 julia_version = "1.10.4"
 manifest_format = "2.0"
-project_hash = "620786bb65c2992bc73165cb134818fb2070b24e"
+project_hash = "6e29b825734117b406b953711ef9188548ec6139"
 
 [[deps.ADTypes]]
 git-tree-sha1 = "72af59f5b8f09faee36b4ec48e014a79210f2f4f"
@@ -1056,10 +1056,10 @@ uuid = "e33a78d0-f292-5ffc-b300-72abe9b543c8"
 version = "2.11.2+1"
 
 [[deps.HydroModels]]
-deps = ["Accessors", "CSV", "ComponentArrays", "DataFrames", "DataInterpolations", "Dates", "ForwardDiff", "Graphs", "Integrals", "IterTools", "LinearAlgebra", "LinearSolve", "Lux", "LuxCore", "ModelingToolkit", "NNlib", "NamedTupleTools", "Optimization", "OptimizationBBO", "OptimizationOptimisers", "OrdinaryDiffEq", "ProgressMeter", "Random", "Reexport", "RuntimeGeneratedFunctions", "SciMLBase", "SciMLSensitivity", "SparseArrays", "StableRNGs", "Statistics", "SymbolicUtils", "Symbolics", "TOML", "Zygote"]
-path = "E:\\JlCode\\HydroModels"
+deps = ["Accessors", "ComponentArrays", "DataInterpolations", "Graphs", "Integrals", "LinearAlgebra", "LinearSolve", "Lux", "LuxCore", "ModelingToolkit", "NNlib", "Random", "Reexport", "RuntimeGeneratedFunctions", "SparseArrays", "StableRNGs", "SymbolicUtils", "Symbolics", "TOML"]
+git-tree-sha1 = "72a42f12341cd13b0a5eaa32197c91639a2b8cbf"
 uuid = "7e3cde01-c141-467b-bff6-5350a0af19fc"
-version = "0.1.0"
+version = "0.1.1"
 
 [[deps.HypergeometricFunctions]]
 deps = ["LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"]
@@ -1173,11 +1173,6 @@ git-tree-sha1 = "630b497eafcc20001bba38a4651b327dcfc491d2"
 uuid = "92d709cd-6900-40b7-9082-c6be49f344b6"
 version = "0.2.2"
 
-[[deps.IterTools]]
-git-tree-sha1 = "42d5f897009e7ff2cf88db414a389e5ed1bdd023"
-uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e"
-version = "1.10.0"
-
 [[deps.IteratorInterfaceExtensions]]
 git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856"
 uuid = "82899510-4779-5014-852e-03e436cf321d"

docs/Project.toml

@@ -6,6 +6,7 @@ ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 DataInterpolations = "82cc6244-b520-54b8-b5a6-8a565e85f1d0"
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
 HydroModels = "7e3cde01-c141-467b-bff6-5350a0af19fc"
 JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
 Lux = "b2108857-7c20-44ae-9111-449ecde12c47"

docs/make.jl

@@ -2,6 +2,7 @@ push!(LOAD_PATH, "../src/")
 using Documenter
 using HydroModels
 
+# English Documentation
 makedocs(
     sitename = "HydroModels.jl",
     authors = "xin jing",
@@ -19,32 +20,82 @@ makedocs(
     doctest = false,
     linkcheck = true,
     source = "src",
-    build = "build",
+    build = "build_en",
+    warnonly = true,
     # 配置页面结构
     pages = [
         "Home" => "index.md",
-        "Model Implementations" => [
-            "construct a ExpHydro Model" => "implements/build_exphydro_model_en.md",
-            "construct a M50 Model" => "implements/build_m50_model_en.md",
-            "construct discharge route model" => "implements/build_discharge_route_en.md",
-        ],
         "Run Models" => [
-            "Run a Bucket Model" => "tutorials/run_a_bucket.md",
-            "Run ExpHydro Model" => "tutorials/run_a_exphydro_model.md"
+            "Run ExpHydro Model" => "tutorials/run_exphydro_model.md",
         ],
-        "Extent Content" => [
-            "Modeling Framework Comparisons" => "extent/framework_comparision_en.md",
+        "Basic Concepts" => "concepts_en.md",
+        "Model Implementations" => [
+            "construct the ExpHydro Model" => "implements/build_exphydro_model_en.md",
+            "construct the M50 Model" => "implements/build_m50_model_en.md",
+            "construct the discharge route model" => "implements/build_discharge_route_en.md",
         ],
-
-    ],
-    # 其他选项
-    checkdocs = :none,
+        "Extend Contents" => [
+            "Why not using ModelingToolkit.jl directly" => "extent/why_not_MTK_en.md",
+            "Framework Comparision" => "extent/framework_comparision_en.md",
+        ]
+    ]
 )
 
+# # Chinese Documentation
+# makedocs(
+#     sitename = "HydroModels.jl",
+#     authors = "xin jing",
+#     format = Documenter.HTML(
+#         # 启用 pretty URLs，移除 .html 后缀
+#         # 设置文档的规范 URL
+#         canonical = "https://chooron.github.io/HydroModels.jl/dev",
+#         # 配置侧边栏
+#         collapselevel = 2,
+#         sidebar_sitename = true
+#     ),
+#     # 配置模块
+#     modules = [HydroModels],
+#     clean = true,
+#     doctest = false,
+#     linkcheck = true,
+#     source = "src",
+#     build = "build_zh",
+#     lang = "zh",
+#     # 配置页面结构
+#     pages = [
+#         "主页" => "index.md",
+#         "入门指南" => "getting_started.md",
+#         "基本概念" => [
+#             "蓄水模型" => "concepts/bucket_model.md",
+#             "通量模型" => "concepts/flux_model.md",
+#             "汇流模型" => "concepts/route_model.md",
+#             "求解器模型" => "concepts/solver_model.md",
+#             "分布式模型" => "concepts/spatial_model.md",
+#             "集总式模型" => "concepts/lumped_model.md",
+#         ],
+#         "模型构建" => [
+#             "构建 ExpHydro 模型" => "implements/build_exphydro_model_zh.md",
+#             "构建 M50 模型" => "implements/build_m50_model_zh.md",
+#             "构建河道汇流模型" => "implements/build_discharge_route_zh.md",
+#         ],
+#         "模型运行" => [
+#             "运行蓄水模型" => "tutorials/run_a_bucket.md",
+#             "运行 ExpHydro 模型" => "tutorials/run_a_exphydro_model.md"
+#         ]
+#     ]
+# )
+
 # 部署配置
 deploydocs(
     repo = "github.com/chooron/HydroModels.jl",
     devbranch = "main",
     push_preview = true,
-    target = "build"  # 确保这里指定了正确的构建目录
-)
+    target = "build_en"  # 确保这里指定了正确的构建目录
+)
+
+# deploydocs(
+#     repo = "github.com/chooron/HydroModels.jl",
+#     devbranch = "main",
+#     push_preview = true,
+#     target = "build_zh"  # 确保这里指定了正确的构建目录
+# )

docs/src/concepts_en.md

@@ -0,0 +1,67 @@
+# HydroModels.jl Concepts
+
+HydroModels.jl implements a modular and extensible architecture, designed to support diverse hydrological modeling paradigms. The framework's core structure comprises four main classes: Flux, Bucket, Route, Model and Wrapper, each playing a vital role in constructing comprehensive hydrological systems. The framework is implemented in Julia programming language for model development, computation, and parameter optimization. It leverages Lux.jl as the deep learning framework and integrates with SciML for scientific computing, symbolic programming, and parameter optimization capabilities. The conceptual framework of the model design philosophy is illustrated in the following diagram:
+
+![Framework Concept](assets/concept.jpg)
+Architecture and ecosystem of the HydroModels.jl framework. (a) illustrates the supporting ecosystem and its functional capabilities, highlighting the key dependencies and their roles in the framework; (b) shows the core architectural design of HydroModels.jl, demonstrating the main components and their interactions.
+
+## Flux class: Water Process Transfer Representation
+
+The Flux class serves as the fundamental building block of HydroModels.jl, drawing inspiration from flux library of the MARRMoT while extending its capabilities. This class encapsulates the physical equations governing water movement throughout the hydrological cycle. The conceptual formulation of the Flux class can be expressed as:
+
+
+```math
+\begin{aligned}
+Y(t) = f(X(t),\theta) && (1) \\
+\end{aligned}
+```
+
+where the functional representation $f$ maps input variables $X(t)$ and parameters $\theta$ to output variables $Y(t)$. Here, $X(t)$ represents input variables like precipitation and temperature, $\theta$ denotes the parameters that can be calibrated, and $Y(t)$ represents output variables like runoff and infiltration. Based on different applications, the Flux class includes:
+
+- HydroFlux: Implements fundamental hydrological processes through mathematical formulations
+- StateFlux: Handles mass-balance equations for state variables like soil moisture and snowpack
+- NeuralFlux: Uses neural networks to model processes or predict parameters, leveraging Lux.jl capabilities
+
+## Bucket Class: Storage Volume Change Simulation
+
+The Bucket class represents fundamental water storage components within the hydrological system. It consists of multiple HydroFlux components (including NeuralFlux) and StateFlux components, which collectively define the water balance dynamics through coupled differential equations. These components can represent various hydrological stores such as soil moisture, groundwater reservoirs, and surface water bodies.
+
+The Bucket class generates two essential functions:
+1. An ODE solver function for state variables
+2. A hydrological flux computation function for output calculations
+
+This dual-function architecture enables efficient computation of hydrological processes, following a two-step process of solving ODEs and computing output fluxes.
+
+## Route Class: Spatial Flow Propagation
+
+The Route class simulates lateral water fluxes across landscapes and river networks, supporting both lumped "integral" and distributed "differential" models. Its core components include:
+
+```math
+\begin{aligned}
+Q_{out}(t) &= f_{rflux}(S_{route}(t),Q_{gen}(t); ps) && (2) \\
+\frac{dS_{route}}{dt} &= Q_{in}(t) - Q_{out}(t) && (3) \\
+Q_{in}(t+1) &= f_{aggr}(Q_{out}(t)) && (4) \\
+\end{aligned}
+```
+
+Where: $Q_{in}$ is inflow, $Q_{out}$ is outflow, $S_{route}$ is the routing state variable, and $f_{aggr}$ is the aggregation function.
+
+The class supports various routing methods including:
+- Standard state-update approaches
+- Unit hydrograph through convolution operations
+- Dynamic routing methods (Muskingum, kinematic wave)
+
+## Model Class: Hydrological Process Integration
+
+The Model class serves as the central management component, orchestrating the integration of Flux, Bucket, and Route components. It facilitates:
+- Construction of distributed hydrological models
+- Integration of vertical water movement with lateral connectivity
+- Systematic parameter optimization and sensitivity analysis
+- Efficient computation through metadata-driven approaches
+
+## Wrapper Class: Enhanced Component Capabilities
+
+The Wrapper class extends component customization capabilities while maintaining interface uniformity. Key features include:
+- NamedTupleIO wrapper for customized input/output specifications
+- EstimateParam wrapper for parameter prediction based on basin characteristics
+- RecordComponentState wrapper for state variable storage, supporting both batch training and online forecasting

docs/src/extent/why_not_MTK.md

@@ -0,0 +1,19 @@
+# HydroModels.jl与ModelingToolkit.jl
+
+## 为什么不直接使用[ModelingToolkit.jl](https://github.com/SciML/ModelingToolkit.jl)
+
+HydroModels.jl的设计理念事实上是与ModelingToolkit.jl完全一致的,两者都是使用Symbolics.jl进行符号计算的框架,ModelingToolkit.jl是一个更通用符号系统,支持多种问题的构建,同时针对特定领域[ModelingToolkitStandardLibrary.jl](https://github.com/SciML/ModelingToolkitStandardLibrary.jl)提供了基础组件的支持.
+但是我之前在ModelingToolkit.jl的使用中存在一些问题:
+
+- 尽管ModelingToolkit.jl同样能够根据符号编程表达水文模型的常微分方程,但对水文模型中单位线计算,汇流过程计算等计算过程的支持却并不是特别理想
+- 在同一种模块中,比如蒸发计算公式和土壤计算模块,不同区域所需要构建的模块会因为公式的不同存在差异,需要分别构建计算公式支撑模型模型构建
+- 水文模型通常包括多个常微分方程,使用一个ODESystem构建会相对混乱,使用多个ODESystem会相对复杂
+- 同时对于多节点输入,空间汇流过程计算,ODESystem或无法直接支持.
+- 对于神经网络模型的嵌入,尽管存在ModelingToolkitNeuralNets.jl,然而嵌入性却没有想象的那么简单
+- 基于ModelingToolkit.jl对于自动微分的支持,尤其是Zygote.jl我在使用时也存在问题.
+
+因此我决定自己构建一个模型库,参考了符号编程的模型构建方式,使其更能够支撑水文模型的一些建模需求.
+
+## 未来的工作
+
+不可否认ModelingToolkit.jl是一个非常好的框架,在满足了水文模型的需求之后,我会尽量将HydroModels.jl中的模块能够成为    ModelingToolkitStandardLibrary.jl中相似的模块,并继承ModelingToolkit.jl提供的AbstractSystem类的支持,提供一致的功能支持,从而兼容更多SciML生态的计算功能.

docs/src/extent/why_not_MTK_en.md

@@ -0,0 +1,20 @@
+# HydroModels.jl and ModelingToolkit.jl
+
+## Why Not Use [ModelingToolkit.jl](https://github.com/SciML/ModelingToolkit.jl) Directly
+
+The design philosophy of HydroModels.jl is actually completely aligned with ModelingToolkit.jl. Both frameworks use Symbolics.jl for symbolic computation. ModelingToolkit.jl is a more general symbolic system that supports the construction of various problems, and [ModelingToolkitStandardLibrary.jl](https://github.com/SciML/ModelingToolkitStandardLibrary.jl) provides support for basic components in specific domains.
+
+However, I encountered several issues when using ModelingToolkit.jl:
+
+- Although ModelingToolkit.jl can express the ordinary differential equations of hydrological models through symbolic programming, its support for calculations like unit hydrograph and routing processes in hydrological models is not particularly ideal
+- Within the same module, such as evaporation calculation formulas and soil calculation modules, the modules needed for different regions may differ due to different formulas, requiring separate construction of calculation formulas to support model building
+- Hydrological models typically include multiple ordinary differential equations. Using a single ODESystem becomes relatively chaotic, while using multiple ODESystems becomes relatively complex
+- Additionally, ODESystem may not directly support multi-node input and spatial routing process calculations
+- Although ModelingToolkitNeuralNets.jl exists for neural network model embedding, the integration is not as straightforward as imagined
+- I also encountered issues with automatic differentiation support based on ModelingToolkit.jl, especially with Zygote.jl
+
+Therefore, I decided to build my own model library, referencing the symbolic programming model construction approach, to better support the modeling needs of hydrological models.
+
+## Future Work
+
+It's undeniable that ModelingToolkit.jl is an excellent framework. After meeting the requirements of hydrological models, I will try to make the modules in HydroModels.jl become similar modules in ModelingToolkitStandardLibrary.jl, and inherit the support of ModelingToolkit.jl's AbstractSystem class to provide consistent functional support, thereby maintaining compatibility with more computational capabilities in the SciML ecosystem.

docs/src/implements/build_discharge_route_en.md

@@ -32,6 +32,7 @@ Q_{rf,n} &= S_{rf,n} \cdot \frac{1}{\text{LAG}_{rf} + 1} && (5)
 The $Q_{rf,n-1}$ term is removed because in continuous change, $S_{rf,n}$ is constantly evolving, making it unnecessary to consider $Q_{rf,n-1}$ in calculating $Q_{rf,n}$. Additionally, there are dimensional differences between $S_{rf,n}$ and $Q_{rf,n}$, and including $Q_{rf,n}$ would add an instantaneous state variable, which is unsuitable as a state variable. In HydroRoute construction, the primary focus is expressing the outflow calculation formula. Like HydroBucket, HydroRoute accepts HydroFlux to represent the $Q_{rf,n}$ calculation formula:
 
 ```julia
+using HydroModels
 # Define parameters and variables
 @variables q q_routed s_river
 @parameters lag
@@ -48,6 +49,7 @@ The method of obtaining upstream flow inputs is another crucial component of the
 The Vector-Based routing model is based on watershed subdivision and topological relationships, using an adjacency matrix for flow accumulation:
 
 ```julia
+using Graphs
 # Build river network topology
 network = DiGraph(9)
 add_edge!(network, 1, 2)

docs/src/implements/build_exphydro_model_en.md

@@ -37,7 +37,6 @@ using HydroModels
 @variables snowpack soilwater
 @parameters Tmin Tmax Df Smax Qmax f
 
-# define step function
 step_func(x) = (tanh(5.0 * x) + 1.0) * 0.5
 
 # define snowpack bucket
@@ -91,7 +90,9 @@ In this code segment, we define all variables used in the ExpHydro model, includ
 The definition of `HydroFlux` requires determining input/output variables and model parameters based on calculation formulas. For example, in the rain-snow partitioning formula, the input variables are `prcp` and `temp`, output variables are `snowfall` and `rainfall`, and the model parameter is `Tmin`. The formula translation to `HydroFlux` looks like this:
 
 ```julia
+# define the smooth function
 step_func(x) = (tanh(5.0 * x) + 1.0) * 0.5
+# define the rain-snow partitioning flux
 split_flux = HydroFlux([prcp, temp] => [snowfall, rainfall], [Tmin], exprs=[step_func(Tmin - temp) * prcp, step_func(temp - Tmin) * prcp])
 ```
 

docs/src/implements/build_exphydro_model_zh.md

@@ -92,7 +92,9 @@ using HydroModels
 `HydroFlux`的定义需要根据计算公式确定模型的输入输出变量和模型参数,例如在模型的雨雪划分计算公式,该公式的输入变量为`prcp`和`temp`,输出变量为`snowfall`和`rainfall`,模型参数为`Tmin`, 公式转译为`HydroFlux`的结果如下所示:
 
 ```julia
+# 定义平滑函数
 step_func(x) = (tanh(5.0 * x) + 1.0) * 0.5
+# 定义雨雪划分函数
 split_flux = HydroFlux([prcp, temp] => [snowfall, rainfall], [Tmin], exprs=[step_func(Tmin - temp) * prcp, step_func(temp - Tmin) * prcp])
 ```
 

docs/src/implements/build_m50_model_en.md

@@ -43,9 +43,34 @@ While this code follows the standard style of the DifferentialEquations.jl libra
 
 ## M50 Implementation in HydroModels.jl
 
+First, we need to define the parameters and variables designed in the M50 model:
+
+```julia
+using HydroModels
+
+#! parameters in the Exp-Hydro model
+@parameters Tmin Tmax Df Smax f Qmax
+#! parameters in normalize flux
+@parameters snowpack_std snowpack_mean
+@parameters soilwater_std soilwater_mean
+@parameters prcp_std prcp_mean
+@parameters temp_std temp_mean
+
+#! hydrological flux in the Exp-Hydro model
+@variables prcp temp lday pet rainfall snowfall
+@variables snowpack soilwater lday pet
+@variables melt log_evap_div_lday log_flow flow
+@variables norm_snw norm_slw norm_temp norm_prcp
+
+step_func(x) = (tanh(5.0 * x) + 1.0) * 0.5
+```
+
 Compared to conventional conceptual hydrological models, the M50 model uses neural networks to replace hydrological flux calculation formulas (`ET` and `Q`). These neural networks differ significantly from standard calculation formulas, so HydroModels.jl uses `NeuralFlux` to represent neural networks. The NeuralFlux for ET and Q predictions is shown below:
 
 ```julia
+using Lux
+using StableRNGs
+
 # define the ET NN and Q NN
 ep_nn = Lux.Chain(
     Lux.Dense(3 => 16, tanh),
@@ -94,7 +119,7 @@ soil_funcs = [
     q_nn_flux
 ]
 state_expr = rainfall + melt - step_func(soilwater) * lday * exp(log_evap_div_lday) - step_func(soilwater) * exp(log_flow)
-soil_dfuncs = [StateFlux([soilwater, rainfall, melt, lday, log_evap_div_lday, log_flow], soilwater, Num[], expr=state_expr)]
+soil_dfuncs = [StateFlux([soilwater, rainfall, melt, lday, log_evap_div_lday, log_flow], soilwater, expr=state_expr)]
 soil_ele = HydroBucket(name=:m50_soil, funcs=soil_funcs, dfuncs=soil_dfuncs)
 convert_flux = HydroFlux([log_flow] => [flow], exprs=[exp(log_flow)])
 # define the Exp-Hydro model