More configurable benchmarks

.github/workflows/DependaBot.yml

@@ -0,0 +1,7 @@
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"

.github/workflows/docs-nobenchmark.yml → .github/workflows/docs.yml

@@ -1,4 +1,4 @@
-name: docs-nobenchmark
+name: docs
 on:
   push:
     branches:

Project.toml

@@ -32,7 +32,6 @@ DocStringExtensions = "0.9"
 LinearAlgebra = "1.6"
 PrecompileTools = "1.1"
 Random = "1.6"
-RequiredInterfaces = "0.1.3"
 Requires = "1.3"
 SimpleUnPack = "1.1"
 StatsAPI = "1.6"
@@ -57,9 +56,8 @@ SimpleUnPack = "ce78b400-467f-4804-87d8-8f486da07d0a"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
-Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [targets]
-test = ["Aqua", "BenchmarkTools", "Distributions", "Documenter", "FiniteDifferences", "ForwardDiff", "HMMBase", "JET", "JuliaFormatter", "LinearAlgebra", "LogarithmicNumbers", "Pkg", "Random", "SimpleUnPack", "SparseArrays", "StaticArrays", "Statistics", "Suppressor", "Test", "Zygote"]
+test = ["Aqua", "BenchmarkTools", "Distributions", "Documenter", "FiniteDifferences", "ForwardDiff", "HMMBase", "JET", "JuliaFormatter", "LinearAlgebra", "LogarithmicNumbers", "Pkg", "Random", "SimpleUnPack", "SparseArrays", "StaticArrays", "Statistics", "Test", "Zygote"]

benchmark/HMMBenchmark/src/HMMBenchmark.jl

@@ -12,7 +12,7 @@ using BenchmarkTools
 using HiddenMarkovModels
 using SimpleUnPack
 
-export define_suite
+export define_suite, run_suite, parse_results
 
 include("hmms.jl")
 include("hmmbase.jl")

benchmark/HMMBenchmark/src/hmmbase.jl

@@ -21,30 +21,37 @@ end
 function benchmarkables_hmmbase(; algos, N, D, T, K, I)
     rand_model_hmmbase(; N, D)
     if D == 1
-        obs_mat = randn(K * T)
+        obs_mats = [randn(T) for k in 1:K]
     else
-        obs_mat = randn(K * T, D)
+        obs_mats = [randn(T, D) for k in 1:K]
     end
+    obs_mats_concat = randn(K * T, D)
     benchs = Dict()
     if "logdensity" in algos
-        benchs["logdensity"] = @benchmarkable HMMBase.forward(model, $obs_mat) setup = (
-            model = rand_model_hmmbase(; N=$N, D=$D)
-        )
+        benchs["logdensity"] = @benchmarkable begin
+            for k in 1:($K)
+                HMMBase.forward(model, $obs_mats[k])
+            end
+        end setup = (model = rand_model_hmmbase(; N=$N, D=$D))
     end
     if "viterbi" in algos
-        benchs["viterbi"] = @benchmarkable HMMBase.viterbi(model, $obs_mat) setup = (
-            model = rand_model_hmmbase(; N=$N, D=$D)
-        )
+        benchs["viterbi"] = @benchmarkable begin
+            for k in 1:($K)
+                HMMBase.viterbi(model, $obs_mats[k])
+            end
+        end setup = (model = rand_model_hmmbase(; N=$N, D=$D))
     end
     if "forward_backward" in algos
-        benchs["forward_backward"] = @benchmarkable HMMBase.posteriors(model, $obs_mat) setup = (
-            model = rand_model_hmmbase(; N=$N, D=$D)
-        )
+        benchs["forward_backward"] = @benchmarkable begin
+            for k in 1:($K)
+                HMMBase.posteriors(model, $obs_mats[k])
+            end
+        end setup = (model = rand_model_hmmbase(; N=$N, D=$D))
     end
     if "baum_welch" in algos
-        benchs["baum_welch"] = @benchmarkable HMMBase.fit_mle(
-            model, $obs_mat; maxiter=$I, tol=-Inf
-        ) setup = (model = rand_model_hmmbase(; N=$N, D=$D))
+        benchs["baum_welch"] = @benchmarkable begin
+            HMMBase.fit_mle(model, $obs_mats_concat; maxiter=$I, tol=-Inf)
+        end setup = (model = rand_model_hmmbase(; N=$N, D=$D))
     end
     return benchs
 end

benchmark/HMMBenchmark/src/hmms.jl

@@ -26,24 +26,32 @@ function benchmarkables_hmms(; algos, N, D, T, K, I)
     end
     benchs = Dict()
     if "logdensity" in algos
-        benchs["logdensity"] = @benchmarkable HiddenMarkovModels.logdensityof(
-            model, $obs_seqs, $K
-        ) setup = (model = rand_model_hmms(; N=$N, D=$D))
+        benchs["logdensity"] = @benchmarkable begin
+            for k in 1:($K)
+                HiddenMarkovModels.logdensityof(model, $obs_seqs[k])
+            end
+        end setup = (model = rand_model_hmms(; N=$N, D=$D))
     end
     if "viterbi" in algos
-        benchs["viterbi"] = @benchmarkable HiddenMarkovModels.viterbi(model, $obs_seqs, $K) setup = (
-            model = rand_model_hmms(; N=$N, D=$D)
-        )
+        benchs["viterbi"] = @benchmarkable begin
+            for k in 1:($K)
+                HiddenMarkovModels.viterbi(model, $obs_seqs[k])
+            end
+        end setup = (model = rand_model_hmms(; N=$N, D=$D))
     end
     if "forward_backward" in algos
-        benchs["forward_backward"] = @benchmarkable HiddenMarkovModels.forward_backward(
-            model, $obs_seqs, $K
-        ) setup = (model = rand_model_hmms(; N=$N, D=$D))
+        benchs["forward_backward"] = @benchmarkable begin
+            for k in 1:($K)
+                HiddenMarkovModels.forward_backward(model, $obs_seqs[k])
+            end
+        end setup = (model = rand_model_hmms(; N=$N, D=$D))
     end
     if "baum_welch" in algos
-        benchs["baum_welch"] = @benchmarkable HiddenMarkovModels.baum_welch(
-            model, $obs_seqs, $K; max_iterations=$I, atol=-Inf
-        ) setup = (model = rand_model_hmms(; N=$N, D=$D))
+        benchs["baum_welch"] = @benchmarkable begin
+            HiddenMarkovModels.baum_welch(
+                model, $obs_seqs, $K; max_iterations=$I, atol=-Inf
+            )
+        end setup = (model = rand_model_hmms(; N=$N, D=$D))
     end
     return benchs
 end

benchmark/HMMBenchmark/src/suite.jl

@@ -12,7 +12,7 @@ function benchmarkables_by_implem(; implem, algos, kwargs...)
     end
 end
 
-function define_suite(; implems, algos, N_vals, D_vals, T_vals, K_vals, I)
+function define_suite(; implems, algos, N_vals, D_vals, T_vals, K_vals, I_vals)
     SUITE = BenchmarkGroup()
     if ("HMMBase.jl" in implems) && any(>(1), K_vals)
         @warn "HMMBase.jl doesn't support multiple observation sequences, concatenating instead."
@@ -24,7 +24,7 @@ function define_suite(; implems, algos, N_vals, D_vals, T_vals, K_vals, I)
             SUITE[implem][algo] = BenchmarkGroup()
             SUITE[implem][algo][(1, 1, 2, 1, 1)] = bench
         end
-        for N in N_vals, D in D_vals, T in T_vals, K in K_vals
+        for (N, D, T, K, I) in zip(N_vals, D_vals, T_vals, K_vals, I_vals)
             bench_tup = benchmarkables_by_implem(; implem, algos, N, D, T, K, I)
             for (algo, bench) in pairs(bench_tup)
                 SUITE[implem][algo][(N, D, T, K, I)] = bench

benchmark/Manifest.toml

@@ -696,15 +696,15 @@ version = "10.42.0+0"
 
 [[deps.PDMats]]
 deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"]
-git-tree-sha1 = "f6f85a2edb9c356b829934ad3caed2ad0ebbfc99"
+git-tree-sha1 = "66b2fcd977db5329aa35cac121e5b94dd6472198"
 uuid = "90014a1f-27ba-587c-ab20-58faa44d9150"
-version = "0.11.29"
+version = "0.11.28"
 
 [[deps.Parsers]]
 deps = ["Dates", "PrecompileTools", "UUIDs"]
-git-tree-sha1 = "a935806434c9d4c506ba941871b327b96d41f2bf"
+git-tree-sha1 = "716e24b21538abc91f6205fd1d8363f39b442851"
 uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
-version = "2.8.0"
+version = "2.7.2"
 
 [[deps.Pipe]]
 git-tree-sha1 = "6842804e7867b115ca9de748a0cf6b364523c16d"

benchmark/benchmarks.jl

@@ -4,11 +4,11 @@ using BenchmarkTools
 SUITE = define_suite(;
     implems=("HMMs.jl",),
     algos=("logdensity", "viterbi", "forward_backward", "baum_welch"),
-    N_vals=2:2:20,
-    D_vals=10,
-    T_vals=100,
-    K_vals=10,
-    I=10,
+    N_vals=4:4:20,
+    D_vals=fill(10, 5),
+    T_vals=fill(100, 5),
+    K_vals=fill(10, 5),
+    I_vals=fill(10, 5),
 )
 
 BenchmarkTools.save(joinpath(@__DIR__, "tune.json"), BenchmarkTools.params(SUITE));

docs/Project.toml

@@ -4,7 +4,6 @@ Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 DocumenterCitations = "daee34ce-89f3-4625-b898-19384cb65244"
 HiddenMarkovModels = "84ca31d5-effc-45e0-bfda-5a68cd981f47"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
-RequiredInterfaces = "97f35ef4-7bc5-4ec1-a41a-dcc69c7308c6"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 StatsAPI = "82ae8749-77ed-4fe6-ae5f-f523153014b0"
 

docs/make.jl

@@ -22,21 +22,18 @@ open(joinpath(joinpath(@__DIR__, "src"), "index.md"), "w") do io
     end
 end
 
-benchmarks_done = false
-
 pages = [
     "Home" => "index.md",
-    "Essentials" => ["Background" => "background.md", "API reference" => "api.md"],
+    "Essentials" => [
+        "Background" => "background.md",
+        "API reference" => "api.md",
+        "Competitors" => "competitors.md",
+    ],
     "Tutorials" => [
         "Built-in HMM" => "builtin.md",
         "Custom HMM" => "custom.md",
         "Debugging" => "debugging.md",
     ],
-    "Alternatives" => if benchmarks_done
-        ["Features" => "features.md", "Benchmarks" => "benchmarks.md"]
-    else
-        ["Features" => "features.md"]
-    end,
     "Advanced" => ["Formulas" => "formulas.md", "Roadmap" => "roadmap.md"],
 ]
 

docs/src/api.md

@@ -38,21 +38,40 @@ fit!
 ## Misc
 
 ```@docs
-check_hmm
 rand_prob_vec
 rand_trans_mat
+HiddenMarkovModels.fit_element_from_sequence!
+HiddenMarkovModels.LightDiagNormal
 ```
 
-## Internals
+## In-place algorithms (internals)
+
+### Storage types
 
 ```@docs
 HiddenMarkovModels.ForwardStorage
 HiddenMarkovModels.ViterbiStorage
 HiddenMarkovModels.ForwardBackwardStorage
 HiddenMarkovModels.BaumWelchStorage
-HiddenMarkovModels.fit_element_from_sequence!
-HiddenMarkovModels.LightDiagNormal
-HiddenMarkovModels.PermutedHMM
+```
+
+### Initializing storage
+
+```@docs
+HiddenMarkovModels.initialize_forward
+HiddenMarkovModels.initialize_viterbi
+HiddenMarkovModels.initialize_forward_backward
+HiddenMarkovModels.initialize_baum_welch
+HiddenMarkovModels.initialize_logL_evolution
+```
+
+### Modifying storage
+
+```@docs
+HiddenMarkovModels.forward!
+HiddenMarkovModels.viterbi!
+HiddenMarkovModels.forward_backward!
+HiddenMarkovModels.baum_welch!
 ```
 
 ## Notations
@@ -82,7 +101,7 @@ HiddenMarkovModels.PermutedHMM
 - `β`: scaled backward variables
 - `γ`: state marginals
 - `ξ`: transition marginals
-- `logL`: posterior loglikelihood of a sequence of observations
+- `logL`: loglikelihood of a sequence of observations
 
 ## Index
 

docs/src/builtin.md

@@ -1,4 +1,4 @@
-# Tutorial - built-in HMM
+# Built-in HMM
 
 ```@example tuto
 using Distributions
@@ -78,7 +78,7 @@ perm = sortperm(1:3, by=i->d_est[i].μ)
 ```
 
 ```@example tuto
-hmm_est = HiddenMarkovModels.PermutedHMM(hmm_est, perm)
+hmm_est = HiddenMarkovModels.permute(hmm_est, perm)
 ```
 
 Evaluating errors:

docs/src/features.md → docs/src/competitors.md

@@ -1,4 +1,4 @@
-# Alternatives - features
+# Competitors
 
 We compare features among the following Julia packages:
 

docs/src/custom.md

@@ -1,4 +1,4 @@
-# Tutorial - custom HMM
+# Custom HMM
 
 ```@example tuto
 using Distributions