first pass at benchmarks. need post-processing, formatting, dynamism.…

.buildkite/build_docs.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+pwd; hostname; date
+
+if [ $# -ne 1 ]; then
+    echo "Usage: $0 VERSION"
+    echo "Example: $0 1.10.0"
+    exit 1
+fi
+
+VERSION=$1
+
+module load julia/$VERSION
+
+echo "Building documentation..."
+julia --project=docs -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.status(); Pkg.instantiate(); include("docs/make.jl")'

.buildkite/pipeline.yml

@@ -1,24 +1,15 @@
-env:
-  JULIA_VERSION: "1.10.2"
-  GATAS_HOME: "~/../../blue/fairbanksj/.gatas/bk/agents/$BUILDKITE_AGENT_NAME"
-
 steps:
 
-  - label: ":hammer: Build Project"
-    env:
-      JULIA_DEPOT_PATH: "$GATAS_HOME"
-    command: 
-      - "module load julia"
-      - "julia --project=docs --color=yes -e 'using Pkg; Pkg.update(); Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate(); Pkg.precompile()'"
-
-  - wait 
-
-  - label: ":scroll: Build docs and run tests"
+  - label: "Running benchmarks"
     env:
-      JULIA_DEPOT_PATH: "$GATAS_HOME"
-      JULIA_PROJECT: "docs/"
-    command:
-      - "srun --cpus-per-task=16 --mem=64G --time=1:00:00 --output=.buildkite/log_%j.log --unbuffered .buildkite/jobscript.sh"
+      JULIA_PROJECT = "benchmarks/"
+    command: |
+      ./benchmarks/scripts/main.sh
 
   - wait
 
+  - label: ":arrow_down: Load AlgebraicJulia pipeline"
+    command: |
+      curl -s https://raw.githubusercontent.com/AlgebraicJulia/.github/main/buildkite/pipeline.yml | buildkite-agent pipeline upload
+
+  - wait

.buildkite/run_tests.sh

@@ -0,0 +1,14 @@
+pwd; hostname; date
+
+if [ $# -ne 1 ]; then
+    echo "Usage: $0 VERSION"
+    echo "Example: $0 1.10.0"
+    exit 1
+fi
+
+VERSION=$1
+
+module load julia/$VERSION
+
+echo "Running tests..."
+julia --project -e "using Pkg; Pkg.status(); Pkg.test()"

.gitignore

@@ -1,9 +1,14 @@
 .DS_Store
 .ipynb_checkpoints
-/Manifest.toml
+**/Manifest.toml
 /dev/
 *.mem
 .vscode
 .buildkite/toolbox/config.json
 .buildkite/toolbox/username
 docs/Manifest.toml
+benchmarks/data/sims/
+benchmarks/src/physics/
+benchmarks/scripts/slurm_logs/
+benchmarks/data/exp_pro/heat/
+**/*.swp

benchmarks/scripts/final.jl

@@ -10,4 +10,4 @@ const physics = ARGS[2]
 
 aggregate_data(slurm_id, physics)
 
-run(`julia --threads=auto $(postprocessdir("default_out.jl")) $slurm_id $physics`)
+run(`julia --threads=auto $(postprocessdir("docs_pp.jl")) $slurm_id $physics`)

benchmarks/scripts/main.jl

@@ -26,7 +26,7 @@ elseif length(ARGS) == 3
 
     run_physics = SimNameData(physics, arch, tag)
     is_valid_config_instance(run_physics)
-    run_single_physics(physics, [run_physics])
+    run_single_physics(physics, [run_physics]) # final.sh
 else
     error("Usage: ['physics' 'architecture' 'tag']")
 end

benchmarks/src/main_config.toml

@@ -1,2 +1,5 @@
 [heat.cpu.test]
-[heat.cuda.test]
+[heat.cuda.test]
+
+[brussel.cpu.default]
+[brussel.cuda.default]

docs/benchmark_helper.jl

@@ -0,0 +1,30 @@
+"""
+    insert_benchmarks!(file::String, benchmark_data::Dict)
+
+Insert benchmark results into a markdown file at specified placeholder locations.
+Placeholders should be in the format: <!-- BENCHMARK:benchmark_name -->
+
+Parameters:
+- file: Path to the markdown file
+- benchmark_data: Dictionary mapping benchmark names to their result tables
+"""
+function insert_benchmarks!(file::String, benchmark_data::Dict{String,String})
+	ks = keys(benchmark_data);
+	open(file, "r+") do f
+		foreach(readlines(file)) do line
+			stub = filter(u -> startswith(line, "<!-- BENCHMARK:$u -->"), ks);
+			if !isempty(stub)
+				# get the most recent sims
+				eligible_sims = readdir(benchmark_data[only(stub)], join=true)
+				most_recent = filter(eligible_sims) do path
+					last(splitpath(path)) == maximum(last.(splitpath.(eligible_sims)))
+                end
+				foreach(eachline(joinpath(only(most_recent), "default_output.md"))) do line
+					println(f, line)
+				end
+			else
+				println(f, line)
+			end
+		end 
+	end
+end

docs/make.jl

@@ -31,6 +31,11 @@ end
 #   end
 # end
 
+@info "Interpolating benchmarks into docs $(@__DIR__))"
+include("benchmark_helper.jl")
+insert_benchmarks!("docs/src/benchmarks.md",
+	Dict("heat" => "benchmarks/data/exp_pro/heat/"))
+
 @info "Building Documenter.jl docs"
 makedocs(
   modules   = [Decapodes],
@@ -47,23 +52,24 @@ makedocs(
                       r"Decapodes\.jl/dev"], # 404, probably due to bad self-rerference
   pages     = Any[
     "Decapodes.jl" => "index.md",
-    "Overview" => "overview/overview.md",
-    "Equations" => "equations/equations.md",
-    "Vortices" => "navier_stokes/ns.md",
-    "Harmonics" => "harmonics/harmonics.md",
-    "Cahn-Hilliard" => "ch/cahn-hilliard.md",
-    "Klausmeier" => "klausmeier/klausmeier.md",
-    "CISM v2.1" => "cism/cism.md",
-    "Glacial Flow" => "ice_dynamics/ice_dynamics.md",
-    "Grigoriev Ice Cap" => "grigoriev/grigoriev.md", # Requires ice_dynamics
-    "Budyko-Sellers-Halfar" => "bsh/budyko_sellers_halfar.md", # Requires ice_dynamics
-    "Halfar-NS" => "halmo/halmo.md", # Requires grigoriev
-    "NHS" => "nhs/nhs_lite.md",
-    "Pipe Flow" => "poiseuille/poiseuille.md",
-    "Misc Features" => "bc/bc_debug.md", # Requires overview
-    "ASCII Operators" => "ascii.md",
-    "Canonical Models" => "canon.md",
-    "Library Reference" => "api.md"
+    #"Overview" => "overview/overview.md",
+    #"Equations" => "equations/equations.md",
+    #"Vortices" => "navier_stokes/ns.md",
+    #"Harmonics" => "harmonics/harmonics.md",
+    #"Cahn-Hilliard" => "ch/cahn-hilliard.md",
+    #"Klausmeier" => "klausmeier/klausmeier.md",
+    #"CISM v2.1" => "cism/cism.md",
+    #"Glacial Flow" => "ice_dynamics/ice_dynamics.md",
+    #"Grigoriev Ice Cap" => "grigoriev/grigoriev.md", # Requires ice_dynamics
+    #"Budyko-Sellers-Halfar" => "bsh/budyko_sellers_halfar.md", # Requires ice_dynamics
+    #"Halfar-NS" => "halmo/halmo.md", # Requires grigoriev
+    #"NHS" => "nhs/nhs_lite.md",
+    #"Pipe Flow" => "poiseuille/poiseuille.md",
+    #"Misc Features" => "bc/bc_debug.md", # Requires overview
+    "Benchmarks" => "benchmarks.md"
+    #"ASCII Operators" => "ascii.md",
+    #"Canonical Models" => "canon.md",
+    #"Library Reference" => "api.md"
   ]
 )
 

docs/src/benchmarks.md

@@ -0,0 +1,17 @@
+# Benchmarks
+
+## Heat
+| Task ID |                   statsfile |                        benchfile | resolution | code_target | float_type | Setup Median time | Mesh Median time | Simulate Median time | Solve Median time |   nf |
+|---------|-----------------------------|----------------------------------|------------|-------------|------------|-------------------|------------------|----------------------|-------------------|------|
+|       3 |  stats_heat_cpu_test_3.jld2 |  benchmarks_heat_cpu_test_3.json |          1 |   CPUTarget |    Float32 |        0.00440888 |         0.279692 |           0.00309735 |          0.558636 | 9327 |
+|       6 |  stats_heat_cpu_test_6.jld2 |  benchmarks_heat_cpu_test_6.json |          1 |   CPUTarget |    Float64 |        0.00447047 |         0.324688 |           0.00329365 |           0.63329 | 9297 |
+|       3 | stats_heat_cuda_test_3.jld2 | benchmarks_heat_cuda_test_3.json |          1 |  CUDATarget |    Float32 |         0.0052702 |         0.382094 |           0.00457613 |           0.79329 | 9321 |
+|       6 | stats_heat_cuda_test_6.jld2 | benchmarks_heat_cuda_test_6.json |          1 |  CUDATarget |    Float64 |        0.00491387 |         0.386088 |            0.0046531 |          0.801826 | 9297 |
+|       2 |  stats_heat_cpu_test_2.jld2 |  benchmarks_heat_cpu_test_2.json |          2 |   CPUTarget |    Float32 |         0.0044489 |        0.0662042 |          0.000771337 |         0.0368417 | 2409 |
+|       5 |  stats_heat_cpu_test_5.jld2 |  benchmarks_heat_cpu_test_5.json |          2 |   CPUTarget |    Float64 |        0.00449127 |        0.0769773 |           0.00080348 |         0.0415073 | 2373 |
+|       2 | stats_heat_cuda_test_2.jld2 | benchmarks_heat_cuda_test_2.json |          2 |  CUDATarget |    Float32 |        0.00496288 |        0.0869387 |           0.00202888 |          0.205252 | 2409 |
+|       5 | stats_heat_cuda_test_5.jld2 | benchmarks_heat_cuda_test_5.json |          2 |  CUDATarget |    Float64 |         0.0049011 |        0.0873349 |           0.00192332 |          0.197789 | 2373 |
+|       1 |  stats_heat_cpu_test_1.jld2 |  benchmarks_heat_cpu_test_1.json |          5 |   CPUTarget |    Float32 |        0.00444552 |        0.0101653 |          0.000141233 |        0.00136106 |  471 |
+|       4 |  stats_heat_cpu_test_4.jld2 |  benchmarks_heat_cpu_test_4.json |          5 |   CPUTarget |    Float64 |        0.00448736 |        0.0124717 |          0.000148126 |        0.00146659 |  435 |
+|       1 | stats_heat_cuda_test_1.jld2 | benchmarks_heat_cuda_test_1.json |          5 |  CUDATarget |    Float32 |        0.00492554 |        0.0136025 |           0.00122809 |         0.0408824 |  471 |
+|       4 | stats_heat_cuda_test_4.jld2 | benchmarks_heat_cuda_test_4.json |          5 |  CUDATarget |    Float64 |        0.00524847 |        0.0150585 |           0.00122604 |         0.0378521 |  435 |