Use bencher (#539)

Trying out bencher.dev, here's a dashboard link: https://bencher.dev/perf/vortex?report=30841f3c-fcbb-4cdc-930d-13c5e103b5a7&branches=c60fea65-8440-45ed-9f5a-586b9363a67d&testbeds=228d0d7b-dbcd-4483-9b97-f318d448c5aa&benchmarks=aa9da1cb-2f07-4c82-8e24-df4d1b2bd618%2C068a3433-dd6e-454e-9a48-c1f78b11ec58%2C12fb3e97-b42d-4359-be24-3ca947a92ec0%2Cf8ffab38-a93c-4289-b29c-2e9c79ed6ea0&measures=2c3aeb1c-a621-4c2f-a220-8b1204a119ba&start_time=1720019326000&end_time=1722611387000&clear=true&key=true&reports_per_page=4&branches_per_page=8&testbeds_per_page=8&benchmarks_per_page=8&plots_per_page=8&reports_page=1&branches_page=1&testbeds_page=1&benchmarks_page=1&plots_page=1 I changed the schedule to run on all pushes to develop to make testing quicker, we can switch it back to scheduled daily if we find that it's using up too many resources
spiraldb · Aug 2, 2024 · 7f66c45 · 7f66c45
1 parent 4f4d2cc
commit 7f66c45
Show file tree

Hide file tree

Showing 5 changed files with 20 additions and 138 deletions.
diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
@@ -1,9 +1,8 @@
 name: Benchmarks
 
 on:
-  schedule:
-    # Run M-F at 9:30 UTC (either 9:30 or 10:30 British time depending on season).
-    - cron: "30 9 * * 1-5"
+  push:
+    branches: [develop]
 
 permissions:
   actions: read
@@ -26,16 +25,17 @@ jobs:
       - name: Install Protoc
         uses: arduino/setup-protoc@v3
 
-      - name: Bench - Vortex
-        run: cargo bench | tee bench.txt
+      - name: Setup bencher.dev CLI
+        uses: bencherdev/bencher@main
+
+      - name: Upload Vortex Benchmarks
+        run: |
+          bencher run \
+            --project vortex \
+            --token "${BENCHER_TOKEN}" \
+            --adapter rust_criterion \
+            --testbed "github-actions" \
+            "cargo bench -p bench-vortex"
+        env:
+          BENCHER_TOKEN: ${{ secrets.BENCHER_TOKEN }}
 
-      - name: Store benchmark result
-        uses: benchmark-action/[email protected]
-        with:
-          name: Vortex Benchmarks
-          tool: cargo
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          output-file-path: bench.txt
-          summary-always: true
-          auto-push: true
-          fail-on-alert: false
diff --git a/bench-vortex/Cargo.toml b/bench-vortex/Cargo.toml
@@ -60,6 +60,11 @@ xshell = { workspace = true }
 [dev-dependencies]
 criterion = { workspace = true, features = ["html_reports", "async_tokio"] }
 
+[[bin]]
+name = "tpch_benchmark"
+test = false
+bench = false
+
 [[bench]]
 name = "compress_benchmark"
 harness = false

diff --git a/bench-vortex/src/bin/comparison.rs b/bench-vortex/src/bin/comparison.rs
diff --git a/bench-vortex/src/bin/compress.rs b/bench-vortex/src/bin/compress.rs
diff --git a/bench-vortex/src/bin/serde.rs b/bench-vortex/src/bin/serde.rs