From ff952d1f0cb98ce86a29b6190d4a03897c5e9a6a Mon Sep 17 00:00:00 2001
From: Robert Kruszewski <github@robertk.io>
Date: Mon, 12 Aug 2024 18:55:34 +0100
Subject: [PATCH] Use github benchmark action to run benchmarks (#602)

---
 .github/workflows/bench.yml            | 31 +++++++++++++-------------
 bench-vortex/benches/tpch_benchmark.rs | 20 +++++++++++++++++
 2 files changed, 35 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 6635d18866..3270757dc2 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -2,7 +2,7 @@ name: Benchmarks
 
 on:
   push:
-    branches: [develop]
+    branches: [ develop ]
 
 permissions:
   actions: read
@@ -20,30 +20,29 @@ jobs:
       - uses: ./.github/actions/setup-rust
       - uses: ./.github/actions/setup-python
 
-
-      - name: Setup bencher.dev CLI
-        uses: bencherdev/bencher@main
-
       # The compression benchmarks rely on DuckDB being installed to convert CSV to Parquet
       - name: Install DuckDB
         uses: opt-nc/setup-duckdb-action@v1.0.8
         with:
           version: v1.0.0
 
-      - name: Upload Vortex Benchmarks
-        run: |
-          bencher run \
-            --project vortex \
-            --token "${BENCHER_TOKEN}" \
-            --adapter rust_criterion \
-            --testbed "github-actions" \
-            "cargo bench -p bench-vortex"
+      - name: Run benchmark
+        run: cargo bench --bench tpch_benchmark -- --output-format bencher | tee output.txt
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Vortex benchmarks
+          tool: 'cargo'
+          gh-pages-branch: gh-pages-bench
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          output-file-path: output.txt
+          summary-always: true
+          auto-push: true
+          fail-on-alert: false
         env:
-          BENCHER_TOKEN: ${{ secrets.BENCHER_TOKEN }}
           # AWS Credentials for R2 storage tests
           AWS_BUCKET: vortex-test
           AWS_ENDPOINT: ${{ secrets.AWS_ENDPOINT }}
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-
-
diff --git a/bench-vortex/benches/tpch_benchmark.rs b/bench-vortex/benches/tpch_benchmark.rs
index c18d059d94..299489d2a5 100644
--- a/bench-vortex/benches/tpch_benchmark.rs
+++ b/bench-vortex/benches/tpch_benchmark.rs
@@ -41,6 +41,14 @@ fn benchmark(c: &mut Criterion) {
             },
         ))
         .unwrap();
+    let persistent_uncompressed_vortex_ctx = runtime
+        .block_on(load_datasets(
+            &data_dir,
+            Format::OnDiskVortex {
+                enable_compression: false,
+            },
+        ))
+        .unwrap();
 
     for (q, query) in tpch_queries() {
         let mut group = c.benchmark_group(format!("tpch_q{q}"));
@@ -105,6 +113,18 @@ fn benchmark(c: &mut Criterion) {
                     .unwrap()
             })
         });
+
+        group.bench_function("persistent_uncompressed_vortex", |b| {
+            b.to_async(&runtime).iter(|| async {
+                persistent_uncompressed_vortex_ctx
+                    .sql(&query)
+                    .await
+                    .unwrap()
+                    .collect()
+                    .await
+                    .unwrap()
+            })
+        });
     }
 }