Add Datafusion (#4)

* add databend * add datafusion * fix tests --------- Co-authored-by: root <[email protected]>
lmangani · Nov 14, 2023 · 03d1027 · 03d1027
1 parent 669ad86
commit 03d1027
Show file tree

Hide file tree

Showing 7 changed files with 129 additions and 4 deletions.
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        DBNAME: [ "chdb", "duckdb", "glaredb", "databend"]
+        DBNAME: [ "chdb", "duckdb", "glaredb", "databend", "datafusion"]
 
     steps:
     - uses: actions/checkout@v3

diff --git a/benchmark.py b/benchmark.py
@@ -13,7 +13,7 @@
 
 DBNAME = os.getenv('DBNAME', '*')
 ITERATIONS = int(os.getenv('ITERATIONS', 3))
-BENCHMARKS = ["version", "count", "groupby", "groupby-local"]
+BENCHMARKS = ["version", "count", "groupby"]
 
 @contextmanager
 def suppress_stdout():
@@ -80,6 +80,10 @@ def main():
             print("Testing databend")
             databendx = SessionContext()
             benchmark_db("databend", lambda query: databendx.sql(query).collect())
+        case "datafusion":
+            print("Testing datafusion")
+            databendx = SessionContext()
+            benchmark_db("datafusion", lambda query: databendx.sql(query).collect())
 
 if __name__ == "__main__":
     main()
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "embedded-olap-benchmarks"
-version = "0.1.2"
+version = "0.1.3"
 description = ""
 authors = ["Lorenzo Mangani <[email protected]>"]
 
@@ -11,6 +11,7 @@ chdb = "^0.16.0rc2"
 duckdb = "^0.9.1"
 glaredb = "^0.5.1"
 databend = "^1.2.207"
+datafusion = "^32.0.0"
 
 [tool.poetry.dev-dependencies]
 

diff --git a/queries/count.datafusion.sql b/queries/count.datafusion.sql
@@ -0,0 +1 @@
+SELECT count(*) FROM 'https://shell.duckdb.org/data/tpch/0_01/parquet/lineitem.parquet';
diff --git a/queries/groupby.datafusion.sql b/queries/groupby.datafusion.sql
@@ -0,0 +1,8 @@
+SELECT TO_DATE(tpep_pickup_datetime::date) as day, 
+       PULocationID as location, 
+       count(*) as trips, 
+       sum(fare_amount) + sum(mta_tax) + sum(tolls_amount) + sum(tip_amount) as revenue 
+FROM 'https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2023-01.parquet' 
+WHERE trip_distance > 5 
+GROUP BY tpep_pickup_datetime, location
+ORDER BY day
diff --git a/queries/version.datafusion.sql b/queries/version.datafusion.sql
@@ -0,0 +1 @@
+SELECT version()