From 8e704cae9a1a00448fb8d75cc2d8723da8b11f1d Mon Sep 17 00:00:00 2001 From: Matt Green Date: Tue, 13 Aug 2024 14:54:57 -0700 Subject: [PATCH] upgrade to datafusion 41.0.0 --- Cargo.lock | 60 ++++++++++++------------- Cargo.toml | 9 +++- examples/examples/csv_streaming.rs | 3 +- examples/examples/kafka_rideshare.rs | 3 +- examples/examples/simple_aggregation.rs | 4 +- 5 files changed, 43 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9720ae0..ffeb915 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -796,8 +796,8 @@ dependencies = [ [[package]] name = "datafusion" -version = "40.0.0" -source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#7dc9d395099b4c3830852dbbf32f1882173332bb" +version = "41.0.0" +source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#f5ef67c91bf1f17166fa63a5005acd628501e5b0" dependencies = [ "ahash", "arrow", @@ -851,8 +851,8 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "40.0.0" -source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#7dc9d395099b4c3830852dbbf32f1882173332bb" +version = "41.0.0" +source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#f5ef67c91bf1f17166fa63a5005acd628501e5b0" dependencies = [ "arrow-schema", "async-trait", @@ -864,8 +864,8 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "40.0.0" -source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#7dc9d395099b4c3830852dbbf32f1882173332bb" +version = "41.0.0" +source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#f5ef67c91bf1f17166fa63a5005acd628501e5b0" dependencies = [ "ahash", "arrow", @@ -890,16 +890,16 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "40.0.0" -source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#7dc9d395099b4c3830852dbbf32f1882173332bb" +version = "41.0.0" +source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#f5ef67c91bf1f17166fa63a5005acd628501e5b0" dependencies = [ "tokio", ] [[package]] name = "datafusion-execution" -version = "40.0.0" -source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#7dc9d395099b4c3830852dbbf32f1882173332bb" +version = "41.0.0" +source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#f5ef67c91bf1f17166fa63a5005acd628501e5b0" dependencies = [ "arrow", "chrono", @@ -918,8 +918,8 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "40.0.0" -source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#7dc9d395099b4c3830852dbbf32f1882173332bb" +version = "41.0.0" +source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#f5ef67c91bf1f17166fa63a5005acd628501e5b0" dependencies = [ "ahash", "arrow", @@ -936,8 +936,8 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "40.0.0" -source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#7dc9d395099b4c3830852dbbf32f1882173332bb" +version = "41.0.0" +source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#f5ef67c91bf1f17166fa63a5005acd628501e5b0" dependencies = [ "arrow", "arrow-buffer", @@ -962,8 +962,8 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "40.0.0" -source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#7dc9d395099b4c3830852dbbf32f1882173332bb" +version = "41.0.0" +source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#f5ef67c91bf1f17166fa63a5005acd628501e5b0" dependencies = [ "ahash", "arrow", @@ -979,8 +979,8 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "40.0.0" -source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#7dc9d395099b4c3830852dbbf32f1882173332bb" +version = "41.0.0" +source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#f5ef67c91bf1f17166fa63a5005acd628501e5b0" dependencies = [ "arrow", "arrow-array", @@ -1000,8 +1000,8 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "40.0.0" -source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#7dc9d395099b4c3830852dbbf32f1882173332bb" +version = "41.0.0" +source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#f5ef67c91bf1f17166fa63a5005acd628501e5b0" dependencies = [ "arrow", "async-trait", @@ -1019,8 +1019,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "40.0.0" -source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#7dc9d395099b4c3830852dbbf32f1882173332bb" +version = "41.0.0" +source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#f5ef67c91bf1f17166fa63a5005acd628501e5b0" dependencies = [ "ahash", "arrow", @@ -1048,8 +1048,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "40.0.0" -source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#7dc9d395099b4c3830852dbbf32f1882173332bb" +version = "41.0.0" +source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#f5ef67c91bf1f17166fa63a5005acd628501e5b0" dependencies = [ "ahash", "arrow", @@ -1061,8 +1061,8 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "40.0.0" -source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#7dc9d395099b4c3830852dbbf32f1882173332bb" +version = "41.0.0" +source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#f5ef67c91bf1f17166fa63a5005acd628501e5b0" dependencies = [ "datafusion-common", "datafusion-execution", @@ -1072,8 +1072,8 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "40.0.0" -source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#7dc9d395099b4c3830852dbbf32f1882173332bb" +version = "41.0.0" +source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#f5ef67c91bf1f17166fa63a5005acd628501e5b0" dependencies = [ "ahash", "arrow", @@ -1105,8 +1105,8 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "40.0.0" -source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#7dc9d395099b4c3830852dbbf32f1882173332bb" +version = "41.0.0" +source = "git+https://github.com/probably-nothing-labs/arrow-datafusion#f5ef67c91bf1f17166fa63a5005acd628501e5b0" dependencies = [ "arrow", "arrow-array", diff --git a/Cargo.toml b/Cargo.toml index 38ca59c..8df5af0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ description = "Embeddable stream processing engine" [workspace.dependencies] df-streams-core = { path = "crates/core" } -datafusion = "40.0" +datafusion = "41.0.0" arrow = { version = "52.0.0", features = ["prettyprint"] } arrow-array = { version = "52.0.0", default-features = false, features = [ @@ -39,7 +39,12 @@ futures = "0.3" tracing = "0.1.40" tracing-log = "0.2.0" tracing-subscriber = "0.3.18" -tokio = { version = "1.36", features = ["macros", "rt", "sync", "rt-multi-thread"] } +tokio = { version = "1.36", features = [ + "macros", + "rt", + "sync", + "rt-multi-thread", +] } async-trait = "0.1.81" rdkafka = "0.36.2" log = "^0.4" diff --git a/examples/examples/csv_streaming.rs b/examples/examples/csv_streaming.rs index aee299f..b51930f 100644 --- a/examples/examples/csv_streaming.rs +++ b/examples/examples/csv_streaming.rs @@ -2,7 +2,8 @@ use datafusion::common::test_util::datafusion_test_data; use datafusion::dataframe::DataFrameWriteOptions; use datafusion::datasource::MemTable; use datafusion::error::Result; -use datafusion::logical_expr::{col, max, min}; +use datafusion::functions_aggregate::expr_fn::{max, min}; +use datafusion::logical_expr::col; use datafusion::prelude::*; /// This example demonstrates executing a simple query against an Arrow data source (CSV) and diff --git a/examples/examples/kafka_rideshare.rs b/examples/examples/kafka_rideshare.rs index 90902bd..694b8af 100644 --- a/examples/examples/kafka_rideshare.rs +++ b/examples/examples/kafka_rideshare.rs @@ -1,7 +1,8 @@ use datafusion::error::Result; use datafusion::functions::core::expr_ext::FieldAccessor; use datafusion::functions_aggregate::count::count; -use datafusion::logical_expr::{col, max, min}; +use datafusion::functions_aggregate::expr_fn::{max, min}; +use datafusion::logical_expr::col; use df_streams_core::context::Context; use df_streams_core::datasource::kafka::{ConnectionOpts, KafkaTopicBuilder}; diff --git a/examples/examples/simple_aggregation.rs b/examples/examples/simple_aggregation.rs index 556192e..4097ea5 100644 --- a/examples/examples/simple_aggregation.rs +++ b/examples/examples/simple_aggregation.rs @@ -3,8 +3,8 @@ use std::time::Duration; use datafusion::error::Result; use datafusion::functions_aggregate::average::avg; use datafusion::functions_aggregate::count::count; -use datafusion::logical_expr::lit; -use datafusion::logical_expr::{col, max, min}; +use datafusion::functions_aggregate::expr_fn::{max, min}; +use datafusion::logical_expr::{col, lit}; use df_streams_core::context::Context; use df_streams_core::datasource::kafka::{ConnectionOpts, KafkaTopicBuilder};