From 8c8212862e57df47c9db5e432b0d7f934ca44666 Mon Sep 17 00:00:00 2001
From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com>
Date: Sun, 11 Aug 2024 21:48:11 +0000
Subject: [PATCH] chore(deps): update datafusion to v41 (major) (#595)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
[](https://renovatebot.com)
This PR contains the following updates:
| Package | Type | Update | Change |
|---|---|---|---|
| [datafusion](https://datafusion.apache.org)
([source](https://togithub.com/apache/datafusion)) |
workspace.dependencies | major | `40.0.0` -> `41.0.0` |
| [datafusion-common](https://datafusion.apache.org)
([source](https://togithub.com/apache/datafusion)) |
workspace.dependencies | major | `40.0.0` -> `41.0.0` |
| [datafusion-execution](https://datafusion.apache.org)
([source](https://togithub.com/apache/datafusion)) |
workspace.dependencies | major | `40.0.0` -> `41.0.0` |
| [datafusion-expr](https://datafusion.apache.org)
([source](https://togithub.com/apache/datafusion)) |
workspace.dependencies | major | `40.0.0` -> `41.0.0` |
| [datafusion-physical-expr](https://datafusion.apache.org)
([source](https://togithub.com/apache/datafusion)) |
workspace.dependencies | major | `40.0.0` -> `41.0.0` |
| [datafusion-physical-plan](https://datafusion.apache.org)
([source](https://togithub.com/apache/datafusion)) |
workspace.dependencies | major | `40.0.0` -> `41.0.0` |
---
### Release Notes
apache/datafusion (datafusion)
###
[`v41.0.0`](https://togithub.com/apache/datafusion/compare/40.0.0...41.0.0)
[Compare
Source](https://togithub.com/apache/datafusion/compare/40.0.0...41.0.0)
---
### Configuration
📅 **Schedule**: Branch creation - At any time (no schedule defined),
Automerge - At any time (no schedule defined).
🚦 **Automerge**: Disabled by config. Please merge this manually once you
are satisfied.
â™» **Rebasing**: Whenever PR becomes conflicted, or you tick the
rebase/retry checkbox.
🔕 **Ignore**: Close this PR and you won't be reminded about these
updates again.
---
- [ ] If you want to rebase/retry this PR, check
this box
---
This PR was generated by [Mend
Renovate](https://www.mend.io/free-developer-tools/renovate/). View the
[repository job log](https://developer.mend.io/github/spiraldb/vortex).
---------
Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Co-authored-by: Robert Kruszewski
---
Cargo.lock | 97 +++++++++++++-------
Cargo.toml | 12 +--
vortex-datafusion/src/memory.rs | 6 +-
vortex-datafusion/src/persistent/provider.rs | 4 +-
4 files changed, 75 insertions(+), 44 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 07bf73eee7..3cba3cc421 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -961,11 +961,12 @@ dependencies = [
[[package]]
name = "dashmap"
-version = "5.5.3"
+version = "6.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
+checksum = "804c8821570c3f8b70230c2ba75ffa5c0f9a4189b9a432b6656c536712acae28"
dependencies = [
"cfg-if",
+ "crossbeam-utils",
"hashbrown",
"lock_api",
"once_cell",
@@ -974,9 +975,9 @@ dependencies = [
[[package]]
name = "datafusion"
-version = "40.0.0"
+version = "41.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab9d55a9cd2634818953809f75ebe5248b00dd43c3227efb2a51a2d5feaad54e"
+checksum = "e4fd4a99fc70d40ef7e52b243b4a399c3f8d353a40d5ecb200deee05e49c61bb"
dependencies = [
"ahash",
"arrow",
@@ -989,16 +990,18 @@ dependencies = [
"bzip2",
"chrono",
"dashmap",
+ "datafusion-catalog",
"datafusion-common",
"datafusion-common-runtime",
"datafusion-execution",
"datafusion-expr",
"datafusion-functions",
"datafusion-functions-aggregate",
- "datafusion-functions-array",
+ "datafusion-functions-nested",
"datafusion-optimizer",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
+ "datafusion-physical-optimizer",
"datafusion-physical-plan",
"datafusion-sql",
"flate2",
@@ -1026,11 +1029,25 @@ dependencies = [
"zstd",
]
+[[package]]
+name = "datafusion-catalog"
+version = "41.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e13b3cfbd84c6003594ae1972314e3df303a27ce8ce755fcea3240c90f4c0529"
+dependencies = [
+ "arrow-schema",
+ "async-trait",
+ "datafusion-common",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-physical-plan",
+]
+
[[package]]
name = "datafusion-common"
-version = "40.0.0"
+version = "41.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "def66b642959e7f96f5d2da22e1f43d3bd35598f821e5ce351a0553e0f1b7367"
+checksum = "44fdbc877e3e40dcf88cc8f283d9f5c8851f0a3aa07fee657b1b75ac1ad49b9c"
dependencies = [
"ahash",
"arrow",
@@ -1050,18 +1067,18 @@ dependencies = [
[[package]]
name = "datafusion-common-runtime"
-version = "40.0.0"
+version = "41.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f104bb9cb44c06c9badf8a0d7e0855e5f7fa5e395b887d7f835e8a9457dc1352"
+checksum = "8a7496d1f664179f6ce3a5cbef6566056ccaf3ea4aa72cc455f80e62c1dd86b1"
dependencies = [
"tokio",
]
[[package]]
name = "datafusion-execution"
-version = "40.0.0"
+version = "41.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ac0fd8b5d80bbca3fc3b6f40da4e9f6907354824ec3b18bbd83fee8cf5c3c3e"
+checksum = "799e70968c815b611116951e3dd876aef04bf217da31b72eec01ee6a959336a1"
dependencies = [
"arrow",
"chrono",
@@ -1080,9 +1097,9 @@ dependencies = [
[[package]]
name = "datafusion-expr"
-version = "40.0.0"
+version = "41.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2103d2cc16fb11ef1fa993a6cac57ed5cb028601db4b97566c90e5fa77aa1e68"
+checksum = "1c1841c409d9518c17971d15c9bae62e629eb937e6fb6c68cd32e9186f8b30d2"
dependencies = [
"ahash",
"arrow",
@@ -1099,11 +1116,12 @@ dependencies = [
[[package]]
name = "datafusion-functions"
-version = "40.0.0"
+version = "41.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a369332afd0ef5bd565f6db2139fb9f1dfdd0afa75a7f70f000b74208d76994f"
+checksum = "a8e481cf34d2a444bd8fa09b65945f0ce83dc92df8665b761505b3d9f351bebb"
dependencies = [
"arrow",
+ "arrow-buffer",
"base64",
"blake2",
"blake3",
@@ -1125,9 +1143,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate"
-version = "40.0.0"
+version = "41.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92718db1aff70c47e5abf9fc975768530097059e5db7c7b78cd64b5e9a11fc77"
+checksum = "2b4ece19f73c02727e5e8654d79cd5652de371352c1df3c4ac3e419ecd6943fb"
dependencies = [
"ahash",
"arrow",
@@ -1142,10 +1160,10 @@ dependencies = [
]
[[package]]
-name = "datafusion-functions-array"
-version = "40.0.0"
+name = "datafusion-functions-nested"
+version = "41.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30bb80f46ff3dcf4bb4510209c2ba9b8ce1b716ac8b7bf70c6bf7dca6260c831"
+checksum = "a1474552cc824e8c9c88177d454db5781d4b66757d4aca75719306b8343a5e8d"
dependencies = [
"arrow",
"arrow-array",
@@ -1160,13 +1178,14 @@ dependencies = [
"itertools 0.12.1",
"log",
"paste",
+ "rand",
]
[[package]]
name = "datafusion-optimizer"
-version = "40.0.0"
+version = "41.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82f34692011bec4fdd6fc18c264bf8037b8625d801e6dd8f5111af15cb6d71d3"
+checksum = "791ff56f55608bc542d1ea7a68a64bdc86a9413f5a381d06a39fd49c2a3ab906"
dependencies = [
"arrow",
"async-trait",
@@ -1184,9 +1203,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr"
-version = "40.0.0"
+version = "41.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "45538630defedb553771434a437f7ca8f04b9b3e834344aafacecb27dc65d5e5"
+checksum = "9a223962b3041304a3e20ed07a21d5de3d88d7e4e71ca192135db6d24e3365a4"
dependencies = [
"ahash",
"arrow",
@@ -1214,9 +1233,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-common"
-version = "40.0.0"
+version = "41.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d8a72b0ca908e074aaeca52c14ddf5c28d22361e9cb6bc79bb733cd6661b536"
+checksum = "db5e7d8532a1601cd916881db87a70b0a599900d23f3db2897d389032da53bc6"
dependencies = [
"ahash",
"arrow",
@@ -1226,11 +1245,23 @@ dependencies = [
"rand",
]
+[[package]]
+name = "datafusion-physical-optimizer"
+version = "41.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fdb9c78f308e050f5004671039786a925c3fee83b90004e9fcfd328d7febdcc0"
+dependencies = [
+ "datafusion-common",
+ "datafusion-execution",
+ "datafusion-physical-expr",
+ "datafusion-physical-plan",
+]
+
[[package]]
name = "datafusion-physical-plan"
-version = "40.0.0"
+version = "41.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b504eae6107a342775e22e323e9103f7f42db593ec6103b28605b7b7b1405c4a"
+checksum = "8d1116949432eb2d30f6362707e2846d942e491052a206f2ddcb42d08aea1ffe"
dependencies = [
"ahash",
"arrow",
@@ -1262,9 +1293,9 @@ dependencies = [
[[package]]
name = "datafusion-sql"
-version = "40.0.0"
+version = "41.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5db33f323f41b95ae201318ba654a9bf11113e58a51a1dff977b1a836d3d889"
+checksum = "b45d0180711165fe94015d7c4123eb3e1cf5fb60b1506453200b8d1ce666bef0"
dependencies = [
"arrow",
"arrow-array",
@@ -2860,7 +2891,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5bb182580f71dd070f88d01ce3de9f4da5021db7115d2e1c3605a754153b77c1"
dependencies = [
"bytes",
- "heck 0.4.1",
+ "heck 0.5.0",
"itertools 0.13.0",
"log",
"multimap",
@@ -3606,9 +3637,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
[[package]]
name = "sqlparser"
-version = "0.47.0"
+version = "0.49.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "295e9930cd7a97e58ca2a070541a3ca502b17f5d1fa7157376d0fabd85324f25"
+checksum = "a4a404d0e14905361b918cb8afdb73605e25c1d5029312bd9785142dcb3aa49e"
dependencies = [
"log",
"sqlparser_derive",
diff --git a/Cargo.toml b/Cargo.toml
index 1fea1ab2ea..016c90600b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -55,12 +55,12 @@ clap = "4.5.13"
criterion = { version = "0.5.1", features = ["html_reports"] }
croaring = "2.0.0"
csv = "1.3.0"
-datafusion = "40.0.0"
-datafusion-common = "40.0.0"
-datafusion-execution = "40.0.0"
-datafusion-expr = "40.0.0"
-datafusion-physical-expr = "40.0.0"
-datafusion-physical-plan = "40.0.0"
+datafusion = "41.0.0"
+datafusion-common = "41.0.0"
+datafusion-execution = "41.0.0"
+datafusion-expr = "41.0.0"
+datafusion-physical-expr = "41.0.0"
+datafusion-physical-plan = "41.0.0"
divan = "0.1.14"
duckdb = "1.0.0"
enum-iterator = "2.0.0"
diff --git a/vortex-datafusion/src/memory.rs b/vortex-datafusion/src/memory.rs
index 95db91c7a0..b7da3a95d6 100644
--- a/vortex-datafusion/src/memory.rs
+++ b/vortex-datafusion/src/memory.rs
@@ -3,8 +3,8 @@ use std::sync::Arc;
use arrow_schema::SchemaRef;
use async_trait::async_trait;
+use datafusion::catalog::Session;
use datafusion::datasource::TableProvider;
-use datafusion::execution::context::SessionState;
use datafusion::prelude::*;
use datafusion_common::Result as DFResult;
use datafusion_expr::{TableProviderFilterPushDown, TableType};
@@ -75,7 +75,7 @@ impl TableProvider for VortexMemTable {
/// The array is flattened directly into the nearest Arrow-compatible encoding.
async fn scan(
&self,
- state: &SessionState,
+ state: &dyn Session,
projection: Option<&Vec>,
filters: &[Expr],
_limit: Option,
@@ -194,7 +194,7 @@ fn make_filter_then_take_plan(
filter_projection: Vec,
chunked_array: ChunkedArray,
output_projection: Vec,
- _session_state: &SessionState,
+ _session_state: &dyn Session,
) -> Arc {
let row_selector_op = Arc::new(RowSelectorExec::new(
filter_exprs,
diff --git a/vortex-datafusion/src/persistent/provider.rs b/vortex-datafusion/src/persistent/provider.rs
index 3cd4433bd6..baeefd4f35 100644
--- a/vortex-datafusion/src/persistent/provider.rs
+++ b/vortex-datafusion/src/persistent/provider.rs
@@ -3,9 +3,9 @@ use std::sync::Arc;
use arrow_schema::SchemaRef;
use async_trait::async_trait;
+use datafusion::catalog::Session;
use datafusion::datasource::physical_plan::FileScanConfig;
use datafusion::datasource::TableProvider;
-use datafusion::execution::context::SessionState;
use datafusion_common::{
project_schema, DataFusionError, Result as DFResult, Statistics, ToDFSchema,
};
@@ -54,7 +54,7 @@ impl TableProvider for VortexFileTableProvider {
async fn scan(
&self,
- state: &SessionState,
+ state: &dyn Session,
projection: Option<&Vec>,
filters: &[Expr],
_limit: Option,