From bd636805e25656d9bbbb79c25352b1e9884413f5 Mon Sep 17 00:00:00 2001 From: Matthew Turner Date: Wed, 16 Oct 2024 10:13:19 -0400 Subject: [PATCH] README and comment --- Cargo.toml | 3 ++- README.md | 16 ++++++++++++++-- crates/datafusion-functions-parquet/src/lib.rs | 2 ++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 07dabc2..d872706 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ crossterm = { version = "0.28.1", features = ["event-stream"] } datafusion = "41.0.0" datafusion-common = "41.0.0" datafusion-functions-json = { version = "0.41.0", optional = true } -datafusion-functions-parquet = { version = "0.1.0", path = "crates/datafusion-functions-parquet" } +datafusion-functions-parquet = { version = "0.1.0", path = "crates/datafusion-functions-parquet", optional = true } deltalake = { version = "0.19.0", features = ["datafusion"], optional = true } directories = "5.0.1" env_logger = "0.11.5" @@ -62,6 +62,7 @@ flightsql = ["dep:arrow-flight", "dep:tonic"] experimental-flightsql-server = ["dep:arrow-flight", "dep:tonic"] s3 = ["object_store/aws", "url"] functions-json = ["dep:datafusion-functions-json"] +functions-parquet = ["dep:datafusion-functions-parquet"] [[bin]] name = "dft" diff --git a/README.md b/README.md index 302d0ed..05eb24f 100644 --- a/README.md +++ b/README.md @@ -116,9 +116,21 @@ Currently, the only supported packaging is on [crates.io](https://crates.io/sear Once installed you can run `dft` to start the application. -#### Optional Features (Rust Crate Features) +#### Internal Optional Features (Workspace Features) -`dft` has several optional (conditionally compiled features) integrations which are controlled by [Rust Crate Features] +`dft` incubates several optional features in it's `crates` directory. This provides us with the ability to quickly iterate on new features and test them in the main application while at the same time making it easy to export them to their own crates when they are ready. + +#### Parquet Functions (`--features=functions-parquet`) + +Includes functions from [datafusion-function-parquet] for querying Parquet files in DataFusion in `dft`. For example: + +```sql +SELECT * FROM parquet_metadata('my_parquet_file.parquet') +``` + +#### External Optional Features (Rust Crate Features) + +`dft` also has several external optional (conditionally compiled features) integrations which are controlled by [Rust Crate Features] To build with all features, you can run diff --git a/crates/datafusion-functions-parquet/src/lib.rs b/crates/datafusion-functions-parquet/src/lib.rs index 2708861..18de22a 100644 --- a/crates/datafusion-functions-parquet/src/lib.rs +++ b/crates/datafusion-functions-parquet/src/lib.rs @@ -36,6 +36,8 @@ use parquet::file::statistics::Statistics; use std::fs::File; use std::sync::Arc; +// Copied from https://github.com/apache/datafusion/blob/main/datafusion-cli/src/functions.rs + /// PARQUET_META table function struct ParquetMetadataTable { schema: SchemaRef,