From 4f823711d418f4e06e304ff5dac2769cc0f7a3cb Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 5 Nov 2024 11:23:05 -0700 Subject: [PATCH] chore: Prepare 43.0.0 release (#13254) * bump version and generate changelog * bump version and generate changelog * remove GH_TOKEN reference from script --- Cargo.toml | 50 +-- datafusion-cli/Cargo.lock | 369 ++++++++++++++++---- datafusion-cli/Cargo.toml | 4 +- dev/changelog/43.0.0.md | 545 ++++++++++++++++++++++++++++++ dev/release/create-tarball.sh | 5 - docs/source/user-guide/configs.md | 2 +- 6 files changed, 882 insertions(+), 93 deletions(-) create mode 100644 dev/changelog/43.0.0.md diff --git a/Cargo.toml b/Cargo.toml index 21079c484ce0..dd70535be88a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,7 +60,7 @@ license = "Apache-2.0" readme = "README.md" repository = "https://github.com/apache/datafusion" rust-version = "1.79" -version = "42.2.0" +version = "43.0.0" [workspace.dependencies] # We turn off default-features for some dependencies here so the workspaces which inherit them can @@ -93,30 +93,30 @@ bytes = "1.4" chrono = { version = "0.4.38", default-features = false } ctor = "0.2.0" dashmap = "6.0.1" -datafusion = { path = "datafusion/core", version = "42.2.0", default-features = false } -datafusion-catalog = { path = "datafusion/catalog", version = "42.2.0" } -datafusion-common = { path = "datafusion/common", version = "42.2.0", default-features = false } -datafusion-common-runtime = { path = "datafusion/common-runtime", version = "42.2.0" } -datafusion-execution = { path = "datafusion/execution", version = "42.2.0" } -datafusion-expr = { path = "datafusion/expr", version = "42.2.0" } -datafusion-expr-common = { path = "datafusion/expr-common", version = "42.2.0" } -datafusion-ffi = { path = "datafusion/ffi", version = "42.2.0" } -datafusion-functions = { path = "datafusion/functions", version = "42.2.0" } -datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", version = "42.2.0" } -datafusion-functions-aggregate-common = { path = "datafusion/functions-aggregate-common", version = "42.2.0" } -datafusion-functions-nested = { path = "datafusion/functions-nested", version = "42.2.0" } -datafusion-functions-window = { path = "datafusion/functions-window", version = "42.2.0" } -datafusion-functions-window-common = { path = "datafusion/functions-window-common", version = "42.2.0" } -datafusion-optimizer = { path = "datafusion/optimizer", version = "42.2.0", default-features = false } -datafusion-physical-expr = { path = "datafusion/physical-expr", version = "42.2.0", default-features = false } -datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "42.2.0", default-features = false } -datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", version = "42.2.0" } -datafusion-physical-plan = { path = "datafusion/physical-plan", version = "42.2.0" } -datafusion-proto = { path = "datafusion/proto", version = "42.2.0" } -datafusion-proto-common = { path = "datafusion/proto-common", version = "42.2.0" } -datafusion-sql = { path = "datafusion/sql", version = "42.2.0" } -datafusion-sqllogictest = { path = "datafusion/sqllogictest", version = "42.2.0" } -datafusion-substrait = { path = "datafusion/substrait", version = "42.2.0" } +datafusion = { path = "datafusion/core", version = "43.0.0", default-features = false } +datafusion-catalog = { path = "datafusion/catalog", version = "43.0.0" } +datafusion-common = { path = "datafusion/common", version = "43.0.0", default-features = false } +datafusion-common-runtime = { path = "datafusion/common-runtime", version = "43.0.0" } +datafusion-execution = { path = "datafusion/execution", version = "43.0.0" } +datafusion-expr = { path = "datafusion/expr", version = "43.0.0" } +datafusion-expr-common = { path = "datafusion/expr-common", version = "43.0.0" } +datafusion-ffi = { path = "datafusion/ffi", version = "43.0.0" } +datafusion-functions = { path = "datafusion/functions", version = "43.0.0" } +datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", version = "43.0.0" } +datafusion-functions-aggregate-common = { path = "datafusion/functions-aggregate-common", version = "43.0.0" } +datafusion-functions-nested = { path = "datafusion/functions-nested", version = "43.0.0" } +datafusion-functions-window = { path = "datafusion/functions-window", version = "43.0.0" } +datafusion-functions-window-common = { path = "datafusion/functions-window-common", version = "43.0.0" } +datafusion-optimizer = { path = "datafusion/optimizer", version = "43.0.0", default-features = false } +datafusion-physical-expr = { path = "datafusion/physical-expr", version = "43.0.0", default-features = false } +datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "43.0.0", default-features = false } +datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", version = "43.0.0" } +datafusion-physical-plan = { path = "datafusion/physical-plan", version = "43.0.0" } +datafusion-proto = { path = "datafusion/proto", version = "43.0.0" } +datafusion-proto-common = { path = "datafusion/proto-common", version = "43.0.0" } +datafusion-sql = { path = "datafusion/sql", version = "43.0.0" } +datafusion-sqllogictest = { path = "datafusion/sqllogictest", version = "43.0.0" } +datafusion-substrait = { path = "datafusion/substrait", version = "43.0.0" } doc-comment = "0.3" env_logger = "0.11" futures = "0.3" diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index 3348adb10386..5140ae7eb0dc 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -84,9 +84,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.17" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23a1e53f0f5d86382dafe1cf314783b2044280f406e7e1506368220ad11b1338" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" dependencies = [ "anstyle", "anstyle-parse", @@ -690,9 +690,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.7.2" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e086682a53d3aa241192aa110fa8dfce98f2f5ac2ead0de84d41582c7e8fdb96" +checksum = "92165296a47a812b267b4f41032ff8069ab7ff783696d217f0994a0d7ab585cd" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -707,9 +707,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.8" +version = "1.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07c9cdc179e6afbf5d391ab08c85eac817b51c87e1892a5edb5f7bbdc64314b4" +checksum = "4fbd94a32b3a7d55d3806fe27d98d3ad393050439dd05eb53ece36ec5e3d3510" dependencies = [ "base64-simd", "bytes", @@ -917,9 +917,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.34" +version = "1.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b9470d453346108f93a59222a9a1a5724db32d0a4727b7ab7ace4b4d822dc9" +checksum = "0f57c4b4da2a9d619dd035f27316d7a426305b75be93d09e92f2b9229c34feaf" dependencies = [ "jobserver", "libc", @@ -1188,7 +1188,7 @@ dependencies = [ [[package]] name = "datafusion" -version = "42.2.0" +version = "43.0.0" dependencies = [ "ahash", "apache-avro", @@ -1245,7 +1245,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "42.2.0" +version = "43.0.0" dependencies = [ "arrow-schema", "async-trait", @@ -1258,7 +1258,7 @@ dependencies = [ [[package]] name = "datafusion-cli" -version = "42.2.0" +version = "43.0.0" dependencies = [ "arrow", "assert_cmd", @@ -1288,7 +1288,7 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "42.2.0" +version = "43.0.0" dependencies = [ "ahash", "apache-avro", @@ -1312,7 +1312,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "42.2.0" +version = "43.0.0" dependencies = [ "log", "tokio", @@ -1320,7 +1320,7 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "42.2.0" +version = "43.0.0" dependencies = [ "arrow", "chrono", @@ -1339,7 +1339,7 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "42.2.0" +version = "43.0.0" dependencies = [ "ahash", "arrow", @@ -1361,7 +1361,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "42.2.0" +version = "43.0.0" dependencies = [ "arrow", "datafusion-common", @@ -1371,7 +1371,7 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "42.2.0" +version = "43.0.0" dependencies = [ "arrow", "arrow-buffer", @@ -1396,7 +1396,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "42.2.0" +version = "43.0.0" dependencies = [ "ahash", "arrow", @@ -1415,7 +1415,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "42.2.0" +version = "43.0.0" dependencies = [ "ahash", "arrow", @@ -1427,7 +1427,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "42.2.0" +version = "43.0.0" dependencies = [ "arrow", "arrow-array", @@ -1448,7 +1448,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "42.2.0" +version = "43.0.0" dependencies = [ "datafusion-common", "datafusion-expr", @@ -1461,7 +1461,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "42.2.0" +version = "43.0.0" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1469,7 +1469,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "42.2.0" +version = "43.0.0" dependencies = [ "arrow", "async-trait", @@ -1487,7 +1487,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "42.2.0" +version = "43.0.0" dependencies = [ "ahash", "arrow", @@ -1513,7 +1513,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "42.2.0" +version = "43.0.0" dependencies = [ "ahash", "arrow", @@ -1526,7 +1526,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "42.2.0" +version = "43.0.0" dependencies = [ "arrow", "arrow-schema", @@ -1540,7 +1540,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "42.2.0" +version = "43.0.0" dependencies = [ "ahash", "arrow", @@ -1573,7 +1573,7 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "42.2.0" +version = "43.0.0" dependencies = [ "arrow", "arrow-array", @@ -1634,6 +1634,17 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "doc-comment" version = "0.3.3" @@ -1953,9 +1964,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" [[package]] name = "heck" @@ -2193,14 +2204,143 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "idna" -version = "0.5.0" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "icu_normalizer", + "icu_properties", ] [[package]] @@ -2210,7 +2350,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.15.0", + "hashbrown 0.15.1", ] [[package]] @@ -2408,6 +2548,12 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + [[package]] name = "lock_api" version = "0.4.12" @@ -2948,9 +3094,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e346e016eacfff12233c243718197ca12f148c84e1e84268a896699b41c71780" +checksum = "7d5a626c6807713b15cac82a6acaccd6043c9a5408c24baae07611fec3f243da" dependencies = [ "cfg_aliases 0.2.1", "libc", @@ -3189,9 +3335,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.38" +version = "0.38.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a" +checksum = "375116bee2be9ed569afe2154ea6a99dfdffd257f533f187498c2a8f5feaf4ee" dependencies = [ "bitflags 2.6.0", "errno", @@ -3554,6 +3700,12 @@ dependencies = [ "syn", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "static_assertions" version = "1.1.0" @@ -3633,6 +3785,17 @@ dependencies = [ "futures-core", ] +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tempfile" version = "3.13.0" @@ -3654,18 +3817,18 @@ checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" [[package]] name = "thiserror" -version = "1.0.66" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d171f59dbaa811dbbb1aee1e73db92ec2b122911a48e1390dfe327a821ddede" +checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.66" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b08be0f17bd307950653ce45db00cd31200d82b624b36e181337d9c7d92765b5" +checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" dependencies = [ "proc-macro2", "quote", @@ -3722,6 +3885,16 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.8.0" @@ -3896,27 +4069,12 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" -[[package]] -name = "unicode-bidi" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" - [[package]] name = "unicode-ident" version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" -[[package]] -name = "unicode-normalization" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" -dependencies = [ - "tinyvec", -] - [[package]] name = "unicode-segmentation" version = "1.12.0" @@ -3937,9 +4095,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.2" +version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" dependencies = [ "form_urlencoded", "idna", @@ -3952,6 +4110,18 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -4309,6 +4479,18 @@ dependencies = [ "memchr", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + [[package]] name = "xmlparser" version = "0.13.6" @@ -4324,6 +4506,30 @@ dependencies = [ "lzma-sys", ] +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.35" @@ -4345,12 +4551,55 @@ dependencies = [ "syn", ] +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zstd" version = "0.12.4" diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index 049f87f08e69..784d47220c7c 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -18,7 +18,7 @@ [package] name = "datafusion-cli" description = "Command Line Client for DataFusion query engine." -version = "42.2.0" +version = "43.0.0" authors = ["Apache DataFusion "] edition = "2021" keywords = ["arrow", "datafusion", "query", "sql"] @@ -39,7 +39,7 @@ aws-sdk-sts = "1.43.0" # end pin aws-sdk crates aws-credential-types = "1.2.0" clap = { version = "4.5.16", features = ["derive", "cargo"] } -datafusion = { path = "../datafusion/core", version = "42.2.0", features = [ +datafusion = { path = "../datafusion/core", version = "43.0.0", features = [ "avro", "crypto_expressions", "datetime_expressions", diff --git a/dev/changelog/43.0.0.md b/dev/changelog/43.0.0.md new file mode 100644 index 000000000000..e1fcc55b4b91 --- /dev/null +++ b/dev/changelog/43.0.0.md @@ -0,0 +1,545 @@ + + +# Apache DataFusion 43.0.0 Changelog + +This release consists of 403 commits from 96 contributors. See credits at the end of this changelog for more information. + +**Breaking changes:** + +- Remove Arc wrapping from create_udf's return_type [#12489](https://github.com/apache/datafusion/pull/12489) (findepi) +- Make make_scalar_function() result candidate for inlining, by removing the `Arc` [#12477](https://github.com/apache/datafusion/pull/12477) (findepi) +- Bump MSRV to 1.78 [#12398](https://github.com/apache/datafusion/pull/12398) (comphead) +- fix: DataFusion panics with "No candidates provided" [#12469](https://github.com/apache/datafusion/pull/12469) (Weijun-H) +- Implement PartialOrd for Expr and sub fields/structs without using hash values [#12481](https://github.com/apache/datafusion/pull/12481) (ngli-me) +- Add `field` trait method to `WindowUDFImpl`, remove `return_type`/`nullable` [#12374](https://github.com/apache/datafusion/pull/12374) (jcsherin) +- parquet: Make page_index/pushdown metrics consistent with row_group metrics [#12545](https://github.com/apache/datafusion/pull/12545) (progval) +- Make SessionContext::enable_url_table consume self [#12573](https://github.com/apache/datafusion/pull/12573) (alamb) +- LexRequirement as a struct, instead of a type [#12583](https://github.com/apache/datafusion/pull/12583) (ngli-me) +- Require `Debug` for `AnalyzerRule`, `FunctionRewriter`, and `OptimizerRule` [#12556](https://github.com/apache/datafusion/pull/12556) (alamb) +- Require `Debug` for `TableProvider`, `TableProviderFactory` and `PartitionStream` [#12557](https://github.com/apache/datafusion/pull/12557) (alamb) +- Require `Debug` for `PhysicalOptimizerRule` [#12624](https://github.com/apache/datafusion/pull/12624) (AnthonyZhOon) +- Rename aggregation modules, GroupColumn [#12619](https://github.com/apache/datafusion/pull/12619) (alamb) +- Update `register_table` functions args to take `Into` [#12630](https://github.com/apache/datafusion/pull/12630) (JasonLi-cn) +- Derive `Debug` for `SessionStateBuilder`, adding `Debug` requirements to fields [#12632](https://github.com/apache/datafusion/pull/12632) (AnthonyZhOon) +- Support REPLACE INTO for INSERT statements [#12516](https://github.com/apache/datafusion/pull/12516) (fmeringdal) +- Add `PartitionEvaluatorArgs` to `WindowUDFImpl::partition_evaluator` [#12804](https://github.com/apache/datafusion/pull/12804) (jcsherin) +- Convert `rank` / `dense_rank` and `percent_rank` builtin functions to UDWF [#12718](https://github.com/apache/datafusion/pull/12718) (jatin510) +- Bug-fix: MemoryExec sort expressions do NOT refer to the projected schema [#12876](https://github.com/apache/datafusion/pull/12876) (berkaysynnada) +- Minor: add flags for temporary ddl [#12561](https://github.com/apache/datafusion/pull/12561) (hailelagi) +- Convert `BuiltInWindowFunction::{Lead, Lag}` to a user defined window function [#12857](https://github.com/apache/datafusion/pull/12857) (jcsherin) +- Improve performance for physical plan creation with many columns [#12950](https://github.com/apache/datafusion/pull/12950) (askalt) +- Improve recursive `unnest` options API [#12836](https://github.com/apache/datafusion/pull/12836) (duongcongtoai) +- fix(substrait): disallow union with a single input [#13023](https://github.com/apache/datafusion/pull/13023) (tokoko) +- feat: support arbitrary expressions in `LIMIT` plan [#13028](https://github.com/apache/datafusion/pull/13028) (jonahgao) +- Remove unused `LogicalPlan::CrossJoin` as it is unused [#13076](https://github.com/apache/datafusion/pull/13076) (buraksenn) +- Minor: make `Expr::volatile` infallible [#13206](https://github.com/apache/datafusion/pull/13206) (alamb) +- Convert LexOrdering `type` to `struct`. [#13146](https://github.com/apache/datafusion/pull/13146) (ngli-me) + +**Implemented enhancements:** + +- feat(unparser): adding alias for table scan filter in sql unparser [#12453](https://github.com/apache/datafusion/pull/12453) (Lordworms) +- feat(substrait): set ProjectRel output_mapping in producer [#12495](https://github.com/apache/datafusion/pull/12495) (vbarua) +- feat:Support applying parquet bloom filters to StringView columns [#12503](https://github.com/apache/datafusion/pull/12503) (my-vegetable-has-exploded) +- feat: Support adding a single new table factory to SessionStateBuilder [#12563](https://github.com/apache/datafusion/pull/12563) (Weijun-H) +- feat(planner): Allowing setting sort order of parquet files without specifying the schema [#12466](https://github.com/apache/datafusion/pull/12466) (devanbenz) +- feat: add support for Substrait ExtendedExpression [#12728](https://github.com/apache/datafusion/pull/12728) (westonpace) +- feat(substrait): add intersect support to consumer [#12830](https://github.com/apache/datafusion/pull/12830) (tokoko) +- feat: Implement grouping function using grouping id [#12704](https://github.com/apache/datafusion/pull/12704) (eejbyfeldt) +- feat(substrait): add set operations to consumer, update substrait to `0.45.0` [#12863](https://github.com/apache/datafusion/pull/12863) (tokoko) +- feat(substrait): add wildcard handling to producer [#12987](https://github.com/apache/datafusion/pull/12987) (tokoko) +- feat: Add regexp_count function [#12970](https://github.com/apache/datafusion/pull/12970) (Omega359) +- feat: Decorrelate more predicate subqueries [#12945](https://github.com/apache/datafusion/pull/12945) (eejbyfeldt) +- feat: Run (logical) optimizers on subqueries [#13066](https://github.com/apache/datafusion/pull/13066) (eejbyfeldt) +- feat: Convert CumeDist to UDWF [#13051](https://github.com/apache/datafusion/pull/13051) (jonathanc-n) +- feat: Migrate Map Functions [#13047](https://github.com/apache/datafusion/pull/13047) (jonathanc-n) +- feat: improve type inference for WindowFrame [#13059](https://github.com/apache/datafusion/pull/13059) (notfilippo) +- feat: Move subquery check from analyzer to PullUpCorrelatedExpr (Fix TPC-DS q41) [#13091](https://github.com/apache/datafusion/pull/13091) (eejbyfeldt) +- feat: Add `Date32`/`Date64` in aggregate fuzz testing [#13041](https://github.com/apache/datafusion/pull/13041) (LeslieKid) +- feat(substrait): support order_by in aggregate functions [#13114](https://github.com/apache/datafusion/pull/13114) (bvolpato) +- feat: Support Substrait's IntervalCompound type/literal instead of interval-month-day-nano UDT [#12112](https://github.com/apache/datafusion/pull/12112) (Blizzara) +- feat: Implement LeftMark join to fix subquery correctness issue [#13134](https://github.com/apache/datafusion/pull/13134) (eejbyfeldt) +- feat: support logical plan for `EXECUTE` statement [#13194](https://github.com/apache/datafusion/pull/13194) (jonahgao) +- feat(substrait): handle emit_kind when consuming Substrait plans [#13127](https://github.com/apache/datafusion/pull/13127) (vbarua) +- feat(substrait): AggregateRel grouping_expressions support [#13173](https://github.com/apache/datafusion/pull/13173) (akoshchiy) + +**Fixed bugs:** + +- fix: Panic/correctness issue in variance GroupsAccumulator [#12615](https://github.com/apache/datafusion/pull/12615) (eejbyfeldt) +- fix: coalesce schema issues [#12308](https://github.com/apache/datafusion/pull/12308) (mesejo) +- fix: Correct results for grouping sets when columns contain nulls [#12571](https://github.com/apache/datafusion/pull/12571) (eejbyfeldt) +- fix(substrait): remove optimize calls from substrait consumer [#12800](https://github.com/apache/datafusion/pull/12800) (tokoko) +- fix(substrait): consuming AggregateRel as last node [#12875](https://github.com/apache/datafusion/pull/12875) (tokoko) +- fix: Update TO_DATE, TO_TIMESTAMP scalar functions to support LargeUtf8, Utf8View [#12929](https://github.com/apache/datafusion/pull/12929) (Omega359) +- fix: Add Int32 type override for Dialects [#12916](https://github.com/apache/datafusion/pull/12916) (peasee) +- fix: using simple string match replace regex match for contains udf [#12931](https://github.com/apache/datafusion/pull/12931) (zhuliquan) +- fix: Dialect requires derived table alias [#12994](https://github.com/apache/datafusion/pull/12994) (peasee) +- fix: join swap for projected semi/anti joins [#13022](https://github.com/apache/datafusion/pull/13022) (korowa) +- fix: Verify supported type for Unary::Plus in sql planner [#13019](https://github.com/apache/datafusion/pull/13019) (eejbyfeldt) +- fix: Do NOT preserve names (aliases) of Exprs for simplification in TableScan filters [#13048](https://github.com/apache/datafusion/pull/13048) (eejbyfeldt) +- fix: planning of prepare statement with limit clause [#13088](https://github.com/apache/datafusion/pull/13088) (jonahgao) +- fix: add missing `NotExpr::evaluate_bounds` [#13082](https://github.com/apache/datafusion/pull/13082) (crepererum) +- fix: Order by mentioning missing column multiple times [#13158](https://github.com/apache/datafusion/pull/13158) (eejbyfeldt) +- fix: import JoinTestType without triggering unused_qualifications lint [#13170](https://github.com/apache/datafusion/pull/13170) (smarticen) +- fix: default UDWFImpl::expressions returns all expressions [#13169](https://github.com/apache/datafusion/pull/13169) (Michael-J-Ward) +- fix: date_bin() on timstamps before 1970 [#13204](https://github.com/apache/datafusion/pull/13204) (mhilton) +- fix: array_resize null fix [#13209](https://github.com/apache/datafusion/pull/13209) (jonathanc-n) +- fix: CSV Infer Schema now properly supports escaped characters. [#13214](https://github.com/apache/datafusion/pull/13214) (mnorfolk03) + +**Documentation updates:** + +- chore: Prepare 42.0.0 Release [#12465](https://github.com/apache/datafusion/pull/12465) (andygrove) +- Minor: improve ParquetOpener docs [#12456](https://github.com/apache/datafusion/pull/12456) (alamb) +- Improve doc wording around scalar authoring [#12478](https://github.com/apache/datafusion/pull/12478) (findepi) +- Minor: improve `GroupsAccumulator` docs [#12501](https://github.com/apache/datafusion/pull/12501) (alamb) +- Minor: improve `GroupsAccumulatorAdapter` docs [#12502](https://github.com/apache/datafusion/pull/12502) (alamb) +- Improve flamegraph profiling instructions [#12521](https://github.com/apache/datafusion/pull/12521) (alamb) +- docs: :memo: Add expected answers to `DataFrame` method examples [#12564](https://github.com/apache/datafusion/pull/12564) (Eason0729) +- parquet: Add finer metrics on operations covered by `time_elapsed_opening` [#12585](https://github.com/apache/datafusion/pull/12585) (progval) +- Update scalar_functions.md [#12627](https://github.com/apache/datafusion/pull/12627) (Abdullahsab3) +- Move `kurtosis_pop` to datafusion-functions-extra and out of core [#12647](https://github.com/apache/datafusion/pull/12647) (dharanad) +- Update introduction.md for `blaze` project [#12577](https://github.com/apache/datafusion/pull/12577) (liyuance) +- docs: improve the documentation for Aggregate code [#12617](https://github.com/apache/datafusion/pull/12617) (alamb) +- doc: Fix malformed hex string literal in user guide [#12708](https://github.com/apache/datafusion/pull/12708) (kawadakk) +- docs: Update DataFusion introduction to clarify that DataFusion does provide an "out of the box" query engine [#12666](https://github.com/apache/datafusion/pull/12666) (andygrove) +- Framework for generating function docs from embedded code documentation [#12668](https://github.com/apache/datafusion/pull/12668) (Omega359) +- Fix misformatted links on project index page [#12750](https://github.com/apache/datafusion/pull/12750) (amoeba) +- Add `DocumentationBuilder::with_standard_argument` to reduce copy/paste [#12747](https://github.com/apache/datafusion/pull/12747) (alamb) +- Minor: doc how field name is to be set for `WindowUDF` [#12757](https://github.com/apache/datafusion/pull/12757) (jcsherin) +- Port / Add Documentation for `VarianceSample` and `VariancePopulation` [#12742](https://github.com/apache/datafusion/pull/12742) (alamb) +- Transformed::new_transformed: Fix documentation formatting [#12787](https://github.com/apache/datafusion/pull/12787) (progval) +- Migrate documentation for all string functions from scalar_functions.md to code [#12775](https://github.com/apache/datafusion/pull/12775) (Omega359) +- Minor: add README to Catalog Folder [#12797](https://github.com/apache/datafusion/pull/12797) (jonathanc-n) +- Remove redundant aggregate/window/scalar function documentation [#12745](https://github.com/apache/datafusion/pull/12745) (alamb) +- Improve description of function migration [#12743](https://github.com/apache/datafusion/pull/12743) (alamb) +- Crypto Function Migration [#12840](https://github.com/apache/datafusion/pull/12840) (jonathanc-n) +- Minor: more doc to `MemoryPool` module [#12849](https://github.com/apache/datafusion/pull/12849) (2010YOUY01) +- Migrate documentation for all core functions from scalar_functions.md to code [#12854](https://github.com/apache/datafusion/pull/12854) (Omega359) +- Migrate documentation for Aggregate Functions to code [#12861](https://github.com/apache/datafusion/pull/12861) (jonathanc-n) +- Wordsmith project description [#12778](https://github.com/apache/datafusion/pull/12778) (matthewmturner) +- Migrate Regex Functions from static docs [#12886](https://github.com/apache/datafusion/pull/12886) (jonathanc-n) +- Migrate documentation for all math functions from scalar_functions.md to code [#12908](https://github.com/apache/datafusion/pull/12908) (juroberttyb) +- Combine the logic of rank, dense_rank and percent_rank udwf to reduce duplications [#12893](https://github.com/apache/datafusion/pull/12893) (jatin510) +- Migrate Array function Documentation to code [#12948](https://github.com/apache/datafusion/pull/12948) (jonathanc-n) +- Minor: fix Aggregation Docs from review [#12880](https://github.com/apache/datafusion/pull/12880) (jonathanc-n) +- Minor: expr-doc small fixes [#12960](https://github.com/apache/datafusion/pull/12960) (jonathanc-n) +- docs: Add documentation about conventional commits [#12971](https://github.com/apache/datafusion/pull/12971) (andygrove) +- Migrate datetime documentation to code [#12966](https://github.com/apache/datafusion/pull/12966) (jatin510) +- Fix CI on main ( regenerate function docs) [#12991](https://github.com/apache/datafusion/pull/12991) (alamb) +- Split output batches of joins that do not respect batch size [#12969](https://github.com/apache/datafusion/pull/12969) (alihan-synnada) +- Minor: Fixed regexpr_match docs [#13008](https://github.com/apache/datafusion/pull/13008) (jonathanc-n) +- Minor: Fix spelling in regexpr_count docs [#13014](https://github.com/apache/datafusion/pull/13014) (jonathanc-n) +- Update version to 42.1.0, add CHANGELOG (#12986) [#12989](https://github.com/apache/datafusion/pull/12989) (alamb) +- Added expresion to "with_standard_argument" [#12926](https://github.com/apache/datafusion/pull/12926) (jonathanc-n) +- Migrate documentation for `regr*` aggregate functions to code [#12871](https://github.com/apache/datafusion/pull/12871) (alamb) +- Minor: Add documentation for `cot` [#13069](https://github.com/apache/datafusion/pull/13069) (alamb) +- Documentation: Add API deprecation policy [#13083](https://github.com/apache/datafusion/pull/13083) (comphead) +- docs: Fixed generate_series docs [#13097](https://github.com/apache/datafusion/pull/13097) (jonathanc-n) +- [docs]: migrate lead/lag window function docs to new docs [#13095](https://github.com/apache/datafusion/pull/13095) (buraksenn) +- minor: Add deprecated policy to the contributor guide contents [#13100](https://github.com/apache/datafusion/pull/13100) (comphead) +- Introduce `binary_as_string` parquet option, upgrade to arrow/parquet `53.2.0` [#12816](https://github.com/apache/datafusion/pull/12816) (goldmedal) +- Convert `ntile` builtIn function to UDWF [#13040](https://github.com/apache/datafusion/pull/13040) (jatin510) +- docs: Added Special Functions Page [#13102](https://github.com/apache/datafusion/pull/13102) (jonathanc-n) +- [docs]: added `alternative_syntax` function for docs [#13140](https://github.com/apache/datafusion/pull/13140) (jonathanc-n) +- Minor: Delete old cume_dist and percent_rank docs [#13137](https://github.com/apache/datafusion/pull/13137) (jonathanc-n) +- docs: Add alternative syntax for extract, trim and substring. [#13143](https://github.com/apache/datafusion/pull/13143) (Omega359) +- docs: switch completely to generated docs for scalar and aggregate functions [#13161](https://github.com/apache/datafusion/pull/13161) (Omega359) +- Minor: improve testing docs, mention `cargo nextest` [#13160](https://github.com/apache/datafusion/pull/13160) (alamb) +- minor: Update HOWTO to help with updating new docs [#13172](https://github.com/apache/datafusion/pull/13172) (jonathanc-n) +- Add config option `skip_physical_aggregate_schema_check ` [#13176](https://github.com/apache/datafusion/pull/13176) (alamb) +- Enable reading `StringViewArray` by default from Parquet (8% improvement for entire ClickBench suite) [#13101](https://github.com/apache/datafusion/pull/13101) (alamb) +- Forward port changes for `42.2.0` release (#13191) [#13193](https://github.com/apache/datafusion/pull/13193) (alamb) +- [minor] overload from_unixtime func to have optional timezone parameter [#13130](https://github.com/apache/datafusion/pull/13130) (buraksenn) + +**Other:** + +- Impl `convert_to_state` for `GroupsAccumulatorAdapter` (faster median for high cardinality aggregates) [#11827](https://github.com/apache/datafusion/pull/11827) (Rachelint) +- Upgrade sqlparser-rs to 0.51.0, support new interval logic from `sqlparse-rs` [#12222](https://github.com/apache/datafusion/pull/12222) (samuelcolvin) +- Do not silently ignore unsupported `CREATE TABLE` and `CREATE VIEW` syntax [#12450](https://github.com/apache/datafusion/pull/12450) (alamb) +- use FileFormat::get_ext as the default file extension filter [#12417](https://github.com/apache/datafusion/pull/12417) (waruto210) +- fix interval units parsing [#12448](https://github.com/apache/datafusion/pull/12448) (samuelcolvin) +- test(substrait): update TPCH tests [#12462](https://github.com/apache/datafusion/pull/12462) (vbarua) +- Add "Extended Clickbench" benchmark for median and approx_median for high cardinality aggregates [#12438](https://github.com/apache/datafusion/pull/12438) (alamb) +- date_trunc small update for readability [#12479](https://github.com/apache/datafusion/pull/12479) (findepi) +- cleanup `array_has` [#12460](https://github.com/apache/datafusion/pull/12460) (samuelcolvin) +- chore: bump chrono to 0.4.38 [#12485](https://github.com/apache/datafusion/pull/12485) (my-vegetable-has-exploded) +- Remove deprecated ScalarUDF::new [#12487](https://github.com/apache/datafusion/pull/12487) (findepi) +- Remove deprecated config setup functions [#12486](https://github.com/apache/datafusion/pull/12486) (findepi) +- Remove unnecessary shifts in gcd() [#12480](https://github.com/apache/datafusion/pull/12480) (findepi) +- Return TableProviderFilterPushDown::Exact when Parquet Pushdown Enabled [#12135](https://github.com/apache/datafusion/pull/12135) (itsjunetime) +- Update substrait requirement from 0.41 to 0.42, `prost-build` to `0.13.2` [#12483](https://github.com/apache/datafusion/pull/12483) (dependabot[bot]) +- Faster strpos() string function for ASCII-only case [#12401](https://github.com/apache/datafusion/pull/12401) (goldmedal) +- Specialize ASCII case for substr() [#12444](https://github.com/apache/datafusion/pull/12444) (2010YOUY01) +- Improve SQLite subquery tables aliasing unparsing [#12482](https://github.com/apache/datafusion/pull/12482) (sgrebnov) +- Minor: use Option rather than Result for not found suggestion [#12512](https://github.com/apache/datafusion/pull/12512) (alamb) +- Remove deprecated datafusion_physical_expr::functions module [#12505](https://github.com/apache/datafusion/pull/12505) (findepi) +- Remove deprecated AggregateUDF::new [#12508](https://github.com/apache/datafusion/pull/12508) (findepi) +- Make `required_guarantees` output to be deterministic [#12484](https://github.com/apache/datafusion/pull/12484) (austin362667) +- Deprecate unused ScalarUDF::fun [#12506](https://github.com/apache/datafusion/pull/12506) (findepi) +- Remove deprecated WindowUDF::new [#12507](https://github.com/apache/datafusion/pull/12507) (findepi) +- Preserve the order of right table in NestedLoopJoinExec [#12504](https://github.com/apache/datafusion/pull/12504) (alihan-synnada) +- Improve benchmark for ltrim [#12513](https://github.com/apache/datafusion/pull/12513) (Rachelint) +- Fix: check ambiguous column reference [#12467](https://github.com/apache/datafusion/pull/12467) (HuSen8891) +- Minor: move imports to top in `row_hash.rs` [#12530](https://github.com/apache/datafusion/pull/12530) (Rachelint) +- tests: Fix typo in config setting name [#12535](https://github.com/apache/datafusion/pull/12535) (progval) +- Expose DataFrame select_exprs method [#12520](https://github.com/apache/datafusion/pull/12520) (milenkovicm) +- Replace some usages of `Expr::to_field` with `Expr::qualified_name` [#12522](https://github.com/apache/datafusion/pull/12522) (jonahgao) +- Bump aws-sdk-sso to 1.43.0, aws-sdk-sts to 1.43.0 and aws-sdk-ssooidc from 1.40.0 to 1.44.0 in /datafusion-cli [#12409](https://github.com/apache/datafusion/pull/12409) (dependabot[bot]) +- Fix NestedLoopJoin performance regression [#12531](https://github.com/apache/datafusion/pull/12531) (alihan-synnada) +- Produce informative error message on insert plan type mismatch [#12540](https://github.com/apache/datafusion/pull/12540) (findepi) +- Fix unparse table scan with the projection pushdown [#12534](https://github.com/apache/datafusion/pull/12534) (goldmedal) +- Automate sqllogictest for String, LargeString and StringView behavior [#12525](https://github.com/apache/datafusion/pull/12525) (goldmedal) +- Fix unparsing offset [#12539](https://github.com/apache/datafusion/pull/12539) (Stazer) +- support EXTRACT on intervals and durations [#12514](https://github.com/apache/datafusion/pull/12514) (nrc) +- Support List type coercion for CASE-WHEN-THEN expression [#12490](https://github.com/apache/datafusion/pull/12490) (goldmedal) +- Sort metrics alphabetically in EXPLAIN ANALYZE output [#12568](https://github.com/apache/datafusion/pull/12568) (progval) +- Add `RuntimeEnv::try_new` and deprecate `RuntimeEnv::new` [#12566](https://github.com/apache/datafusion/pull/12566) (OussamaSaoudi) +- Reorgnize the StringView tests in sqllogictests [#12572](https://github.com/apache/datafusion/pull/12572) (goldmedal) +- fix parquet infer statistics for BinaryView types [#12575](https://github.com/apache/datafusion/pull/12575) (XiangpengHao) +- Minor: add example to of assert_batches_eq [#12580](https://github.com/apache/datafusion/pull/12580) (alamb) +- Use qualified aliases to simplify searching DFSchema [#12546](https://github.com/apache/datafusion/pull/12546) (jonahgao) +- return absent stats when filters are pushed down [#12471](https://github.com/apache/datafusion/pull/12471) (waruto210) +- Minor: add new() function for ParquetReadOptions [#12579](https://github.com/apache/datafusion/pull/12579) (Smith-Cruise) +- make `Debug` for `MemoryExec` prettier [#12582](https://github.com/apache/datafusion/pull/12582) (samuelcolvin) +- Add `SessionStateBuilder::with_object_store` method [#12578](https://github.com/apache/datafusion/pull/12578) (OussamaSaoudi) +- Fix and Improve Sort Pushdown for Nested Loop and Hash Join [#12559](https://github.com/apache/datafusion/pull/12559) (berkaysynnada) +- Add Docs and Examples and helper methods to `PhysicalSortExpr` [#12589](https://github.com/apache/datafusion/pull/12589) (alamb) +- Warn instead of error for unused imports [#12588](https://github.com/apache/datafusion/pull/12588) (samuelcolvin) +- Update prost-build requirement from =0.13.2 to =0.13.3 [#12587](https://github.com/apache/datafusion/pull/12587) (dependabot[bot]) +- Add JOB benchmark dataset [1/N] (imdb dataset) [#12497](https://github.com/apache/datafusion/pull/12497) (doupache) +- Improve documentation and add `Display` impl to `EquivalenceProperties` [#12590](https://github.com/apache/datafusion/pull/12590) (alamb) +- physical-plan: Cast nested group values back to dictionary if necessary [#12586](https://github.com/apache/datafusion/pull/12586) (brancz) +- Support `Date32` for `date_trunc` function [#12603](https://github.com/apache/datafusion/pull/12603) (goldmedal) +- Avoid RowConverter for multi column grouping (10% faster clickbench queries) [#12269](https://github.com/apache/datafusion/pull/12269) (jayzhan211) +- Refactor to support recursive unnest in physical plan [#11577](https://github.com/apache/datafusion/pull/11577) (duongcongtoai) +- Use original value when comparing with dictionary column in unparser [#12610](https://github.com/apache/datafusion/pull/12610) (Sevenannn) +- Fix to unparse the plan with multiple UNION statements into an SQL string [#12605](https://github.com/apache/datafusion/pull/12605) (goldmedal) +- Keep the float information in scalar_to_sql [#12609](https://github.com/apache/datafusion/pull/12609) (Sevenannn) +- Add Dictionary String (UTF8) type to String sqllogictests [#12621](https://github.com/apache/datafusion/pull/12621) (goldmedal) +- Improve SanityChecker error message [#12595](https://github.com/apache/datafusion/pull/12595) (alamb) +- Improve performance of `trim` for string view (10%) [#12395](https://github.com/apache/datafusion/pull/12395) (Rachelint) +- Simplify `update_skip_aggregation_probe` method [#12332](https://github.com/apache/datafusion/pull/12332) (lewiszlw) +- Minor: Encapsulate type check in GroupValuesColumn, avoid panic [#12620](https://github.com/apache/datafusion/pull/12620) (alamb) +- Fix sort node deserialization from proto [#12626](https://github.com/apache/datafusion/pull/12626) (palaska) +- Minor: improve documentation to StringView trim [#12629](https://github.com/apache/datafusion/pull/12629) (alamb) +- [MINOR]: Simplifications Sort Operator [#12639](https://github.com/apache/datafusion/pull/12639) (akurmustafa) +- [Minor] Remove redundant member from RepartitionExec [#12638](https://github.com/apache/datafusion/pull/12638) (akurmustafa) +- implement nested identifier access [#12614](https://github.com/apache/datafusion/pull/12614) (Lordworms) +- [MINOR]: Rename get_arrayref_at_indices to take_arrays [#12654](https://github.com/apache/datafusion/pull/12654) (akurmustafa) +- [MINOR]: Use take_arrays in repartition , fix build [#12657](https://github.com/apache/datafusion/pull/12657) (doupache) +- Add binary_view to string_view coercion [#12643](https://github.com/apache/datafusion/pull/12643) (doupache) +- [Minor] Improve error message when bitwise\_\* operator takes wrong unsupported type [#12646](https://github.com/apache/datafusion/pull/12646) (dharanad) +- Minor: Add github link to code that was upstreamed [#12660](https://github.com/apache/datafusion/pull/12660) (alamb) +- Minor: Improve documentation on execution error handling [#12651](https://github.com/apache/datafusion/pull/12651) (alamb) +- Adds `WindowUDFImpl::reverse_expr`trait method + Support for `IGNORE NULLS` [#12662](https://github.com/apache/datafusion/pull/12662) (jcsherin) +- Fill in missing `Debug` fields for `SessionState` [#12663](https://github.com/apache/datafusion/pull/12663) (AnthonyZhOon) +- Minor: add partial assertion for skip aggregation probe [#12640](https://github.com/apache/datafusion/pull/12640) (Rachelint) +- Add more functions for string sqllogictests [#12665](https://github.com/apache/datafusion/pull/12665) (goldmedal) +- Update rstest requirement from 0.22.0 to 0.23.0 [#12678](https://github.com/apache/datafusion/pull/12678) (dependabot[bot]) +- Minor: Change LiteralGuarantee try_new to new [#12669](https://github.com/apache/datafusion/pull/12669) (pgwhalen) +- Refactor PrimitiveGroupValueBuilder to use `MaybeNullBufferBuilder` [#12623](https://github.com/apache/datafusion/pull/12623) (alamb) +- Add `value_from_statisics` to AggregateUDFImpl, remove special case for min/max/count aggregate statistics [#12296](https://github.com/apache/datafusion/pull/12296) (edmondop) +- Provide field and schema metadata missing on distinct aggregations. [#12691](https://github.com/apache/datafusion/pull/12691) (wiedld) +- [MINOR]: Simplify required_input_ordering of BoundedWindowAggExec [#12656](https://github.com/apache/datafusion/pull/12656) (akurmustafa) +- handle 0 and NULL value of NTH_VALUE function [#12676](https://github.com/apache/datafusion/pull/12676) (thinh2) +- Improve documentation for AggregateUDFImpl::value_from_stats [#12689](https://github.com/apache/datafusion/pull/12689) (alamb) +- Add support for external tables with qualified names [#12645](https://github.com/apache/datafusion/pull/12645) (OussamaSaoudi) +- Fix Regex signature types [#12690](https://github.com/apache/datafusion/pull/12690) (blaginin) +- Refactor `ByteGroupValueBuilder` to use `MaybeNullBufferBuilder` [#12681](https://github.com/apache/datafusion/pull/12681) (alamb) +- Simplify match patterns in coercion rules [#12711](https://github.com/apache/datafusion/pull/12711) (findepi) +- Remove aggregate functions dependency on frontend [#12715](https://github.com/apache/datafusion/pull/12715) (findepi) +- Minor: Remove clone in `transform_to_states` [#12707](https://github.com/apache/datafusion/pull/12707) (jayzhan211) +- Refactor tests for union sorting properties, add tests for unions and constants [#12702](https://github.com/apache/datafusion/pull/12702) (alamb) +- Fix: support Qualified Wildcard in count aggregate function [#12673](https://github.com/apache/datafusion/pull/12673) (HuSen8891) +- Reduce code duplication in `PrimitiveGroupValueBuilder` with const generics [#12703](https://github.com/apache/datafusion/pull/12703) (alamb) +- Disallow duplicated qualified field names [#12608](https://github.com/apache/datafusion/pull/12608) (eejbyfeldt) +- Optimize base64/hex decoding by pre-allocating output buffers (~2x faster) [#12675](https://github.com/apache/datafusion/pull/12675) (simonvandel) +- Allow DynamicFileCatalog support to query partitioned file [#12683](https://github.com/apache/datafusion/pull/12683) (goldmedal) +- Support `LIMIT` Push-down logical plan optimization for `Extension` nodes [#12685](https://github.com/apache/datafusion/pull/12685) (austin362667) +- Fix AvroReader: Add union resolving for nested struct arrays [#12686](https://github.com/apache/datafusion/pull/12686) (JonasDev1) +- Adds macros for creating `WindowUDF` and `WindowFunction` expression [#12693](https://github.com/apache/datafusion/pull/12693) (jcsherin) +- Support unparsing plans with both Aggregation and Window functions [#12705](https://github.com/apache/datafusion/pull/12705) (sgrebnov) +- Fix strpos invocation with dictionary and null [#12712](https://github.com/apache/datafusion/pull/12712) (findepi) +- Add IMDB(JOB) Benchmark [2/N] (imdb queries) [#12529](https://github.com/apache/datafusion/pull/12529) (austin362667) +- Minor: avoid clone while calculating union equivalence properties [#12722](https://github.com/apache/datafusion/pull/12722) (alamb) +- Simplify streaming_merge function parameters [#12719](https://github.com/apache/datafusion/pull/12719) (mertak-synnada) +- Provide field and schema metadata missing on cross joins, and union with null fields. [#12729](https://github.com/apache/datafusion/pull/12729) (wiedld) +- Minor: Update string tests for strpos [#12739](https://github.com/apache/datafusion/pull/12739) (alamb) +- Apply `type_union_resolution` to array and values [#12753](https://github.com/apache/datafusion/pull/12753) (jayzhan211) +- fix `equal_to` in `PrimitiveGroupValueBuilder` [#12758](https://github.com/apache/datafusion/pull/12758) (Rachelint) +- Fix `equal_to` in `ByteGroupValueBuilder` [#12770](https://github.com/apache/datafusion/pull/12770) (alamb) +- Allow boolean Expr simplification even when nullable [#12746](https://github.com/apache/datafusion/pull/12746) (eejbyfeldt) +- Fix unnest conjunction with selecting wildcard expression [#12760](https://github.com/apache/datafusion/pull/12760) (goldmedal) +- Improve `round` scalar function unparsing for Postgres [#12744](https://github.com/apache/datafusion/pull/12744) (sgrebnov) +- Fix stack overflow calculating projected orderings [#12759](https://github.com/apache/datafusion/pull/12759) (alamb) +- Upgrade arrow/parquet to `53.1.0` / fix clippy [#12724](https://github.com/apache/datafusion/pull/12724) (alamb) +- Account for constant equivalence properties in union, tests [#12562](https://github.com/apache/datafusion/pull/12562) (alamb) +- Minor: clarify comment about empty dependencies [#12786](https://github.com/apache/datafusion/pull/12786) (alamb) +- Introduce Signature::String and return error if input of `strpos` is integer [#12751](https://github.com/apache/datafusion/pull/12751) (jayzhan211) +- Minor: improve docs on MovingMin/MovingMax [#12790](https://github.com/apache/datafusion/pull/12790) (alamb) +- Add union sorting equivalence end to end tests [#12721](https://github.com/apache/datafusion/pull/12721) (alamb) +- Fix bug in TopK aggregates [#12766](https://github.com/apache/datafusion/pull/12766) (avantgardnerio) +- Minor: clean up TODO comments in unnest.slt [#12795](https://github.com/apache/datafusion/pull/12795) (goldmedal) +- Refactor `DependencyMap` and `Dependencies` into structs [#12761](https://github.com/apache/datafusion/pull/12761) (alamb) +- Remove unnecessary `DFSchema::check_ambiguous_name` [#12805](https://github.com/apache/datafusion/pull/12805) (jonahgao) +- API from `ParquetExec` to `ParquetExecBuilder` [#12799](https://github.com/apache/datafusion/pull/12799) (alamb) +- Minor: add documentation note about `NullState` [#12791](https://github.com/apache/datafusion/pull/12791) (alamb) +- Chore: Move `aggregate statistics` optimizer test from core to optimizer crate [#12783](https://github.com/apache/datafusion/pull/12783) (jayzhan211) +- Clarify documentation on ArrowBytesMap and ArrowBytesViewMap [#12789](https://github.com/apache/datafusion/pull/12789) (alamb) +- Bump cookie and express in /datafusion/wasmtest/datafusion-wasm-app [#12825](https://github.com/apache/datafusion/pull/12825) (dependabot[bot]) +- Remove unused dependencies and features [#12808](https://github.com/apache/datafusion/pull/12808) (jonahgao) +- Add Aggregation fuzzer framework [#12667](https://github.com/apache/datafusion/pull/12667) (Rachelint) +- Retry apt-get and rustup on CI [#12714](https://github.com/apache/datafusion/pull/12714) (findepi) +- Support creating tables via SQL with `FixedSizeList` column (e.g. `a int[3]`) [#12810](https://github.com/apache/datafusion/pull/12810) (jandremarais) +- Make HashJoinExec::join_schema public [#12807](https://github.com/apache/datafusion/pull/12807) (progval) +- Fix convert_to_state bug in `GroupsAccumulatorAdapter` [#12834](https://github.com/apache/datafusion/pull/12834) (alamb) +- Fix: approx_percentile_cont_with_weight Panic [#12823](https://github.com/apache/datafusion/pull/12823) (jonathanc-n) +- Fix clippy error on wasmtest [#12844](https://github.com/apache/datafusion/pull/12844) (jonahgao) +- Fix panic on wrong number of arguments to substr [#12837](https://github.com/apache/datafusion/pull/12837) (eejbyfeldt) +- Fix Bug in Display for ScalarValue::Struct [#12856](https://github.com/apache/datafusion/pull/12856) (avantgardnerio) +- Support DictionaryString for Regex matching operators [#12768](https://github.com/apache/datafusion/pull/12768) (blaginin) +- Minor: Small comment changes in sql folder [#12838](https://github.com/apache/datafusion/pull/12838) (jonathanc-n) +- Add DuckDB struct test and row as alias [#12841](https://github.com/apache/datafusion/pull/12841) (jayzhan211) +- Support struct coercion in `type_union_resolution` [#12839](https://github.com/apache/datafusion/pull/12839) (jayzhan211) +- Added check for aggregate functions in optimizer rules [#12860](https://github.com/apache/datafusion/pull/12860) (jonathanc-n) +- Optimize `iszero` function (3-5x faster) [#12881](https://github.com/apache/datafusion/pull/12881) (simonvandel) +- Macro for creating record batch from literal slice [#12846](https://github.com/apache/datafusion/pull/12846) (timsaucer) +- Implement special min/max accumulator for Strings and Binary (10% faster for Clickbench Q28) [#12792](https://github.com/apache/datafusion/pull/12792) (alamb) +- Make PruningPredicate's rewrite public [#12850](https://github.com/apache/datafusion/pull/12850) (adriangb) +- octet_length + string view == ❤️ [#12900](https://github.com/apache/datafusion/pull/12900) (Omega359) +- Remove Expr clones in `select_to_plan` [#12887](https://github.com/apache/datafusion/pull/12887) (jonahgao) +- Minor: added to docs in expr folder [#12882](https://github.com/apache/datafusion/pull/12882) (jonathanc-n) +- Print undocumented functions to console while generating docs [#12874](https://github.com/apache/datafusion/pull/12874) (alamb) +- Fix: handle NULL offset of NTH_VALUE window function [#12851](https://github.com/apache/datafusion/pull/12851) (HuSen8891) +- Optimize `signum` function (3-25x faster) [#12890](https://github.com/apache/datafusion/pull/12890) (simonvandel) +- re-export PartitionEvaluatorArgs from datafusion_expr::function [#12878](https://github.com/apache/datafusion/pull/12878) (Michael-J-Ward) +- Unparse Sort with pushdown limit to SQL string [#12873](https://github.com/apache/datafusion/pull/12873) (goldmedal) +- Add spilling related metrics for aggregation [#12888](https://github.com/apache/datafusion/pull/12888) (2010YOUY01) +- Move equivalence fuzz testing to fuzz test binary [#12767](https://github.com/apache/datafusion/pull/12767) (alamb) +- Remove unused `math_expressions.rs` [#12917](https://github.com/apache/datafusion/pull/12917) (jonahgao) +- Improve AggregationFuzzer error reporting [#12832](https://github.com/apache/datafusion/pull/12832) (alamb) +- Import Arc consistently [#12899](https://github.com/apache/datafusion/pull/12899) (findepi) +- Optimize `isnan` (2-5x faster) [#12889](https://github.com/apache/datafusion/pull/12889) (simonvandel) +- Minor: Move StringArrayType, StringViewArrayBuilder, etc outside of string module [#12912](https://github.com/apache/datafusion/pull/12912) (Omega359) +- Remove redundant unsafe in test [#12914](https://github.com/apache/datafusion/pull/12914) (findepi) +- Ensure that math functions fulfil the ColumnarValue contract [#12922](https://github.com/apache/datafusion/pull/12922) (joroKr21) +- Optimization: support push down limit when full join [#12963](https://github.com/apache/datafusion/pull/12963) (JasonLi-cn) +- Implement `GroupColumn` support for `StringView` / `ByteView` (faster grouping performance) [#12809](https://github.com/apache/datafusion/pull/12809) (Rachelint) +- Implement native support StringView for `REGEXP_LIKE` [#12897](https://github.com/apache/datafusion/pull/12897) (tlm365) +- Minor: Refactor benchmark imports to use `util` module [#12885](https://github.com/apache/datafusion/pull/12885) (loloxwg) +- Fix zero data type in `expr % 1` simplification [#12913](https://github.com/apache/datafusion/pull/12913) (eejbyfeldt) +- Optimize performance of `math::cot` (~2x faster) [#12910](https://github.com/apache/datafusion/pull/12910) (tlm365) +- Expand wildcard expressions in distinct on [#12941](https://github.com/apache/datafusion/pull/12941) (epsio-banay) +- chores: remove redundant clone [#12964](https://github.com/apache/datafusion/pull/12964) (JasonLi-cn) +- Fix: handle NULL input in lead/lag window function [#12811](https://github.com/apache/datafusion/pull/12811) (HuSen8891) +- Fix logical vs physical schema mismatch for aliased `now()` [#12951](https://github.com/apache/datafusion/pull/12951) (wiedld) +- Optimize performance of `math::trunc` (~2.5x faster) [#12909](https://github.com/apache/datafusion/pull/12909) (tlm365) +- Minor: Add slt test for `DISTINCT ON` with wildcard [#12968](https://github.com/apache/datafusion/pull/12968) (alamb) +- Fix 'Too many open files' on fuzz test. [#12961](https://github.com/apache/datafusion/pull/12961) (dhegberg) +- Increase minimum supported Rust version (MSRV) to 1.79 [#12962](https://github.com/apache/datafusion/pull/12962) (findepi) +- Unparse `SubqueryAlias` without projections to SQL [#12896](https://github.com/apache/datafusion/pull/12896) (goldmedal) +- Fix 2 bugs related to push down partition filters [#12902](https://github.com/apache/datafusion/pull/12902) (eejbyfeldt) +- Move TableConstraint to Constraints conversion [#12953](https://github.com/apache/datafusion/pull/12953) (findepi) +- Added current_timestamp alias [#12958](https://github.com/apache/datafusion/pull/12958) (jonathanc-n) +- Improve unparsing for `ORDER BY`, `UNION`, Windows functions with Aggregation [#12946](https://github.com/apache/datafusion/pull/12946) (sgrebnov) +- Handle one-element array return value in ScalarFunctionExpr [#12965](https://github.com/apache/datafusion/pull/12965) (joroKr21) +- Add links to new_constraint_from_table_constraints doc [#12995](https://github.com/apache/datafusion/pull/12995) (findepi) +- Fix：fix HashJoin projection swap [#12967](https://github.com/apache/datafusion/pull/12967) (my-vegetable-has-exploded) +- refactor(substrait): refactor ReadRel consumer [#12983](https://github.com/apache/datafusion/pull/12983) (tokoko) +- Move SMJ join filtered part out of join_output stage. LeftOuter, LeftSemi [#12764](https://github.com/apache/datafusion/pull/12764) (comphead) +- Remove logical cross join in planning [#12985](https://github.com/apache/datafusion/pull/12985) (Dandandan) +- [MINOR]: Use arrow take_arrays, remove datafusion take_arrays [#13013](https://github.com/apache/datafusion/pull/13013) (akurmustafa) +- Don't preserve functional dependency when generating UNION logical plan [#12979](https://github.com/apache/datafusion/pull/12979) (Sevenannn) +- [Minor]: Add data based sort expression test [#12992](https://github.com/apache/datafusion/pull/12992) (akurmustafa) +- Removed last usages of scalar_inputs, scalar_input_types and inputs2 to use arrow unary/binary for performance [#12972](https://github.com/apache/datafusion/pull/12972) (buraksenn) +- Minor: Update release instructions to include new crates [#13024](https://github.com/apache/datafusion/pull/13024) (alamb) +- Extract CSE logic to `datafusion_common` [#13002](https://github.com/apache/datafusion/pull/13002) (peter-toth) +- Enhance table scan unparsing to avoid unnamed subqueries. [#13006](https://github.com/apache/datafusion/pull/13006) (goldmedal) +- Fix count on all null `VALUES` clause [#13029](https://github.com/apache/datafusion/pull/13029) (findepi) +- Support filter in cross join elimination [#13025](https://github.com/apache/datafusion/pull/13025) (Dandandan) +- [minor]: remove same util functions from the code base. [#13026](https://github.com/apache/datafusion/pull/13026) (akurmustafa) +- Improve `AggregateFuzz` testing: generate random queries [#12847](https://github.com/apache/datafusion/pull/12847) (alamb) +- Fix functions with Volatility::Volatile and parameters [#13001](https://github.com/apache/datafusion/pull/13001) (agscpp) +- refactor: Incorporate RewriteDisjunctivePredicate rule into SimplifyExpressions [#13032](https://github.com/apache/datafusion/pull/13032) (eejbyfeldt) +- Move filtered SMJ right join out of `join_partial` phase [#13053](https://github.com/apache/datafusion/pull/13053) (comphead) +- Remove functions and types deprecated since 37 [#13056](https://github.com/apache/datafusion/pull/13056) (findepi) +- Minor: Cleaned physical-plan Comments [#13055](https://github.com/apache/datafusion/pull/13055) (jonathanc-n) +- improve the condition checking for unparsing table_scan [#13062](https://github.com/apache/datafusion/pull/13062) (goldmedal) +- minor: simplify associated item bound of `hash_array_primitive` [#13070](https://github.com/apache/datafusion/pull/13070) (jonahgao) +- extended log.rs tests for unary/binary and f32/f64 casting [#13034](https://github.com/apache/datafusion/pull/13034) (buraksenn) +- Fix check_not_null_constraints null detection [#13033](https://github.com/apache/datafusion/pull/13033) (findepi) +- [Minor] Update info/list of TPC-DS queries [#13075](https://github.com/apache/datafusion/pull/13075) (Dandandan) +- Fix logical vs physical schema mismatch for UNION where some inputs are constants [#12954](https://github.com/apache/datafusion/pull/12954) (wiedld) +- Improve CSE stats [#13080](https://github.com/apache/datafusion/pull/13080) (peter-toth) +- Infer data type from schema for `Values` and add struct coercion to `coalesce` [#12864](https://github.com/apache/datafusion/pull/12864) (jayzhan211) +- [minor]: use arrow take_batch instead of get_record_batch_indices [#13084](https://github.com/apache/datafusion/pull/13084) (akurmustafa) +- chore: Added a number of physical planning join benchmarks [#13085](https://github.com/apache/datafusion/pull/13085) (mnorfolk03) +- Fix more instances of schema missing metadata [#13068](https://github.com/apache/datafusion/pull/13068) (itsjunetime) +- Bug-fix / Limit with_new_exprs() [#13109](https://github.com/apache/datafusion/pull/13109) (berkaysynnada) +- Minor: doc IMDB in benchmark README [#13107](https://github.com/apache/datafusion/pull/13107) (2010YOUY01) +- removed --prefer_hash_join option from parquet_filter command. [#13106](https://github.com/apache/datafusion/pull/13106) (neyama) +- Make CI error if a function has no documentation [#12938](https://github.com/apache/datafusion/pull/12938) (alamb) +- Allow using `cargo nextest` for running tests [#13045](https://github.com/apache/datafusion/pull/13045) (alamb) +- Add benchmark for memory-limited aggregation [#13090](https://github.com/apache/datafusion/pull/13090) (2010YOUY01) +- Add clickbench parquet based queries to sql_planner benchmark [#13103](https://github.com/apache/datafusion/pull/13103) (Omega359) +- Improve documentation and examples for `SchemaAdapterFactory`, make `record_batch` "hygenic" [#13063](https://github.com/apache/datafusion/pull/13063) (alamb) +- Move filtered SMJ Left Anti filtered join out of `join_partial` phase [#13111](https://github.com/apache/datafusion/pull/13111) (comphead) +- Improve TableScan with filters pushdown unparsing (multiple filters) [#13131](https://github.com/apache/datafusion/pull/13131) (sgrebnov) +- Raise a plan error on union if column count is not the same between plans [#13117](https://github.com/apache/datafusion/pull/13117) (Omega359) +- Add basic support for `unnest` unparsing [#13129](https://github.com/apache/datafusion/pull/13129) (sgrebnov) +- Improve TableScan with filters pushdown unparsing (joins) [#13132](https://github.com/apache/datafusion/pull/13132) (sgrebnov) +- Report offending plan node when In/Exist subquery misused [#13155](https://github.com/apache/datafusion/pull/13155) (findepi) +- Remove unused assert_analyzed_plan_ne test helper [#13121](https://github.com/apache/datafusion/pull/13121) (findepi) +- Fix Utf8View as Join Key [#13115](https://github.com/apache/datafusion/pull/13115) (demetribu) +- Add Support for `modulus` operation in substrait [#13108](https://github.com/apache/datafusion/pull/13108) (LatrecheYasser) +- unify cast_to function of ScalarValue [#13122](https://github.com/apache/datafusion/pull/13122) (JasonLi-cn) +- Add unused_qualifications rustic lint with deny lint level. [#13086](https://github.com/apache/datafusion/pull/13086) (dhegberg) +- [Optimization] Infer predicate under all JoinTypes [#13081](https://github.com/apache/datafusion/pull/13081) (JasonLi-cn) +- Support `negate` arithmetic expression in substrait [#13112](https://github.com/apache/datafusion/pull/13112) (LatrecheYasser) +- Fix to_char signature ordering [#13126](https://github.com/apache/datafusion/pull/13126) (Omega359) +- chore: re-export functions_window_common::ExpressionArgs [#13149](https://github.com/apache/datafusion/pull/13149) (Michael-J-Ward) +- minor: Fix build on main [#13159](https://github.com/apache/datafusion/pull/13159) (eejbyfeldt) +- minor: Update test case for issue #5771 showing it is resolved [#13180](https://github.com/apache/datafusion/pull/13180) (eejbyfeldt) +- Test LIKE with dynamic pattern [#13141](https://github.com/apache/datafusion/pull/13141) (findepi) +- Increase fuzz testing of streaming group by / low cardinality columns [#12990](https://github.com/apache/datafusion/pull/12990) (alamb) +- FFI initial implementation [#12920](https://github.com/apache/datafusion/pull/12920) (timsaucer) +- Report file location and offset when CSV schema mismatch [#13185](https://github.com/apache/datafusion/pull/13185) (findepi) +- Round robin polling between tied winners in sort preserving merge [#13133](https://github.com/apache/datafusion/pull/13133) (jayzhan211) +- Fix rendering of dictionary empty string values in SLT tests [#13198](https://github.com/apache/datafusion/pull/13198) (findepi) +- Improve push down filter of join [#13184](https://github.com/apache/datafusion/pull/13184) (JasonLi-cn) +- Minor: Reduce indirection for finding changlog [#13199](https://github.com/apache/datafusion/pull/13199) (alamb) +- Support `DictionaryArray` in `OVER` clause [#13153](https://github.com/apache/datafusion/pull/13153) (adriangb) +- Allow testing records with sibling whitespace in SLT tests and add more string tests [#13197](https://github.com/apache/datafusion/pull/13197) (findepi) +- Use single file write when an extension is present in the path. [#13079](https://github.com/apache/datafusion/pull/13079) (dhegberg) +- Deprecate ScalarUDF::invoke and invoke_no_args for invoke_batch [#13179](https://github.com/apache/datafusion/pull/13179) (findepi) +- consider volatile function in simply_expression [#13128](https://github.com/apache/datafusion/pull/13128) (Lordworms) +- Fix CI compile failure due to merge conflict [#13219](https://github.com/apache/datafusion/pull/13219) (alamb) +- Revert "Improve push down filter of join (#13184)" [#13229](https://github.com/apache/datafusion/pull/13229) (eejbyfeldt) +- Derive `Clone` for more ExecutionPlans [#13203](https://github.com/apache/datafusion/pull/13203) (alamb) +- feat(logical-types): add NativeType and LogicalType [#12853](https://github.com/apache/datafusion/pull/12853) (notfilippo) +- Apply projection to `Statistics` in `FilterExec` [#13187](https://github.com/apache/datafusion/pull/13187) (alamb) +- Minor: make LeftJoinData into a struct in CrossJoinExec [#13227](https://github.com/apache/datafusion/pull/13227) (alamb) +- Deprecate invoke and invoke_no_args in favor of invoke_batch [#13174](https://github.com/apache/datafusion/pull/13174) (findepi) +- Support timestamp(n) SQL type [#13231](https://github.com/apache/datafusion/pull/13231) (findepi) +- Remove elements deprecated since v 38. [#13245](https://github.com/apache/datafusion/pull/13245) (findepi) + +## Credits + +Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. + +``` + 68 Andrew Lamb + 34 Piotr Findeisen + 24 Jonathan Chen + 19 Emil Ejbyfeldt + 17 Jax Liu + 12 Bruce Ritchie + 11 Jonah Gao + 9 Jay Zhan + 8 Mustafa Akur + 8 kamille + 7 Sergei Grebnov + 7 Tornike Gurgenidze + 6 JasonLi + 6 Oleks V + 6 Val Lorentz + 6 jcsherin + 5 Burak Şen + 5 Samuel Colvin + 5 Yongting You + 5 dependabot[bot] + 4 HuSen + 4 Jagdish Parihar + 4 Simon Vandel Sillesen + 4 wiedld + 3 Alihan Çelikcan + 3 Andy Grove + 3 AnthonyZhOon + 3 Austin Liu + 3 Berkay Şahin + 3 Daniel Hegberg + 3 Daniël Heres + 3 Lordworms + 3 Michael J Ward + 3 OussamaSaoudi + 3 Qianqian + 3 Tai Le Manh + 3 Victor Barua + 3 doupache + 3 ngli-me + 3 yi wang + 2 Adrian Garcia Badaracco + 2 Alex Huang + 2 Brent Gardner + 2 Dharan Aditya + 2 Dmitrii Blaginin + 2 Duong Cong Toai + 2 Filippo Rossi + 2 Georgi Krastev + 2 June + 2 Max Norfolk + 2 Peter Toth + 2 Tim Saucer + 2 Yasser Latreche + 2 peasee + 2 waruto + 1 Abdullah Sabaa Allil + 1 Agaev Guseyn + 1 Albert Skalt + 1 Andrey Koshchiy + 1 Arttu + 1 Baris Palaska + 1 Bruno Volpato + 1 Bryce Mecum + 1 Daniel Mesejo + 1 Dmitry Bugakov + 1 Eason + 1 Edmondo Porcu + 1 Eduard Karacharov + 1 Frederic Branczyk + 1 Fredrik Meringdal + 1 Haile + 1 Jan + 1 JonasDev1 + 1 Justus Flerlage + 1 Leslie Su + 1 Marco Neumann + 1 Marko Milenković + 1 Martin Hilton + 1 Matthew Turner + 1 Nick Cameron + 1 Paul + 1 Smith Cruise + 1 Tomoaki Kawada + 1 WeblWabl + 1 Weston Pace + 1 Xiangpeng Hao + 1 Xwg + 1 Yuance.Li + 1 epsio-banay + 1 iamthinh + 1 juroberttyb + 1 mertak-synnada + 1 neyama + 1 smarticen + 1 zhuliquan + 1 张林伟 +``` + +Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. diff --git a/dev/release/create-tarball.sh b/dev/release/create-tarball.sh index c43e02bbbba5..1e59fad2691c 100755 --- a/dev/release/create-tarball.sh +++ b/dev/release/create-tarball.sh @@ -53,11 +53,6 @@ if [ "$#" -ne 2 ]; then exit fi -if [[ -z "${GH_TOKEN}" ]]; then - echo "Please set personal github token through GH_TOKEN environment variable" - exit -fi - version=$1 rc=$2 tag="${version}-rc${rc}" diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index bd8591b5d723..6a49fda668a9 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -67,7 +67,7 @@ Environment variables are read during `SessionConfig` initialisation so they mus | datafusion.execution.parquet.statistics_enabled | page | (writing) Sets if statistics are enabled for any column Valid values are: "none", "chunk", and "page" These values are not case sensitive. If NULL, uses default parquet writer setting | | datafusion.execution.parquet.max_statistics_size | 4096 | (writing) Sets max statistics size for any column. If NULL, uses default parquet writer setting | | datafusion.execution.parquet.max_row_group_size | 1048576 | (writing) Target maximum number of rows in each row group (defaults to 1M rows). Writing larger row groups requires more memory to write, but can get better compression and be faster to read. | -| datafusion.execution.parquet.created_by | datafusion version 42.2.0 | (writing) Sets "created by" property | +| datafusion.execution.parquet.created_by | datafusion version 43.0.0 | (writing) Sets "created by" property | | datafusion.execution.parquet.column_index_truncate_length | 64 | (writing) Sets column index truncate length | | datafusion.execution.parquet.data_page_row_count_limit | 20000 | (writing) Sets best effort maximum number of rows in data page | | datafusion.execution.parquet.encoding | NULL | (writing) Sets default encoding for any column. Valid values are: plain, plain_dictionary, rle, bit_packed, delta_binary_packed, delta_length_byte_array, delta_byte_array, rle_dictionary, and byte_stream_split. These values are not case sensitive. If NULL, uses default parquet writer setting |