diff --git a/Cargo.lock b/Cargo.lock index d0d5358626..59eefc6eec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -118,32 +118,68 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219d05930b81663fd3b32e3bde8ce5bff3c4d23052a99f11a8fa50a3b47b2658" dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-csv", - "arrow-data", - "arrow-ipc", - "arrow-json", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", + "arrow-arith 51.0.0", + "arrow-array 51.0.0", + "arrow-buffer 51.0.0", + "arrow-cast 51.0.0", + "arrow-csv 51.0.0", + "arrow-data 51.0.0", + "arrow-ipc 51.0.0", + "arrow-json 51.0.0", + "arrow-ord 51.0.0", + "arrow-row 51.0.0", + "arrow-schema 51.0.0", + "arrow-select 51.0.0", + "arrow-string 51.0.0", "pyo3", ] +[[package]] +name = "arrow" +version = "52.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ae9728f104939be6d8d9b368a354b4929b0569160ea1641f0721b55a861ce38" +dependencies = [ + "arrow-arith 52.0.0", + "arrow-array 52.0.0", + "arrow-buffer 52.0.0", + "arrow-cast 52.0.0", + "arrow-csv 52.0.0", + "arrow-data 52.0.0", + "arrow-ipc 52.0.0", + "arrow-json 52.0.0", + "arrow-ord 52.0.0", + "arrow-row 52.0.0", + "arrow-schema 52.0.0", + "arrow-select 52.0.0", + "arrow-string 52.0.0", +] + [[package]] name = "arrow-arith" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0272150200c07a86a390be651abdd320a2d12e84535f0837566ca87ecd8f95e0" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 51.0.0", + "arrow-buffer 51.0.0", + "arrow-data 51.0.0", + "arrow-schema 51.0.0", + "chrono", + "half", + "num", +] + +[[package]] +name = "arrow-arith" +version = "52.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7029a5b3efbeafbf4a12d12dc16b8f9e9bff20a410b8c25c5d28acc089e1043" +dependencies = [ + "arrow-array 52.0.0", + "arrow-buffer 52.0.0", + "arrow-data 52.0.0", + "arrow-schema 52.0.0", "chrono", "half", "num", @@ -156,11 +192,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8010572cf8c745e242d1b632bd97bd6d4f40fefed5ed1290a8f433abaa686fea" dependencies = [ "ahash", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-buffer 51.0.0", + "arrow-data 51.0.0", + "arrow-schema 51.0.0", "chrono", - "chrono-tz", + "chrono-tz 0.8.6", + "half", + "hashbrown", + "num", +] + +[[package]] +name = "arrow-array" +version = "52.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d33238427c60271710695f17742f45b1a5dc5bcfc5c15331c25ddfe7abf70d97" +dependencies = [ + "ahash", + "arrow-buffer 52.0.0", + "arrow-data 52.0.0", + "arrow-schema 52.0.0", + "chrono", + "chrono-tz 0.9.0", "half", "hashbrown", "num", @@ -177,17 +230,49 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-buffer" +version = "52.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe9b95e825ae838efaf77e366c00d3fc8cca78134c9db497d6bda425f2e7b7c1" +dependencies = [ + "bytes", + "half", + "num", +] + [[package]] name = "arrow-cast" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9abc10cd7995e83505cc290df9384d6e5412b207b79ce6bdff89a10505ed2cba" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 51.0.0", + "arrow-buffer 51.0.0", + "arrow-data 51.0.0", + "arrow-schema 51.0.0", + "arrow-select 51.0.0", + "atoi", + "base64 0.22.1", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num", + "ryu", +] + +[[package]] +name = "arrow-cast" +version = "52.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cf8385a9d5b5fcde771661dd07652b79b9139fea66193eda6a88664400ccab" +dependencies = [ + "arrow-array 52.0.0", + "arrow-buffer 52.0.0", + "arrow-data 52.0.0", + "arrow-schema 52.0.0", + "arrow-select 52.0.0", "atoi", "base64 0.22.1", "chrono", @@ -204,11 +289,30 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95cbcba196b862270bf2a5edb75927380a7f3a163622c61d40cbba416a6305f2" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", + "arrow-array 51.0.0", + "arrow-buffer 51.0.0", + "arrow-cast 51.0.0", + "arrow-data 51.0.0", + "arrow-schema 51.0.0", + "chrono", + "csv", + "csv-core", + "lazy_static", + "lexical-core", + "regex", +] + +[[package]] +name = "arrow-csv" +version = "52.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cea5068bef430a86690059665e40034625ec323ffa4dd21972048eebb0127adc" +dependencies = [ + "arrow-array 52.0.0", + "arrow-buffer 52.0.0", + "arrow-cast 52.0.0", + "arrow-data 52.0.0", + "arrow-schema 52.0.0", "chrono", "csv", "csv-core", @@ -223,8 +327,20 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2742ac1f6650696ab08c88f6dd3f0eb68ce10f8c253958a18c943a68cd04aec5" dependencies = [ - "arrow-buffer", - "arrow-schema", + "arrow-buffer 51.0.0", + "arrow-schema 51.0.0", + "half", + "num", +] + +[[package]] +name = "arrow-data" +version = "52.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb29be98f987bcf217b070512bb7afba2f65180858bca462edf4a39d84a23e10" +dependencies = [ + "arrow-buffer 52.0.0", + "arrow-schema 52.0.0", "half", "num", ] @@ -235,27 +351,61 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a42ea853130f7e78b9b9d178cb4cd01dee0f78e64d96c2949dc0a915d6d9e19d" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", - "flatbuffers", + "arrow-array 51.0.0", + "arrow-buffer 51.0.0", + "arrow-cast 51.0.0", + "arrow-data 51.0.0", + "arrow-schema 51.0.0", + "flatbuffers 23.5.26", "lz4_flex", "zstd", ] +[[package]] +name = "arrow-ipc" +version = "52.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffc68f6523970aa6f7ce1dc9a33a7d9284cfb9af77d4ad3e617dbe5d79cc6ec8" +dependencies = [ + "arrow-array 52.0.0", + "arrow-buffer 52.0.0", + "arrow-cast 52.0.0", + "arrow-data 52.0.0", + "arrow-schema 52.0.0", + "flatbuffers 24.3.25", +] + [[package]] name = "arrow-json" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaafb5714d4e59feae964714d724f880511500e3569cc2a94d02456b403a2a49" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", + "arrow-array 51.0.0", + "arrow-buffer 51.0.0", + "arrow-cast 51.0.0", + "arrow-data 51.0.0", + "arrow-schema 51.0.0", + "chrono", + "half", + "indexmap", + "lexical-core", + "num", + "serde", + "serde_json", +] + +[[package]] +name = "arrow-json" +version = "52.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2041380f94bd6437ab648e6c2085a045e45a0c44f91a1b9a4fe3fed3d379bfb1" +dependencies = [ + "arrow-array 52.0.0", + "arrow-buffer 52.0.0", + "arrow-cast 52.0.0", + "arrow-data 52.0.0", + "arrow-schema 52.0.0", "chrono", "half", "indexmap", @@ -271,11 +421,26 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e6b61e3dc468f503181dccc2fc705bdcc5f2f146755fa5b56d0a6c5943f412" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 51.0.0", + "arrow-buffer 51.0.0", + "arrow-data 51.0.0", + "arrow-schema 51.0.0", + "arrow-select 51.0.0", + "half", + "num", +] + +[[package]] +name = "arrow-ord" +version = "52.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb56ed1547004e12203652f12fe12e824161ff9d1e5cf2a7dc4ff02ba94f413" +dependencies = [ + "arrow-array 52.0.0", + "arrow-buffer 52.0.0", + "arrow-data 52.0.0", + "arrow-schema 52.0.0", + "arrow-select 52.0.0", "half", "num", ] @@ -287,10 +452,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "848ee52bb92eb459b811fb471175ea3afcf620157674c8794f539838920f9228" dependencies = [ "ahash", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 51.0.0", + "arrow-buffer 51.0.0", + "arrow-data 51.0.0", + "arrow-schema 51.0.0", + "half", + "hashbrown", +] + +[[package]] +name = "arrow-row" +version = "52.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "575b42f1fc588f2da6977b94a5ca565459f5ab07b60545e17243fb9a7ed6d43e" +dependencies = [ + "ahash", + "arrow-array 52.0.0", + "arrow-buffer 52.0.0", + "arrow-data 52.0.0", + "arrow-schema 52.0.0", "half", "hashbrown", ] @@ -304,6 +484,12 @@ dependencies = [ "bitflags 2.5.0", ] +[[package]] +name = "arrow-schema" +version = "52.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32aae6a60458a2389c0da89c9de0b7932427776127da1a738e2efc21d32f3393" + [[package]] name = "arrow-select" version = "51.0.0" @@ -311,10 +497,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "849524fa70e0e3c5ab58394c770cb8f514d0122d20de08475f7b472ed8075830" dependencies = [ "ahash", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 51.0.0", + "arrow-buffer 51.0.0", + "arrow-data 51.0.0", + "arrow-schema 51.0.0", + "num", +] + +[[package]] +name = "arrow-select" +version = "52.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de36abaef8767b4220d7b4a8c2fe5ffc78b47db81b03d77e2136091c3ba39102" +dependencies = [ + "ahash", + "arrow-array 52.0.0", + "arrow-buffer 52.0.0", + "arrow-data 52.0.0", + "arrow-schema 52.0.0", "num", ] @@ -324,11 +524,28 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9373cb5a021aee58863498c37eb484998ef13377f69989c6c5ccfbd258236cdb" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 51.0.0", + "arrow-buffer 51.0.0", + "arrow-data 51.0.0", + "arrow-schema 51.0.0", + "arrow-select 51.0.0", + "memchr", + "num", + "regex", + "regex-syntax", +] + +[[package]] +name = "arrow-string" +version = "52.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e435ada8409bcafc910bc3e0077f532a4daa20e99060a496685c0e3e53cc2597" +dependencies = [ + "arrow-array 52.0.0", + "arrow-buffer 52.0.0", + "arrow-data 52.0.0", + "arrow-schema 52.0.0", + "arrow-select 52.0.0", "memchr", "num", "regex", @@ -748,8 +965,8 @@ dependencies = [ name = "bench-vortex" version = "0.1.0" dependencies = [ - "arrow-array", - "arrow-select", + "arrow-array 51.0.0", + "arrow-select 51.0.0", "bzip2", "criterion", "csv", @@ -1036,7 +1253,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e" dependencies = [ "chrono", - "chrono-tz-build", + "chrono-tz-build 0.2.1", + "phf", +] + +[[package]] +name = "chrono-tz" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" +dependencies = [ + "chrono", + "chrono-tz-build 0.3.0", "phf", ] @@ -1051,6 +1279,17 @@ dependencies = [ "phf_codegen", ] +[[package]] +name = "chrono-tz-build" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1" +dependencies = [ + "parse-zoneinfo", + "phf", + "phf_codegen", +] + [[package]] name = "ciborium" version = "0.2.2" @@ -1383,18 +1622,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85069782056753459dc47e386219aa1fdac5b731f26c28abb8c0ffd4b7c5ab11" dependencies = [ "ahash", - "arrow", - "arrow-array", - "arrow-ipc", - "arrow-schema", + "arrow 51.0.0", + "arrow-array 51.0.0", + "arrow-ipc 51.0.0", + "arrow-schema 51.0.0", "async-trait", "bytes", "chrono", "dashmap", - "datafusion-common", + "datafusion-common 37.1.0", "datafusion-common-runtime", "datafusion-execution", - "datafusion-expr", + "datafusion-expr 37.1.0", "datafusion-functions", "datafusion-functions-array", "datafusion-optimizer", @@ -1413,7 +1652,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "rand", - "sqlparser", + "sqlparser 0.44.0", "tempfile", "tokio", "url", @@ -1427,17 +1666,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "309d9040751f6dc9e33c85dce6abb55a46ef7ea3644577dd014611c379447ef3" dependencies = [ "ahash", - "arrow", - "arrow-array", - "arrow-buffer", - "arrow-schema", + "arrow 51.0.0", + "arrow-array 51.0.0", + "arrow-buffer 51.0.0", + "arrow-schema 51.0.0", "chrono", "half", "instant", "libc", "num_cpus", "object_store", - "sqlparser", + "sqlparser 0.44.0", +] + +[[package]] +name = "datafusion-common" +version = "39.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "effed030d2c1667eb1e11df5372d4981eaf5d11a521be32220b3985ae5ba6971" +dependencies = [ + "ahash", + "arrow 52.0.0", + "arrow-array 52.0.0", + "arrow-buffer 52.0.0", + "arrow-schema 52.0.0", + "chrono", + "half", + "hashbrown", + "instant", + "libc", + "num_cpus", + "sqlparser 0.47.0", ] [[package]] @@ -1455,11 +1714,11 @@ version = "37.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06a3a29ae36bcde07d179cc33b45656a8e7e4d023623e320e48dcf1200eeee95" dependencies = [ - "arrow", + "arrow 51.0.0", "chrono", "dashmap", - "datafusion-common", - "datafusion-expr", + "datafusion-common 37.1.0", + "datafusion-expr 37.1.0", "futures", "hashbrown", "log", @@ -1477,12 +1736,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a3542aa322029c2121a671ce08000d4b274171070df13f697b14169ccf4f628" dependencies = [ "ahash", - "arrow", - "arrow-array", + "arrow 51.0.0", + "arrow-array 51.0.0", + "chrono", + "datafusion-common 37.1.0", + "paste", + "sqlparser 0.44.0", + "strum", + "strum_macros", +] + +[[package]] +name = "datafusion-expr" +version = "39.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebb192f0055d2ce64e38ac100abc18e4e6ae9734d3c28eee522bbbd6a32108a3" +dependencies = [ + "ahash", + "arrow 52.0.0", + "arrow-array 52.0.0", + "arrow-buffer 52.0.0", "chrono", - "datafusion-common", + "datafusion-common 39.0.0", "paste", - "sqlparser", + "serde_json", + "sqlparser 0.47.0", "strum", "strum_macros", ] @@ -1493,14 +1771,14 @@ version = "37.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd221792c666eac174ecc09e606312844772acc12cbec61a420c2fca1ee70959" dependencies = [ - "arrow", + "arrow 51.0.0", "base64 0.22.1", "blake2", "blake3", "chrono", - "datafusion-common", + "datafusion-common 37.1.0", "datafusion-execution", - "datafusion-expr", + "datafusion-expr 37.1.0", "datafusion-physical-expr", "hex", "itertools 0.12.1", @@ -1518,14 +1796,14 @@ version = "37.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e501801e84d9c6ef54caaebcda1b18a6196a24176c12fb70e969bc0572e03c55" dependencies = [ - "arrow", - "arrow-array", - "arrow-buffer", - "arrow-ord", - "arrow-schema", - "datafusion-common", + "arrow 51.0.0", + "arrow-array 51.0.0", + "arrow-buffer 51.0.0", + "arrow-ord 51.0.0", + "arrow-schema 51.0.0", + "datafusion-common 37.1.0", "datafusion-execution", - "datafusion-expr", + "datafusion-expr 37.1.0", "datafusion-functions", "itertools 0.12.1", "log", @@ -1538,11 +1816,11 @@ version = "37.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76bd7f5087817deb961764e8c973d243b54f8572db414a8f0a8f33a48f991e0a" dependencies = [ - "arrow", + "arrow 51.0.0", "async-trait", "chrono", - "datafusion-common", - "datafusion-expr", + "datafusion-common 37.1.0", + "datafusion-expr 37.1.0", "datafusion-physical-expr", "hashbrown", "itertools 0.12.1", @@ -1557,19 +1835,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5cabc0d9aaa0f5eb1b472112f16223c9ffd2fb04e58cbf65c0a331ee6e993f96" dependencies = [ "ahash", - "arrow", - "arrow-array", - "arrow-buffer", - "arrow-ord", - "arrow-schema", - "arrow-string", + "arrow 51.0.0", + "arrow-array 51.0.0", + "arrow-buffer 51.0.0", + "arrow-ord 51.0.0", + "arrow-schema 51.0.0", + "arrow-string 51.0.0", "base64 0.22.1", "blake2", "blake3", "chrono", - "datafusion-common", + "datafusion-common 37.1.0", "datafusion-execution", - "datafusion-expr", + "datafusion-expr 37.1.0", "half", "hashbrown", "hex", @@ -1592,16 +1870,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17c0523e9c8880f2492a88bbd857dde02bed1ed23f3e9211a89d3d7ec3b44af9" dependencies = [ "ahash", - "arrow", - "arrow-array", - "arrow-buffer", - "arrow-schema", + "arrow 51.0.0", + "arrow-array 51.0.0", + "arrow-buffer 51.0.0", + "arrow-schema 51.0.0", "async-trait", "chrono", - "datafusion-common", + "datafusion-common 37.1.0", "datafusion-common-runtime", "datafusion-execution", - "datafusion-expr", + "datafusion-expr 37.1.0", "datafusion-physical-expr", "futures", "half", @@ -1622,13 +1900,13 @@ version = "37.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49eb54b42227136f6287573f2434b1de249fe1b8e6cd6cc73a634e4a3ec29356" dependencies = [ - "arrow", - "arrow-array", - "arrow-schema", - "datafusion-common", - "datafusion-expr", + "arrow 51.0.0", + "arrow-array 51.0.0", + "arrow-schema 51.0.0", + "datafusion-common 37.1.0", + "datafusion-expr 37.1.0", "log", - "sqlparser", + "sqlparser 0.44.0", "strum", ] @@ -1837,6 +2115,16 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "flatbuffers" +version = "24.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8add37afff2d4ffa83bc748a70b4b1370984f6980768554182424ef71447c35f" +dependencies = [ + "bitflags 1.3.2", + "rustc_version", +] + [[package]] name = "flatc" version = "0.2.2+23.5.26" @@ -2466,14 +2754,14 @@ version = "0.10.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "362b480df322cd9a1d0ae9336197001d29140843d3716a31a8f55ced9c000a54" dependencies = [ - "arrow", - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", + "arrow 51.0.0", + "arrow-arith 51.0.0", + "arrow-array 51.0.0", + "arrow-buffer 51.0.0", + "arrow-ord 51.0.0", + "arrow-row 51.0.0", + "arrow-schema 51.0.0", + "arrow-select 51.0.0", "async-recursion", "async-trait", "async_cell", @@ -2525,12 +2813,12 @@ version = "0.10.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a227e1c408cce3db3270a41e80b353f4abb90d0891de7bc53aa9ee7683f77d8f" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 51.0.0", + "arrow-buffer 51.0.0", + "arrow-cast 51.0.0", + "arrow-data 51.0.0", + "arrow-schema 51.0.0", + "arrow-select 51.0.0", "getrandom", "half", "num-traits", @@ -2544,14 +2832,14 @@ version = "0.10.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a88115f28618a2ca888fe41aeecca49bb90d4c61ba4c25f457c5a86f29fce5f9" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-schema", + "arrow-array 51.0.0", + "arrow-buffer 51.0.0", + "arrow-schema 51.0.0", "async-trait", "byteorder", "bytes", "chrono", - "datafusion-common", + "datafusion-common 37.1.0", "datafusion-sql", "futures", "lance-arrow", @@ -2580,13 +2868,13 @@ version = "0.10.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1d43fb160792ce395d76ed864b7e1c6ef3a9fe46e5fe54f5339409aaf89c037" dependencies = [ - "arrow", - "arrow-array", - "arrow-ord", - "arrow-schema", + "arrow 51.0.0", + "arrow-array 51.0.0", + "arrow-ord 51.0.0", + "arrow-schema 51.0.0", "async-trait", "datafusion", - "datafusion-common", + "datafusion-common 37.1.0", "datafusion-functions", "datafusion-physical-expr", "futures", @@ -2604,10 +2892,10 @@ version = "0.10.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7abd3dc7a5420ba82abb8dc3f826ff0d2b08c8e61ec20e5c9b087695ff583aa8" dependencies = [ - "arrow", - "arrow-array", - "arrow-cast", - "arrow-schema", + "arrow 51.0.0", + "arrow-array 51.0.0", + "arrow-cast 51.0.0", + "arrow-schema 51.0.0", "chrono", "futures", "hex", @@ -2621,12 +2909,12 @@ version = "0.10.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "694a0d2ce6617d83ccbccbe03b092343e03d0d3de389773143a6994575bb6c40" dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-schema", - "arrow-select", + "arrow-arith 51.0.0", + "arrow-array 51.0.0", + "arrow-buffer 51.0.0", + "arrow-cast 51.0.0", + "arrow-schema 51.0.0", + "arrow-select 51.0.0", "bytes", "futures", "lance-arrow", @@ -2648,17 +2936,17 @@ version = "0.10.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa0d5fa512333bf420475d34d631262ba4de6d604c93eaa7fb3d7fa2760c1758" dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-arith 51.0.0", + "arrow-array 51.0.0", + "arrow-buffer 51.0.0", + "arrow-data 51.0.0", + "arrow-schema 51.0.0", + "arrow-select 51.0.0", "async-recursion", "async-trait", "byteorder", "bytes", - "datafusion-common", + "datafusion-common 37.1.0", "futures", "lance-arrow", "lance-core", @@ -2684,16 +2972,16 @@ version = "0.10.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2f1892475738d35b46adb4f474934999545f46d23cdbb38181b45d8cdcbf471" dependencies = [ - "arrow", - "arrow-array", - "arrow-ord", - "arrow-schema", - "arrow-select", + "arrow 51.0.0", + "arrow-array 51.0.0", + "arrow-ord 51.0.0", + "arrow-schema 51.0.0", + "arrow-select 51.0.0", "async-recursion", "async-trait", "datafusion", - "datafusion-common", - "datafusion-expr", + "datafusion-common 37.1.0", + "datafusion-expr 37.1.0", "datafusion-physical-expr", "datafusion-sql", "futures", @@ -2730,14 +3018,14 @@ version = "0.10.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7143cf3e0175ed720be9515a188d7c37f748f2ffe32b08af65d45aed5a0af00" dependencies = [ - "arrow", - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow 51.0.0", + "arrow-arith 51.0.0", + "arrow-array 51.0.0", + "arrow-buffer 51.0.0", + "arrow-cast 51.0.0", + "arrow-data 51.0.0", + "arrow-schema 51.0.0", + "arrow-select 51.0.0", "async-recursion", "async-trait", "aws-config", @@ -2770,9 +3058,9 @@ version = "0.10.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6d737aabcaa7c34c1bd70282d667b45df7549b5cf80e44ed2a3481fbdebd260" dependencies = [ - "arrow-array", - "arrow-ord", - "arrow-schema", + "arrow-array 51.0.0", + "arrow-ord 51.0.0", + "arrow-schema 51.0.0", "cc", "futures", "half", @@ -2793,11 +3081,11 @@ version = "0.10.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9017c6d5aa45b233a835ac156a48621f61e552bebb8fdbb11952e96581c6aca" dependencies = [ - "arrow", - "arrow-array", - "arrow-buffer", - "arrow-ipc", - "arrow-schema", + "arrow 51.0.0", + "arrow-array 51.0.0", + "arrow-buffer 51.0.0", + "arrow-ipc 51.0.0", + "arrow-schema 51.0.0", "async-trait", "aws-credential-types", "byteorder", @@ -3465,13 +3753,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "096795d4f47f65fd3ee1ec5a98b77ab26d602f2cc785b0e4be5443add17ecc32" dependencies = [ "ahash", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-ipc", - "arrow-schema", - "arrow-select", + "arrow-array 51.0.0", + "arrow-buffer 51.0.0", + "arrow-cast 51.0.0", + "arrow-data 51.0.0", + "arrow-ipc 51.0.0", + "arrow-schema 51.0.0", + "arrow-select 51.0.0", "base64 0.22.1", "brotli", "bytes", @@ -3835,7 +4123,7 @@ dependencies = [ name = "pyvortex" version = "0.1.0" dependencies = [ - "arrow", + "arrow 51.0.0", "log", "paste", "pyo3", @@ -4538,6 +4826,16 @@ dependencies = [ "sqlparser_derive", ] +[[package]] +name = "sqlparser" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "295e9930cd7a97e58ca2a070541a3ca502b17f5d1fa7157376d0fabd85324f25" +dependencies = [ + "log", + "sqlparser_derive", +] + [[package]] name = "sqlparser_derive" version = "0.2.2" @@ -5104,13 +5402,13 @@ dependencies = [ name = "vortex-array" version = "0.1.0" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-schema", + "arrow-array 51.0.0", + "arrow-buffer 51.0.0", + "arrow-schema 51.0.0", "build-vortex", "criterion", "enum-iterator", - "flatbuffers", + "flatbuffers 23.5.26", "flexbuffers", "futures-util", "getrandom", @@ -5137,7 +5435,7 @@ dependencies = [ name = "vortex-buffer" version = "0.1.0" dependencies = [ - "arrow-buffer", + "arrow-buffer 51.0.0", "bytes", "flexbuffers", ] @@ -5177,7 +5475,7 @@ name = "vortex-dtype" version = "0.1.0" dependencies = [ "build-vortex", - "flatbuffers", + "flatbuffers 23.5.26", "half", "itertools 0.12.1", "num-traits", @@ -5193,8 +5491,8 @@ dependencies = [ name = "vortex-error" version = "0.1.0" dependencies = [ - "arrow-schema", - "flatbuffers", + "arrow-schema 51.0.0", + "flatbuffers 23.5.26", "flexbuffers", "parquet", "thiserror", @@ -5205,6 +5503,8 @@ dependencies = [ name = "vortex-expr" version = "0.1.0" dependencies = [ + "datafusion-common 39.0.0", + "datafusion-expr 39.0.0", "serde", "vortex-dtype", "vortex-error", @@ -5233,23 +5533,23 @@ dependencies = [ name = "vortex-flatbuffers" version = "0.1.0" dependencies = [ - "flatbuffers", + "flatbuffers 23.5.26", ] [[package]] name = "vortex-ipc" version = "0.1.0" dependencies = [ - "arrow", - "arrow-array", - "arrow-ipc", - "arrow-schema", - "arrow-select", + "arrow 51.0.0", + "arrow-array 51.0.0", + "arrow-ipc 51.0.0", + "arrow-schema 51.0.0", + "arrow-select 51.0.0", "build-vortex", "bytes", "criterion", "derive_builder", - "flatbuffers", + "flatbuffers 23.5.26", "futures-executor", "futures-util", "itertools 0.12.1", @@ -5285,7 +5585,7 @@ dependencies = [ name = "vortex-roaring" version = "0.1.0" dependencies = [ - "arrow-buffer", + "arrow-buffer 51.0.0", "croaring", "log", "num-traits", @@ -5302,7 +5602,8 @@ name = "vortex-scalar" version = "0.1.0" dependencies = [ "build-vortex", - "flatbuffers", + "datafusion-common 39.0.0", + "flatbuffers 23.5.26", "flexbuffers", "itertools 0.12.1", "num-traits", diff --git a/Cargo.toml b/Cargo.toml index 080a89a81a..30be15cffb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,8 @@ cargo_metadata = "0.18.1" criterion = { version = "0.5.1", features = ["html_reports"] } croaring = "1.0.1" csv = "1.3.0" +datafusion-common = "39.0.0" +datafusion-expr = "39.0.0" derive_builder = "0.20.0" divan = "0.1.14" duckdb = { version = "0.10.1", features = ["bundled"] } diff --git a/vortex-array/benches/compare.rs b/vortex-array/benches/compare.rs index 2a00411d9e..fe2135a2e7 100644 --- a/vortex-array/benches/compare.rs +++ b/vortex-array/benches/compare.rs @@ -27,9 +27,7 @@ fn filter_bool_indices(c: &mut Criterion) { group.bench_function("compare_bool", |b| { b.iter(|| { - let indices = - vortex::compute::compare::compare(&arr, &arr2, Operator::GreaterThanOrEqualTo) - .unwrap(); + let indices = vortex::compute::compare::compare(&arr, &arr2, Operator::Gte).unwrap(); black_box(indices); Ok::<(), VortexError>(()) }); @@ -53,9 +51,7 @@ fn filter_indices(c: &mut Criterion) { group.bench_function("compare_int", |b| { b.iter(|| { - let indices = - vortex::compute::compare::compare(&arr, &arr2, Operator::GreaterThanOrEqualTo) - .unwrap(); + let indices = vortex::compute::compare::compare(&arr, &arr2, Operator::Gte).unwrap(); black_box(indices); Ok::<(), VortexError>(()) }); diff --git a/vortex-array/src/array/bool/compute/compare.rs b/vortex-array/src/array/bool/compute/compare.rs index cc9969b1d4..fc26d3d8c7 100644 --- a/vortex-array/src/array/bool/compute/compare.rs +++ b/vortex-array/src/array/bool/compute/compare.rs @@ -13,13 +13,13 @@ impl CompareFn for BoolArray { let lhs = self.boolean_buffer(); let rhs = flattened.boolean_buffer(); let result_buf = match op { - Operator::EqualTo => lhs.bitxor(&rhs).not(), - Operator::NotEqualTo => lhs.bitxor(&rhs), + Operator::Eq => lhs.bitxor(&rhs).not(), + Operator::NotEq => lhs.bitxor(&rhs), - Operator::GreaterThan => lhs.bitand(&rhs.not()), - Operator::GreaterThanOrEqualTo => lhs.bitor(&rhs.not()), - Operator::LessThan => lhs.not().bitand(&rhs), - Operator::LessThanOrEqualTo => lhs.not().bitor(&rhs), + Operator::Gt => lhs.bitand(&rhs.not()), + Operator::Gte => lhs.bitor(&rhs.not()), + Operator::Lt => lhs.not().bitand(&rhs), + Operator::Lte => lhs.not().bitor(&rhs), }; Ok(BoolArray::from( self.validity() @@ -58,10 +58,10 @@ mod test { ) .into_array(); - let matches = compare(&arr, &arr, Operator::EqualTo)?.flatten_bool()?; + let matches = compare(&arr, &arr, Operator::Eq)?.flatten_bool()?; assert_eq!(to_int_indices(matches), [1u64, 2, 3, 4]); - let matches = compare(&arr, &arr, Operator::NotEqualTo)?.flatten_bool()?; + let matches = compare(&arr, &arr, Operator::NotEq)?.flatten_bool()?; let empty: [u64; 0] = []; assert_eq!(to_int_indices(matches), empty); @@ -71,16 +71,16 @@ mod test { ) .into_array(); - let matches = compare(&arr, &other, Operator::LessThanOrEqualTo)?.flatten_bool()?; + let matches = compare(&arr, &other, Operator::Lte)?.flatten_bool()?; assert_eq!(to_int_indices(matches), [2u64, 3, 4]); - let matches = compare(&arr, &other, Operator::LessThan)?.flatten_bool()?; + let matches = compare(&arr, &other, Operator::Lt)?.flatten_bool()?; assert_eq!(to_int_indices(matches), [4u64]); - let matches = compare(&other, &arr, Operator::GreaterThanOrEqualTo)?.flatten_bool()?; + let matches = compare(&other, &arr, Operator::Gte)?.flatten_bool()?; assert_eq!(to_int_indices(matches), [2u64, 3, 4]); - let matches = compare(&other, &arr, Operator::GreaterThan)?.flatten_bool()?; + let matches = compare(&other, &arr, Operator::Gt)?.flatten_bool()?; assert_eq!(to_int_indices(matches), [4u64]); Ok(()) } diff --git a/vortex-array/src/array/primitive/compute/compare.rs b/vortex-array/src/array/primitive/compute/compare.rs index 4d4ce18238..53b0a70740 100644 --- a/vortex-array/src/array/primitive/compute/compare.rs +++ b/vortex-array/src/array/primitive/compute/compare.rs @@ -78,10 +78,10 @@ mod test { ]) .into_array(); - let matches = compare(&arr, &arr, Operator::EqualTo)?.flatten_bool()?; + let matches = compare(&arr, &arr, Operator::Eq)?.flatten_bool()?; assert_eq!(to_int_indices(matches), [0u64, 1, 2, 3, 5, 6, 7, 8, 10]); - let matches = compare(&arr, &arr, Operator::NotEqualTo)?.flatten_bool()?; + let matches = compare(&arr, &arr, Operator::NotEq)?.flatten_bool()?; let empty: [u64; 0] = []; assert_eq!(to_int_indices(matches), empty); @@ -101,16 +101,16 @@ mod test { ]) .into_array(); - let matches = compare(&arr, &other, Operator::LessThanOrEqualTo)?.flatten_bool()?; + let matches = compare(&arr, &other, Operator::Lte)?.flatten_bool()?; assert_eq!(to_int_indices(matches), [0u64, 1, 2, 3, 5, 6, 7, 8, 10]); - let matches = compare(&arr, &other, Operator::LessThan)?.flatten_bool()?; + let matches = compare(&arr, &other, Operator::Lt)?.flatten_bool()?; assert_eq!(to_int_indices(matches), [5u64, 6, 7, 8, 10]); - let matches = compare(&other, &arr, Operator::GreaterThanOrEqualTo)?.flatten_bool()?; + let matches = compare(&other, &arr, Operator::Gte)?.flatten_bool()?; assert_eq!(to_int_indices(matches), [0u64, 1, 2, 3, 5, 6, 7, 8, 10]); - let matches = compare(&other, &arr, Operator::GreaterThan)?.flatten_bool()?; + let matches = compare(&other, &arr, Operator::Gt)?.flatten_bool()?; assert_eq!(to_int_indices(matches), [5u64, 6, 7, 8, 10]); Ok(()) } diff --git a/vortex-dtype/src/field_paths.rs b/vortex-dtype/src/field_paths.rs index 00f2052fe7..07368a0952 100644 --- a/vortex-dtype/src/field_paths.rs +++ b/vortex-dtype/src/field_paths.rs @@ -24,6 +24,10 @@ impl FieldPath { Some(Self::builder().join_all(new_field_names).build()) } } + + pub fn parts(&self) -> &[FieldIdentifier] { + &self.field_names + } } #[derive(Clone, Debug, PartialEq)] diff --git a/vortex-expr/Cargo.toml b/vortex-expr/Cargo.toml index 874b0cb563..b59f87119a 100644 --- a/vortex-expr/Cargo.toml +++ b/vortex-expr/Cargo.toml @@ -15,14 +15,16 @@ rust-version = { workspace = true } workspace = true [dependencies] +datafusion-common = { workspace = true, optional = true } +datafusion-expr = { workspace = true, optional = true } vortex-dtype = { path = "../vortex-dtype" } vortex-error = { path = "../vortex-error" } vortex-scalar = { path = "../vortex-scalar" } serde = { workspace = true, optional = true, features = ["derive"] } - [dev-dependencies] - [features] +default = [] +datafusion = ["dep:datafusion-common", "dep:datafusion-expr", "vortex-scalar/datafusion"] serde = ["dep:serde", "vortex-dtype/serde", "vortex-scalar/serde"] \ No newline at end of file diff --git a/vortex-expr/src/datafusion.rs b/vortex-expr/src/datafusion.rs new file mode 100644 index 0000000000..2d3776665a --- /dev/null +++ b/vortex-expr/src/datafusion.rs @@ -0,0 +1,63 @@ +#![cfg(feature = "datafusion")] +use datafusion_common::Column; +use datafusion_expr::{BinaryExpr, Expr}; +use vortex_dtype::field_paths::{FieldIdentifier, FieldPath}; +use vortex_scalar::Scalar; + +use crate::expressions::{Predicate, Value}; +use crate::operators::Operator; + +impl From for Expr { + fn from(value: Predicate) -> Self { + Expr::BinaryExpr(BinaryExpr::new( + Box::new(FieldPathWrapper(value.left).into()), + value.op.into(), + Box::new(value.right.into()), + )) + } +} + +impl From for datafusion_expr::Operator { + fn from(value: Operator) -> Self { + match value { + Operator::Eq => datafusion_expr::Operator::Eq, + Operator::NotEq => datafusion_expr::Operator::NotEq, + Operator::Gt => datafusion_expr::Operator::Gt, + Operator::Gte => datafusion_expr::Operator::GtEq, + Operator::Lt => datafusion_expr::Operator::Lt, + Operator::Lte => datafusion_expr::Operator::LtEq, + } + } +} + +impl From for Expr { + fn from(value: Value) -> Self { + match value { + Value::Field(field_path) => FieldPathWrapper(field_path).into(), + Value::Literal(literal) => ScalarWrapper(literal).into(), + } + } +} + +struct FieldPathWrapper(FieldPath); +impl From for Expr { + fn from(value: FieldPathWrapper) -> Self { + let mut field = String::new(); + for part in value.0.parts() { + match part { + // TODO(ngates): escape quotes? + FieldIdentifier::Name(identifier) => field.push_str(&format!("\"{}\"", identifier)), + FieldIdentifier::ListIndex(idx) => field.push_str(&format!("[{}]", idx)), + } + } + + Expr::Column(Column::from(field)) + } +} + +struct ScalarWrapper(Scalar); +impl From for Expr { + fn from(value: ScalarWrapper) -> Self { + Expr::Literal(value.0.into()) + } +} diff --git a/vortex-expr/src/display.rs b/vortex-expr/src/display.rs index cc37916207..a3d61323ea 100644 --- a/vortex-expr/src/display.rs +++ b/vortex-expr/src/display.rs @@ -42,12 +42,12 @@ impl Display for Value { impl Display for Operator { fn fmt(&self, f: &mut Formatter) -> fmt::Result { let display = match &self { - Operator::EqualTo => "=", - Operator::NotEqualTo => "!=", - Operator::GreaterThan => ">", - Operator::GreaterThanOrEqualTo => ">=", - Operator::LessThan => "<", - Operator::LessThanOrEqualTo => "<=", + Operator::Eq => "=", + Operator::NotEq => "!=", + Operator::Gt => ">", + Operator::Gte => ">=", + Operator::Lt => "<", + Operator::Lte => "<=", }; write!(f, "{display}") } diff --git a/vortex-expr/src/expressions.rs b/vortex-expr/src/expressions.rs index a03da6f5f2..c6eac21ceb 100644 --- a/vortex-expr/src/expressions.rs +++ b/vortex-expr/src/expressions.rs @@ -50,7 +50,7 @@ impl Value { pub fn eq(self, field: impl Into) -> Predicate { Predicate { left: field.into(), - op: Operator::EqualTo, + op: Operator::Eq, right: self, } } @@ -58,7 +58,7 @@ impl Value { pub fn not_eq(self, field: impl Into) -> Predicate { Predicate { left: field.into(), - op: Operator::NotEqualTo.inverse(), + op: Operator::NotEq.inverse(), right: self, } } @@ -66,7 +66,7 @@ impl Value { pub fn gt(self, field: impl Into) -> Predicate { Predicate { left: field.into(), - op: Operator::GreaterThan.inverse(), + op: Operator::Gt.inverse(), right: self, } } @@ -74,7 +74,7 @@ impl Value { pub fn gte(self, field: impl Into) -> Predicate { Predicate { left: field.into(), - op: Operator::GreaterThanOrEqualTo.inverse(), + op: Operator::Gte.inverse(), right: self, } } @@ -82,7 +82,7 @@ impl Value { pub fn lt(self, field: impl Into) -> Predicate { Predicate { left: field.into(), - op: Operator::LessThan.inverse(), + op: Operator::Lt.inverse(), right: self, } } @@ -90,7 +90,7 @@ impl Value { pub fn lte(self, field: impl Into) -> Predicate { Predicate { left: field.into(), - op: Operator::LessThanOrEqualTo.inverse(), + op: Operator::Lte.inverse(), right: self, } } @@ -109,7 +109,7 @@ mod test { let field = field("id"); let expr = Predicate { left: field, - op: Operator::EqualTo, + op: Operator::Eq, right: value, }; assert_eq!(format!("{}", expr), "($id = 1)"); diff --git a/vortex-expr/src/field_paths.rs b/vortex-expr/src/field_paths.rs index 208abb92d4..b1e0a9c7aa 100644 --- a/vortex-expr/src/field_paths.rs +++ b/vortex-expr/src/field_paths.rs @@ -17,7 +17,7 @@ impl FieldPathOperations for FieldPath { fn eq(self, other: Value) -> Predicate { Predicate { left: self, - op: Operator::EqualTo, + op: Operator::Eq, right: other, } } @@ -25,7 +25,7 @@ impl FieldPathOperations for FieldPath { fn not_eq(self, other: Value) -> Predicate { Predicate { left: self, - op: Operator::NotEqualTo, + op: Operator::NotEq, right: other, } } @@ -33,7 +33,7 @@ impl FieldPathOperations for FieldPath { fn gt(self, other: Value) -> Predicate { Predicate { left: self, - op: Operator::GreaterThan, + op: Operator::Gt, right: other, } } @@ -41,7 +41,7 @@ impl FieldPathOperations for FieldPath { fn gte(self, other: Value) -> Predicate { Predicate { left: self, - op: Operator::GreaterThanOrEqualTo, + op: Operator::Gte, right: other, } } @@ -49,7 +49,7 @@ impl FieldPathOperations for FieldPath { fn lt(self, other: Value) -> Predicate { Predicate { left: self, - op: Operator::LessThan, + op: Operator::Lt, right: other, } } @@ -57,7 +57,7 @@ impl FieldPathOperations for FieldPath { fn lte(self, other: Value) -> Predicate { Predicate { left: self, - op: Operator::LessThanOrEqualTo, + op: Operator::Lte, right: other, } } diff --git a/vortex-expr/src/lib.rs b/vortex-expr/src/lib.rs index 886afbb297..d512ed175b 100644 --- a/vortex-expr/src/lib.rs +++ b/vortex-expr/src/lib.rs @@ -1,6 +1,7 @@ #![feature(iter_intersperse)] extern crate core; +mod datafusion; mod display; pub mod expressions; pub mod field_paths; diff --git a/vortex-expr/src/operators.rs b/vortex-expr/src/operators.rs index b6c6c289af..12b9166c4c 100644 --- a/vortex-expr/src/operators.rs +++ b/vortex-expr/src/operators.rs @@ -8,12 +8,12 @@ use crate::expressions::Predicate; #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub enum Operator { // comparison - EqualTo, - NotEqualTo, - GreaterThan, - GreaterThanOrEqualTo, - LessThan, - LessThanOrEqualTo, + Eq, + NotEq, + Gt, + Gte, + Lt, + Lte, } impl ops::Not for Predicate { @@ -21,12 +21,12 @@ impl ops::Not for Predicate { fn not(self) -> Self::Output { let inverse_op = match self.op { - Operator::EqualTo => Operator::NotEqualTo, - Operator::NotEqualTo => Operator::EqualTo, - Operator::GreaterThan => Operator::LessThanOrEqualTo, - Operator::GreaterThanOrEqualTo => Operator::LessThan, - Operator::LessThan => Operator::GreaterThanOrEqualTo, - Operator::LessThanOrEqualTo => Operator::GreaterThan, + Operator::Eq => Operator::NotEq, + Operator::NotEq => Operator::Eq, + Operator::Gt => Operator::Lte, + Operator::Gte => Operator::Lt, + Operator::Lt => Operator::Gte, + Operator::Lte => Operator::Gt, }; Predicate { left: self.left, @@ -39,23 +39,23 @@ impl ops::Not for Predicate { impl Operator { pub fn inverse(self) -> Self { match self { - Operator::EqualTo => Operator::NotEqualTo, - Operator::NotEqualTo => Operator::EqualTo, - Operator::GreaterThan => Operator::LessThanOrEqualTo, - Operator::GreaterThanOrEqualTo => Operator::LessThan, - Operator::LessThan => Operator::GreaterThanOrEqualTo, - Operator::LessThanOrEqualTo => Operator::GreaterThan, + Operator::Eq => Operator::NotEq, + Operator::NotEq => Operator::Eq, + Operator::Gt => Operator::Lte, + Operator::Gte => Operator::Lt, + Operator::Lt => Operator::Gte, + Operator::Lte => Operator::Gt, } } pub fn to_predicate(&self) -> fn(&T, &T) -> bool { match self { - Operator::EqualTo => PartialEq::eq, - Operator::NotEqualTo => PartialEq::ne, - Operator::GreaterThan => PartialOrd::gt, - Operator::GreaterThanOrEqualTo => PartialOrd::ge, - Operator::LessThan => PartialOrd::lt, - Operator::LessThanOrEqualTo => PartialOrd::le, + Operator::Eq => PartialEq::eq, + Operator::NotEq => PartialEq::ne, + Operator::Gt => PartialOrd::gt, + Operator::Gte => PartialOrd::ge, + Operator::Lt => PartialOrd::lt, + Operator::Lte => PartialOrd::le, } } } diff --git a/vortex-scalar/Cargo.toml b/vortex-scalar/Cargo.toml index e5f8f06172..b9902dd0b6 100644 --- a/vortex-scalar/Cargo.toml +++ b/vortex-scalar/Cargo.toml @@ -12,6 +12,7 @@ edition = { workspace = true } rust-version = { workspace = true } [dependencies] +datafusion-common = { workspace = true, optional = true } flatbuffers = { workspace = true, optional = true } flexbuffers = { workspace = true, optional = true } itertools = { workspace = true } @@ -34,6 +35,7 @@ workspace = true [features] # Uncomment for improved IntelliJ support # default = ["flatbuffers", "proto", "serde"] +datafusion = ["dep:datafusion-common"] flatbuffers = [ "dep:flatbuffers", "dep:flexbuffers", diff --git a/vortex-scalar/src/datafusion.rs b/vortex-scalar/src/datafusion.rs new file mode 100644 index 0000000000..594a6c3c3d --- /dev/null +++ b/vortex-scalar/src/datafusion.rs @@ -0,0 +1,68 @@ +#![cfg(feature = "datafusion")] +use datafusion_common::ScalarValue; +use vortex_dtype::{DType, PType}; + +use crate::{PValue, Scalar}; + +impl From for ScalarValue { + fn from(value: Scalar) -> Self { + match value.dtype { + DType::Null => ScalarValue::Null, + DType::Bool(_) => ScalarValue::Boolean(value.value.as_bool().expect("should be bool")), + DType::Primitive(ptype, _) => { + let pvalue = value.value.as_pvalue().expect("should be pvalue"); + match pvalue { + None => match ptype { + PType::U8 => ScalarValue::UInt8(None), + PType::U16 => ScalarValue::UInt16(None), + PType::U32 => ScalarValue::UInt32(None), + PType::U64 => ScalarValue::UInt64(None), + PType::I8 => ScalarValue::Int8(None), + PType::I16 => ScalarValue::Int16(None), + PType::I32 => ScalarValue::Int32(None), + PType::I64 => ScalarValue::Int64(None), + PType::F16 => ScalarValue::Float16(None), + PType::F32 => ScalarValue::Float32(None), + PType::F64 => ScalarValue::Float64(None), + }, + Some(pvalue) => match pvalue { + PValue::U8(v) => ScalarValue::UInt8(Some(v)), + PValue::U16(v) => ScalarValue::UInt16(Some(v)), + PValue::U32(v) => ScalarValue::UInt32(Some(v)), + PValue::U64(v) => ScalarValue::UInt64(Some(v)), + PValue::I8(v) => ScalarValue::Int8(Some(v)), + PValue::I16(v) => ScalarValue::Int16(Some(v)), + PValue::I32(v) => ScalarValue::Int32(Some(v)), + PValue::I64(v) => ScalarValue::Int64(Some(v)), + PValue::F16(v) => ScalarValue::Float16(Some(v)), + PValue::F32(v) => ScalarValue::Float32(Some(v)), + PValue::F64(v) => ScalarValue::Float64(Some(v)), + }, + } + } + DType::Utf8(_) => ScalarValue::Utf8( + value + .value + .as_buffer_string() + .expect("should be buffer string") + .map(|b| b.as_str().to_string()), + ), + DType::Binary(_) => ScalarValue::Binary( + value + .value + .as_buffer() + .expect("should be buffer") + .map(|b| b.as_slice().to_vec()), + ), + DType::Struct(..) => { + todo!("struct scalar conversion") + } + DType::List(..) => { + todo!("list scalar conversion") + } + DType::Extension(..) => { + todo!("extension scalar conversion") + } + } + } +} diff --git a/vortex-scalar/src/lib.rs b/vortex-scalar/src/lib.rs index 77b5446a47..e628c098de 100644 --- a/vortex-scalar/src/lib.rs +++ b/vortex-scalar/src/lib.rs @@ -4,6 +4,7 @@ use vortex_dtype::DType; mod binary; mod bool; +mod datafusion; mod display; mod extension; mod list;