diff --git a/Cargo.lock b/Cargo.lock index c6ceab6408..f481cfd052 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,9 +19,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.9" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d713b3834d76b85304d4d525563c1276e2e30dc97cc67bfb4585a4a29fc2c89f" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", "const-random", @@ -90,15 +90,15 @@ checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" [[package]] name = "arc-swap" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" +checksum = "7b3d0060af21e8d11a926981cc00c6c1541aa91dd64b9f881985c3da1094425f" [[package]] name = "argminmax" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202108b46429b765ef483f8a24d5c46f48c14acfdacc086dd4ab6dddf6bcdbd2" +checksum = "52424b59d69d69d5056d508b260553afd91c57e21849579cd1f50ee8b8b88eaa" dependencies = [ "num-traits", ] @@ -377,7 +377,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.51", + "syn 2.0.52", "which", ] @@ -437,7 +437,7 @@ checksum = "965ab7eb5f8f97d2a083c799f3a1b994fc397b2fe2da5d1da1626ce15a39f2b1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.51", + "syn 2.0.52", ] [[package]] @@ -460,10 +460,11 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.88" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02f341c093d19155a6e41631ce5971aac4e9a868262212153124c15fa22d1cdc" +checksum = "a0ba8f7aaa012f30d5b2861462f6708eccd49c3c39863fe083a308035f63d723" dependencies = [ + "jobserver", "libc", ] @@ -491,7 +492,7 @@ dependencies = [ "android-tzdata", "iana-time-zone", "num-traits", - "windows-targets 0.52.3", + "windows-targets 0.52.4", ] [[package]] @@ -549,6 +550,7 @@ checksum = "9f3e7391dad68afb0c2ede1bf619f579a3dc9c2ec67f089baa397123a2f3d1eb" dependencies = [ "anstyle", "clap_lex", + "terminal_size", ] [[package]] @@ -586,11 +588,17 @@ dependencies = [ "walkdir", ] +[[package]] +name = "condtype" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf0a07a401f374238ab8e2f11a104d2851bf9ce711ec69804834de8af45c7af" + [[package]] name = "const-random" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaf16c9c2c612020bcfd042e170f6e32de9b9d75adb5277cdbbd2e2c8c8299a" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" dependencies = [ "const-random-macro", ] @@ -756,6 +764,31 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "divan" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0d567df2c9c2870a43f3f2bd65aaeb18dbce1c18f217c3e564b4fbaeb3ee56c" +dependencies = [ + "cfg-if", + "clap", + "condtype", + "divan-macros", + "libc", + "regex-lite", +] + +[[package]] +name = "divan-macros" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27540baf49be0d484d8f0130d7d8da3011c32a44d4fc873368154f1510e574a2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] + [[package]] name = "dyn-clone" version = "1.0.17" @@ -1021,9 +1054,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "379dada1584ad501b383485dd706b8afb7a70fcbc7f4da7d780638a5a6124a60" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "home" @@ -1036,9 +1069,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ "bytes", "fnv", @@ -1149,9 +1182,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.3" +version = "2.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177" +checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" dependencies = [ "equivalent", "hashbrown", @@ -1210,11 +1243,20 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +[[package]] +name = "jobserver" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" -version = "0.3.68" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" dependencies = [ "wasm-bindgen", ] @@ -1309,12 +1351,12 @@ checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libloading" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c571b676ddfc9a8c12f1f3d3085a7b163966a8fd8098a90640953ce5f6170161" +checksum = "2caa5afb8bf9f3a2652760ce7d4f62d21c4d5a423e68466fca30df82f2330164" dependencies = [ "cfg-if", - "windows-sys 0.48.0", + "windows-targets 0.52.4", ] [[package]] @@ -1340,7 +1382,7 @@ checksum = "ee9023a564f8bf7fe3da285a50c3e70de0df3e2bf277ff7c4e76d66008ef93b0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.51", + "syn 2.0.52", ] [[package]] @@ -1361,9 +1403,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.20" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "lz4_flex" @@ -1412,9 +1454,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "wasi", @@ -1581,7 +1623,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.51", + "syn 2.0.52", ] [[package]] @@ -1637,7 +1679,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.51", + "syn 2.0.52", ] [[package]] @@ -1948,7 +1990,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" dependencies = [ "proc-macro2", - "syn 2.0.51", + "syn 2.0.52", ] [[package]] @@ -2027,7 +2069,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.51", + "syn 2.0.52", ] [[package]] @@ -2040,7 +2082,7 @@ dependencies = [ "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.51", + "syn 2.0.52", ] [[package]] @@ -2104,9 +2146,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7237101a77a10773db45d62004a272517633fbcc3df19d96455ede1122e051" +checksum = "e4963ed1bc86e4f3ee217022bd855b297cef07fb9eac5dfa1f788b220b49b3bd" dependencies = [ "either", "rayon-core", @@ -2145,15 +2187,21 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" dependencies = [ "aho-corasick", "memchr", "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b661b2f27137bdbc16f00eda72866a92bb28af1753ffbd56744fb6e2e9cd8e" + [[package]] name = "regex-syntax" version = "0.8.2" @@ -2341,7 +2389,7 @@ checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.51", + "syn 2.0.52", ] [[package]] @@ -2381,9 +2429,9 @@ checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" [[package]] name = "simplelog" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acee08041c5de3d5048c8b3f6f13fafb3026b24ba43c6a695a0c76179b844369" +checksum = "16257adbfaef1ee58b1363bdc0664c9b8e1e30aed86049635fb5f147d065a9c0" dependencies = [ "log", "paris", @@ -2464,9 +2512,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.51" +version = "2.0.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ab617d94515e94ae53b8406c628598680aa0c9587474ecbe58188f7b345d66c" +checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07" dependencies = [ "proc-macro2", "quote", @@ -2526,13 +2574,23 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.1.3" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ "winapi-util", ] +[[package]] +name = "terminal_size" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7" +dependencies = [ + "rustix", + "windows-sys 0.48.0", +] + [[package]] name = "thiserror" version = "1.0.57" @@ -2550,7 +2608,7 @@ checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" dependencies = [ "proc-macro2", "quote", - "syn 2.0.51", + "syn 2.0.52", ] [[package]] @@ -2802,11 +2860,11 @@ dependencies = [ name = "vortex-alp" version = "0.1.0" dependencies = [ - "arrow", - "codecz", + "divan", "itertools 0.12.1", "linkme", "log", + "num-traits", "vortex-array", ] @@ -2931,9 +2989,9 @@ dependencies = [ [[package]] name = "walkdir" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", "winapi-util", @@ -2956,9 +3014,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2966,24 +3024,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.51", + "syn 2.0.52", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.41" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877b9c3f61ceea0e56331985743b13f3d25c406a7098d45180fb5f09bc19ed97" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" dependencies = [ "cfg-if", "js-sys", @@ -2993,9 +3051,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3003,28 +3061,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.51", + "syn 2.0.52", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "web-sys" -version = "0.3.68" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96565907687f7aceb35bc5fc03770a8a0471d82e479f25832f54a0e3f4b28446" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" dependencies = [ "js-sys", "wasm-bindgen", @@ -3079,7 +3137,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.3", + "windows-targets 0.52.4", ] [[package]] @@ -3097,7 +3155,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.3", + "windows-targets 0.52.4", ] [[package]] @@ -3117,17 +3175,17 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.3" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d380ba1dc7187569a8a9e91ed34b8ccfc33123bbacb8c0aed2d1ad7f3ef2dc5f" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" dependencies = [ - "windows_aarch64_gnullvm 0.52.3", - "windows_aarch64_msvc 0.52.3", - "windows_i686_gnu 0.52.3", - "windows_i686_msvc 0.52.3", - "windows_x86_64_gnu 0.52.3", - "windows_x86_64_gnullvm 0.52.3", - "windows_x86_64_msvc 0.52.3", + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", ] [[package]] @@ -3138,9 +3196,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.3" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68e5dcfb9413f53afd9c8f86e56a7b4d86d9a2fa26090ea2dc9e40fba56c6ec6" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" [[package]] name = "windows_aarch64_msvc" @@ -3150,9 +3208,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.3" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dab469ebbc45798319e69eebf92308e541ce46760b49b18c6b3fe5e8965b30f" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" [[package]] name = "windows_i686_gnu" @@ -3162,9 +3220,9 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.3" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a4e9b6a7cac734a8b4138a4e1044eac3404d8326b6c0f939276560687a033fb" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" [[package]] name = "windows_i686_msvc" @@ -3174,9 +3232,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.3" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b0ec9c422ca95ff34a78755cfa6ad4a51371da2a5ace67500cf7ca5f232c58" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" [[package]] name = "windows_x86_64_gnu" @@ -3186,9 +3244,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.3" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "704131571ba93e89d7cd43482277d6632589b18ecf4468f591fbae0a8b101614" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" [[package]] name = "windows_x86_64_gnullvm" @@ -3198,9 +3256,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.3" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42079295511643151e98d61c38c0acc444e52dd42ab456f7ccfd5152e8ecf21c" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" [[package]] name = "windows_x86_64_msvc" @@ -3210,9 +3268,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.3" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0770833d60a970638e989b3fa9fd2bb1aaadcf88963d1659fd7d9990196ed2d6" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" [[package]] name = "winnow" @@ -3256,7 +3314,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.51", + "syn 2.0.52", ] [[package]] diff --git a/vortex-alp/Cargo.toml b/vortex-alp/Cargo.toml index 9ec65e98fc..64911557b3 100644 --- a/vortex-alp/Cargo.toml +++ b/vortex-alp/Cargo.toml @@ -11,13 +11,19 @@ include = { workspace = true } edition = { workspace = true } rust-version = { workspace = true } +[lints] +workspace = true + [dependencies] -arrow = { version = "50.0.0" } vortex-array = { path = "../vortex-array" } linkme = "0.3.22" itertools = "0.12.1" -codecz = { path = "../codecz" } +num-traits = "0.2.18" log = { version = "0.4.20", features = [] } -[lints] -workspace = true +[dev-dependencies] +divan = "0.1.14" + +[[bench]] +name = "alp_compress" +harness = false \ No newline at end of file diff --git a/vortex-alp/benches/alp_compress.rs b/vortex-alp/benches/alp_compress.rs new file mode 100644 index 0000000000..d0cb986d91 --- /dev/null +++ b/vortex-alp/benches/alp_compress.rs @@ -0,0 +1,20 @@ +use vortex::array::primitive::PrimitiveArray; +use vortex::array::ArrayRef; +use vortex_alp::{ALPArray, ALPFloat, Exponents}; + +fn main() { + divan::main(); +} + +#[divan::bench(types = [f32, f64], args = [100_000, 10_000_000])] +fn alp_compress(n: usize) -> (Exponents, Vec, Vec, Vec) { + let values: Vec = vec![T::from(1.234).unwrap(); n]; + T::encode(values.as_slice(), None) +} + +// TODO(ngates): remove this +#[divan::bench(args = [100_000, 10_000_000])] +fn alp_compress_array(n: usize) -> ArrayRef { + let array = PrimitiveArray::from_vec(vec![1.234f64; n]); + ALPArray::encode(&array).unwrap() +} diff --git a/vortex-alp/src/alp.rs b/vortex-alp/src/alp.rs index 65d37418aa..75e410fb71 100644 --- a/vortex-alp/src/alp.rs +++ b/vortex-alp/src/alp.rs @@ -1,171 +1,199 @@ -use std::any::Any; -use std::sync::{Arc, RwLock}; - -pub use codecz::alp::ALPExponents; -use vortex::array::{Array, ArrayKind, ArrayRef, ArrowIterator, Encoding, EncodingId, EncodingRef}; -use vortex::compress::EncodingCompression; -use vortex::dtype::{DType, IntWidth, Signedness}; -use vortex::error::{VortexError, VortexResult}; -use vortex::formatter::{ArrayDisplay, ArrayFormatter}; -use vortex::serde::{ArraySerde, EncodingSerde}; -use vortex::stats::{Stats, StatsSet}; - -use crate::compress::alp_encode; - -#[derive(Debug, Clone)] -pub struct ALPArray { - encoded: ArrayRef, - exponents: ALPExponents, - patches: Option, - dtype: DType, - stats: Arc>, -} - -impl ALPArray { - pub fn new(encoded: ArrayRef, exponents: ALPExponents, patches: Option) -> Self { - Self::try_new(encoded, exponents, patches).unwrap() - } - - pub fn try_new( - encoded: ArrayRef, - exponents: ALPExponents, - patches: Option, - ) -> VortexResult { - let dtype = match encoded.dtype() { - d @ DType::Int(width, Signedness::Signed, nullability) => match width { - IntWidth::_32 => DType::Float(32.into(), *nullability), - IntWidth::_64 => DType::Float(64.into(), *nullability), - _ => return Err(VortexError::InvalidDType(d.clone())), - }, - d => return Err(VortexError::InvalidDType(d.clone())), - }; - Ok(Self { - encoded, - exponents, - patches, - dtype, - stats: Arc::new(RwLock::new(StatsSet::new())), - }) - } - - pub fn encode(array: &dyn Array) -> VortexResult { - match ArrayKind::from(array) { - ArrayKind::Primitive(p) => Ok(alp_encode(p).boxed()), - _ => Err(VortexError::InvalidEncoding(array.encoding().id().clone())), - } - } - - pub fn encoded(&self) -> &dyn Array { - self.encoded.as_ref() - } +use itertools::Itertools; +use num_traits::{Float, NumCast, PrimInt, Zero}; +use std::mem::size_of; - pub fn exponents(&self) -> ALPExponents { - self.exponents - } +const SAMPLE_SIZE: usize = 32; - pub fn patches(&self) -> Option<&dyn Array> { - self.patches.as_deref() - } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Exponents { + pub e: u8, + pub f: u8, } -impl Array for ALPArray { - #[inline] - fn as_any(&self) -> &dyn Any { - self - } - - #[inline] - fn boxed(self) -> ArrayRef { - Box::new(self) - } - - #[inline] - fn into_any(self: Box) -> Box { - self - } - - #[inline] - fn len(&self) -> usize { - self.encoded.len() - } - - #[inline] - fn is_empty(&self) -> bool { - self.encoded.is_empty() - } +pub trait ALPFloat: Float + 'static { + type ALPInt: PrimInt; + + const FRACTIONAL_BITS: u8; + const MAX_EXPONENT: u8; + const SWEET: Self; + const F10: &'static [Self]; + const IF10: &'static [Self]; + + /// Round to the nearest floating integer by shifting in and out of the low precision range. + fn fast_round(self) -> Self { + (self + Self::SWEET) - Self::SWEET + } + + fn as_int(self) -> Option { + ::from(self) + } + + fn find_best_exponents(values: &[Self]) -> Exponents { + let mut best_exp = Exponents { e: 0, f: 0 }; + let mut best_nbytes: usize = usize::MAX; + + let sample = (values.len() > SAMPLE_SIZE).then(|| { + values + .iter() + .step_by(values.len() / SAMPLE_SIZE) + .cloned() + .collect_vec() + }); + + // TODO(wmanning): idea, start with highest e, then find the best f + // after that, try e's in descending order, with a gap no larger than the original e - f + for e in 0..Self::MAX_EXPONENT { + for f in 0..e { + let (_, encoded, exc_pos, exc_patches) = Self::encode( + sample.as_deref().unwrap_or(values), + Some(&Exponents { e, f }), + ); + let size = + (encoded.len() + exc_patches.len()) * size_of::() + (exc_pos.len() * 4); + if size < best_nbytes { + best_nbytes = size; + best_exp = Exponents { e, f }; + } else if size == best_nbytes && e - f < best_exp.e - best_exp.f { + best_exp = Exponents { e, f }; + } + } + } - #[inline] - fn dtype(&self) -> &DType { - &self.dtype + best_exp } - #[inline] - fn stats(&self) -> Stats { - Stats::new(&self.stats, self) - } + fn encode( + values: &[Self], + exponents: Option<&Exponents>, + ) -> (Exponents, Vec, Vec, Vec) { + let exp = exponents.map_or_else(|| Self::find_best_exponents(values), Exponents::clone); - fn iter_arrow(&self) -> Box { - todo!() - } + let mut exc_pos = Vec::new(); + let mut exc_value = Vec::new(); + let mut prev = Self::ALPInt::zero(); + let encoded = values + .iter() + .enumerate() + .map(|(i, v)| { + let encoded = + (*v * Self::F10[exp.e as usize] * Self::IF10[exp.f as usize]).fast_round(); + let decoded = encoded * Self::F10[exp.f as usize] * Self::IF10[exp.e as usize]; - fn slice(&self, start: usize, stop: usize) -> VortexResult { - Ok(Self::try_new( - self.encoded().slice(start, stop)?, - self.exponents(), - self.patches().map(|p| p.slice(start, stop)).transpose()?, - )? - .boxed()) - } + if decoded == *v { + if let Some(e) = encoded.as_int() { + prev = e; + return e; + } + } - #[inline] - fn encoding(&self) -> EncodingRef { - &ALPEncoding - } + exc_pos.push(i as u64); + exc_value.push(*v); + // Emit the last known good value. This helps with run-end encoding. + prev + }) + .collect_vec(); - #[inline] - fn nbytes(&self) -> usize { - self.encoded().nbytes() + self.patches().map(|p| p.nbytes()).unwrap_or(0) + (exp, encoded, exc_pos, exc_value) } - fn serde(&self) -> &dyn ArraySerde { - self + fn decode_single(encoded: Self::ALPInt, exponents: &Exponents) -> Self { + let encoded_float: Self = Self::from(encoded).unwrap(); + encoded_float * Self::F10[exponents.f as usize] * Self::IF10[exponents.e as usize] } } -impl<'arr> AsRef<(dyn Array + 'arr)> for ALPArray { - fn as_ref(&self) -> &(dyn Array + 'arr) { - self - } +impl ALPFloat for f32 { + type ALPInt = i32; + const FRACTIONAL_BITS: u8 = 23; + const MAX_EXPONENT: u8 = 10; + const SWEET: Self = + (1 << Self::FRACTIONAL_BITS) as Self + (1 << (Self::FRACTIONAL_BITS - 1)) as Self; + + const F10: &'static [Self] = &[ + 1.0, + 10.0, + 100.0, + 1000.0, + 10000.0, + 100000.0, + 1000000.0, + 10000000.0, + 100000000.0, + 1000000000.0, + 10000000000.0, + ]; + const IF10: &'static [Self] = &[ + 1.0, + 0.1, + 0.01, + 0.001, + 0.0001, + 0.00001, + 0.000001, + 0.0000001, + 0.00000001, + 0.000000001, + 0.0000000001, + ]; } -impl ArrayDisplay for ALPArray { - fn fmt(&self, f: &mut ArrayFormatter) -> std::fmt::Result { - f.writeln(format!("exponents: {}", self.exponents()))?; - if let Some(p) = self.patches() { - f.writeln("patches:")?; - f.indent(|indent| indent.array(p.as_ref()))?; - } - f.indent(|indent| indent.array(self.encoded())) - } -} - -#[derive(Debug)] -pub struct ALPEncoding; - -impl ALPEncoding { - pub const ID: EncodingId = EncodingId::new("vortex.alp"); -} - -impl Encoding for ALPEncoding { - fn id(&self) -> &EncodingId { - &Self::ID - } - - fn compression(&self) -> Option<&dyn EncodingCompression> { - Some(self) - } - - fn serde(&self) -> Option<&dyn EncodingSerde> { - Some(self) - } +impl ALPFloat for f64 { + type ALPInt = i64; + const FRACTIONAL_BITS: u8 = 52; + const MAX_EXPONENT: u8 = 18; // 10^18 is the maximum i64 + const SWEET: Self = + (1u64 << Self::FRACTIONAL_BITS) as Self + (1u64 << (Self::FRACTIONAL_BITS - 1)) as Self; + const F10: &'static [Self] = &[ + 1.0, + 10.0, + 100.0, + 1000.0, + 10000.0, + 100000.0, + 1000000.0, + 10000000.0, + 100000000.0, + 1000000000.0, + 10000000000.0, + 100000000000.0, + 1000000000000.0, + 10000000000000.0, + 100000000000000.0, + 1000000000000000.0, + 10000000000000000.0, + 100000000000000000.0, + 1000000000000000000.0, + 10000000000000000000.0, + 100000000000000000000.0, + 1000000000000000000000.0, + 10000000000000000000000.0, + 100000000000000000000000.0, + ]; + + const IF10: &'static [Self] = &[ + 1.0, + 0.1, + 0.01, + 0.001, + 0.0001, + 0.00001, + 0.000001, + 0.0000001, + 0.00000001, + 0.000000001, + 0.0000000001, + 0.00000000001, + 0.000000000001, + 0.0000000000001, + 0.00000000000001, + 0.000000000000001, + 0.0000000000000001, + 0.00000000000000001, + 0.000000000000000001, + 0.0000000000000000001, + 0.00000000000000000001, + 0.000000000000000000001, + 0.0000000000000000000001, + 0.00000000000000000000001, + ]; } diff --git a/vortex-alp/src/array.rs b/vortex-alp/src/array.rs new file mode 100644 index 0000000000..978af89680 --- /dev/null +++ b/vortex-alp/src/array.rs @@ -0,0 +1,171 @@ +use std::any::Any; +use std::sync::{Arc, RwLock}; + +use crate::alp::Exponents; +use vortex::array::{Array, ArrayKind, ArrayRef, ArrowIterator, Encoding, EncodingId, EncodingRef}; +use vortex::compress::EncodingCompression; +use vortex::dtype::{DType, IntWidth, Signedness}; +use vortex::error::{VortexError, VortexResult}; +use vortex::formatter::{ArrayDisplay, ArrayFormatter}; +use vortex::serde::{ArraySerde, EncodingSerde}; +use vortex::stats::{Stats, StatsSet}; + +use crate::compress::alp_encode; + +#[derive(Debug, Clone)] +pub struct ALPArray { + encoded: ArrayRef, + exponents: Exponents, + patches: Option, + dtype: DType, + stats: Arc>, +} + +impl ALPArray { + pub fn new(encoded: ArrayRef, exponents: Exponents, patches: Option) -> Self { + Self::try_new(encoded, exponents, patches).unwrap() + } + + pub fn try_new( + encoded: ArrayRef, + exponents: Exponents, + patches: Option, + ) -> VortexResult { + let dtype = match encoded.dtype() { + d @ DType::Int(width, Signedness::Signed, nullability) => match width { + IntWidth::_32 => DType::Float(32.into(), *nullability), + IntWidth::_64 => DType::Float(64.into(), *nullability), + _ => return Err(VortexError::InvalidDType(d.clone())), + }, + d => return Err(VortexError::InvalidDType(d.clone())), + }; + Ok(Self { + encoded, + exponents, + patches, + dtype, + stats: Arc::new(RwLock::new(StatsSet::new())), + }) + } + + pub fn encode(array: &dyn Array) -> VortexResult { + match ArrayKind::from(array) { + ArrayKind::Primitive(p) => Ok(alp_encode(p)?.boxed()), + _ => Err(VortexError::InvalidEncoding(array.encoding().id().clone())), + } + } + + pub fn encoded(&self) -> &dyn Array { + self.encoded.as_ref() + } + + pub fn exponents(&self) -> &Exponents { + &self.exponents + } + + pub fn patches(&self) -> Option<&dyn Array> { + self.patches.as_deref() + } +} + +impl Array for ALPArray { + #[inline] + fn as_any(&self) -> &dyn Any { + self + } + + #[inline] + fn boxed(self) -> ArrayRef { + Box::new(self) + } + + #[inline] + fn into_any(self: Box) -> Box { + self + } + + #[inline] + fn len(&self) -> usize { + self.encoded.len() + } + + #[inline] + fn is_empty(&self) -> bool { + self.encoded.is_empty() + } + + #[inline] + fn dtype(&self) -> &DType { + &self.dtype + } + + #[inline] + fn stats(&self) -> Stats { + Stats::new(&self.stats, self) + } + + fn iter_arrow(&self) -> Box { + todo!() + } + + fn slice(&self, start: usize, stop: usize) -> VortexResult { + Ok(Self::try_new( + self.encoded().slice(start, stop)?, + self.exponents().clone(), + self.patches().map(|p| p.slice(start, stop)).transpose()?, + )? + .boxed()) + } + + #[inline] + fn encoding(&self) -> EncodingRef { + &ALPEncoding + } + + #[inline] + fn nbytes(&self) -> usize { + self.encoded().nbytes() + self.patches().map(|p| p.nbytes()).unwrap_or(0) + } + + fn serde(&self) -> &dyn ArraySerde { + self + } +} + +impl<'arr> AsRef<(dyn Array + 'arr)> for ALPArray { + fn as_ref(&self) -> &(dyn Array + 'arr) { + self + } +} + +impl ArrayDisplay for ALPArray { + fn fmt(&self, f: &mut ArrayFormatter) -> std::fmt::Result { + f.writeln(format!("exponents: {:?}", self.exponents()))?; + if let Some(p) = self.patches() { + f.writeln("patches:")?; + f.indent(|indent| indent.array(p.as_ref()))?; + } + f.indent(|indent| indent.array(self.encoded())) + } +} + +#[derive(Debug)] +pub struct ALPEncoding; + +impl ALPEncoding { + pub const ID: EncodingId = EncodingId::new("vortex.alp"); +} + +impl Encoding for ALPEncoding { + fn id(&self) -> &EncodingId { + &Self::ID + } + + fn compression(&self) -> Option<&dyn EncodingCompression> { + Some(self) + } + + fn serde(&self) -> Option<&dyn EncodingSerde> { + Some(self) + } +} diff --git a/vortex-alp/src/compress.rs b/vortex-alp/src/compress.rs index a742b0924a..83018a53c6 100644 --- a/vortex-alp/src/compress.rs +++ b/vortex-alp/src/compress.rs @@ -1,16 +1,17 @@ use log::debug; -use codecz::alp; -use codecz::alp::{ALPEncoded, ALPExponents, SupportsALP}; +use crate::alp::ALPFloat; use vortex::array::downcast::DowncastArrayBuiltin; use vortex::array::primitive::PrimitiveArray; use vortex::array::sparse::SparseArray; -use vortex::array::{Array, ArrayRef, CloneOptionalArray}; +use vortex::array::{Array, ArrayRef}; use vortex::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression}; +use vortex::error::{VortexError, VortexResult}; use vortex::ptype::{NativePType, PType}; -use crate::alp::{ALPArray, ALPEncoding}; +use crate::array::{ALPArray, ALPEncoding}; use crate::downcast::DowncastALP; +use crate::Exponents; impl EncodingCompression for ALPEncoding { fn compressor( @@ -37,109 +38,87 @@ impl EncodingCompression for ALPEncoding { fn alp_compressor(array: &dyn Array, like: Option<&dyn Array>, ctx: CompressCtx) -> ArrayRef { let like_alp = like.map(|like_array| like_array.as_alp()); + // TODO(ngates): fill forward nulls let parray = array.as_primitive(); - let (encoded, exponents, patches) = like_alp - .map(|alp_like| alp_encode_like_parts(parray, alp_like)) - .unwrap_or_else(|| alp_encode_parts(parray)); - ALPArray::new( - ctx.next_level() - .compress(encoded.as_ref(), like_alp.map(|a| a.encoded())), - exponents, - patches.map(|p| { - ctx.next_level() - .compress(p.as_ref(), like_alp.and_then(|a| a.patches())) - }), - ) - .boxed() -} + let (exponents, encoded, patches) = match parray.ptype() { + PType::F32 => encode_to_array(parray.typed_data::(), like_alp.map(|a| a.exponents())), + PType::F64 => encode_to_array(parray.typed_data::(), like_alp.map(|a| a.exponents())), + _ => panic!("Unsupported ptype"), + }; -pub fn alp_encode(parray: &PrimitiveArray) -> ALPArray { - let (encoded, exponents, patches) = alp_encode_parts(parray); - ALPArray::new(encoded, exponents, patches) -} + let compressed_encoded = ctx + .next_level() + .compress(encoded.as_ref(), like_alp.map(|a| a.encoded())); -fn alp_encode_parts(parray: &PrimitiveArray) -> (ArrayRef, ALPExponents, Option) { - match parray.ptype() { - PType::F32 => { - alp_encode_primitive(parray.buffer().typed_data::(), parray.validity(), None) - } - PType::F64 => { - alp_encode_primitive(parray.buffer().typed_data::(), parray.validity(), None) - } - _ => panic!("Unsupported ptype"), - } -} + let compressed_patches = patches.map(|p| { + ctx.next_level() + .compress(p.as_ref(), like_alp.and_then(|a| a.patches())) + }); -fn alp_encode_like_parts( - parray: &PrimitiveArray, - sample: &ALPArray, -) -> (ArrayRef, ALPExponents, Option) { - match parray.ptype() { - PType::F32 => alp_encode_primitive( - parray.buffer().typed_data::(), - parray.validity(), - Some(sample.exponents()), - ), - PType::F64 => alp_encode_primitive( - parray.buffer().typed_data::(), - parray.validity(), - Some(sample.exponents()), - ), - _ => panic!("Unsupported ptype"), - } + ALPArray::new(compressed_encoded, exponents, compressed_patches).boxed() } -fn alp_encode_primitive( +fn encode_to_array( values: &[T], - validity: Option<&dyn Array>, - exponents: Option, -) -> (ArrayRef, ALPExponents, Option) + exponents: Option<&Exponents>, +) -> (Exponents, ArrayRef, Option) where - T::EncInt: NativePType, + T: ALPFloat + NativePType, + T::ALPInt: NativePType, { - // TODO: actually handle CodecErrors instead of blindly unwrapping - let ALPEncoded { - values, + let (exponents, values, exc_pos, exc) = T::encode(values, exponents); + let len = values.len(); + ( exponents, - exceptions_idx, - num_exceptions, - } = exponents - .map(|exp| alp::encode_with(values, exp)) - .unwrap_or_else(|| alp::encode(values)) - .unwrap(); - let values = PrimitiveArray::from_nullable_in(values, validity.clone_optional()); // move and re-alias - - let patches = if num_exceptions == 0 { - None - } else { - let patch_indices = codecz::utils::into_u64_vec(&exceptions_idx, num_exceptions); - let patch_values = codecz::utils::gather_patches( - values.buffer().typed_data::(), - patch_indices.as_slice(), - ); - Some( + PrimitiveArray::from_vec(values).boxed(), + (!exc.is_empty()).then(|| { SparseArray::new( - PrimitiveArray::from_vec_in(patch_indices).boxed(), - PrimitiveArray::from_vec_in(patch_values).boxed(), - values.len(), + PrimitiveArray::from_vec(exc_pos).boxed(), + PrimitiveArray::from_vec(exc).boxed(), + len, ) - .boxed(), - ) - }; + .boxed() + }), + ) +} - (values.boxed(), exponents, patches) +pub fn alp_encode(parray: &PrimitiveArray) -> VortexResult { + let (exponents, encoded, patches) = match parray.ptype() { + PType::F32 => encode_to_array(parray.typed_data::(), None), + PType::F64 => encode_to_array(parray.typed_data::(), None), + _ => return Err(VortexError::InvalidPType(*parray.ptype())), + }; + Ok(ALPArray::new(encoded, exponents, patches)) } -#[allow(dead_code)] -pub fn alp_decode(parray: &PrimitiveArray, exp: ALPExponents) -> PrimitiveArray { - match parray.ptype() { - PType::I32 => PrimitiveArray::from_vec_in( - alp::decode::(parray.buffer().typed_data::(), exp).unwrap(), - ), - PType::I64 => PrimitiveArray::from_vec_in( - alp::decode::(parray.buffer().typed_data::(), exp).unwrap(), - ), - _ => panic!("Unsupported ptype"), +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_compress() { + let array = PrimitiveArray::from_vec(vec![1.234f32; 1025]); + let encoded = alp_encode(&array).unwrap(); + println!("Encoded {:?}", encoded); + assert!(encoded.patches().is_none()); + assert_eq!( + encoded.encoded().as_primitive().typed_data::(), + vec![1234; 1025] + ); + assert_eq!(encoded.exponents(), &Exponents { e: 4, f: 1 }); + } + + #[test] + fn test_nullable_compress() { + let array = PrimitiveArray::from_iter(vec![None, Some(1.234f32), None]); + let encoded = alp_encode(&array).unwrap(); + println!("Encoded {:?}", encoded); + assert!(encoded.patches().is_none()); + assert_eq!( + encoded.encoded().as_primitive().typed_data::(), + vec![0, 1234, 0] + ); + assert_eq!(encoded.exponents(), &Exponents { e: 4, f: 1 }); } } diff --git a/vortex-alp/src/compute.rs b/vortex-alp/src/compute.rs index d85d6e1072..86426ab162 100644 --- a/vortex-alp/src/compute.rs +++ b/vortex-alp/src/compute.rs @@ -1,10 +1,11 @@ +use crate::alp::ALPFloat; use crate::ALPArray; -use codecz::alp; +use std::f32; use vortex::array::Array; use vortex::compute::scalar_at::{scalar_at, ScalarAtFn}; use vortex::compute::ArrayCompute; use vortex::dtype::{DType, FloatWidth}; -use vortex::error::VortexResult; +use vortex::error::{VortexError, VortexResult}; use vortex::scalar::{NullableScalar, Scalar, ScalarRef}; impl ArrayCompute for ALPArray { @@ -26,22 +27,23 @@ impl ScalarAtFn for ALPArray { let Some(encoded_val) = scalar_at(self.encoded(), index)?.into_nonnull() else { return Ok(NullableScalar::none(self.dtype().clone()).boxed()); }; + match self.dtype() { DType::Float(FloatWidth::_32, _) => { let encoded_val: i32 = encoded_val.try_into().unwrap(); - Ok(alp::decode_single::(encoded_val, self.exponents()) - .unwrap() - .into()) + Ok(ScalarRef::from(::decode_single( + encoded_val, + self.exponents(), + ))) } - DType::Float(FloatWidth::_64, _) => { let encoded_val: i64 = encoded_val.try_into().unwrap(); - Ok(alp::decode_single::(encoded_val, self.exponents()) - .unwrap() - .into()) + Ok(ScalarRef::from(::decode_single( + encoded_val, + self.exponents(), + ))) } - - _ => unreachable!(), + _ => Err(VortexError::InvalidDType(self.dtype().clone())), } } } diff --git a/vortex-alp/src/lib.rs b/vortex-alp/src/lib.rs index 8ff1d432cc..f104396cd5 100644 --- a/vortex-alp/src/lib.rs +++ b/vortex-alp/src/lib.rs @@ -1,8 +1,11 @@ pub use alp::*; +pub use array::*; + use linkme::distributed_slice; use vortex::array::{EncodingRef, ENCODINGS}; mod alp; +mod array; mod compress; mod compute; mod downcast; diff --git a/vortex-alp/src/serde.rs b/vortex-alp/src/serde.rs index 58ef3e090e..9419fc13cc 100644 --- a/vortex-alp/src/serde.rs +++ b/vortex-alp/src/serde.rs @@ -1,12 +1,13 @@ use std::io; use std::io::ErrorKind; -use codecz::alp::ALPExponents; +use crate::alp::Exponents; use vortex::array::{Array, ArrayRef}; use vortex::dtype::{DType, FloatWidth, Signedness}; use vortex::serde::{ArraySerde, EncodingSerde, ReadCtx, WriteCtx}; -use crate::{ALPArray, ALPEncoding}; +use crate::ALPArray; +use crate::ALPEncoding; impl ArraySerde for ALPArray { fn write(&self, ctx: &mut WriteCtx) -> io::Result<()> { @@ -39,7 +40,7 @@ impl EncodingSerde for ALPEncoding { let encoded = ctx.with_schema(&encoded_dtype).read()?; Ok(ALPArray::new( encoded, - ALPExponents { + Exponents { e: exponents[0], f: exponents[1], }, @@ -77,7 +78,8 @@ mod test { 0.0004f64, 1000000.0f64, 0.33f64, - ])); + ])) + .unwrap(); let read_arr = roundtrip_array(arr.as_ref()).unwrap(); let read_alp = read_arr.as_alp(); diff --git a/vortex-array/src/array/chunked/compress.rs b/vortex-array/src/array/chunked/compress.rs index 1268ba6dfd..a5bf11aa54 100644 --- a/vortex-array/src/array/chunked/compress.rs +++ b/vortex-array/src/array/chunked/compress.rs @@ -1,9 +1,8 @@ -use rayon::prelude::*; - use crate::array::chunked::{ChunkedArray, ChunkedEncoding}; use crate::array::downcast::DowncastArrayBuiltin; use crate::array::{Array, ArrayRef}; use crate::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression}; +use itertools::Itertools; impl EncodingCompression for ChunkedEncoding { fn compressor( @@ -27,7 +26,7 @@ fn chunked_compressor(array: &dyn Array, like: Option<&dyn Array>, ctx: Compress .map(|c_like| { chunked_array .chunks() - .par_iter() + .iter() .zip_eq(c_like.chunks()) .map(|(chunk, chunk_like)| ctx.compress(chunk.as_ref(), Some(chunk_like.as_ref()))) .collect() @@ -35,7 +34,7 @@ fn chunked_compressor(array: &dyn Array, like: Option<&dyn Array>, ctx: Compress .unwrap_or_else(|| { chunked_array .chunks() - .par_iter() + .iter() .map(|chunk| ctx.compress(chunk.as_ref(), None)) .collect() }); diff --git a/vortex-array/src/lib.rs b/vortex-array/src/lib.rs index 0fbd8789b5..fad5ab605f 100644 --- a/vortex-array/src/lib.rs +++ b/vortex-array/src/lib.rs @@ -1,5 +1,3 @@ -#![feature(iterator_try_collect)] - pub mod array; pub mod arrow; pub mod scalar;