From 30d92bcc82c533cb28762ce43d92cf4f74ebd9ec Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Tue, 27 Feb 2024 18:26:14 +0000 Subject: [PATCH] Rename to vortex (#3) --- .github/workflows/ci.yml | 4 +- Cargo.lock | 292 +++++++++--------- Cargo.toml | 20 +- {enc-bench => bench-vortex}/Cargo.toml | 18 +- .../benches/compress_benchmark.rs | 30 +- {enc-bench => bench-vortex}/src/lib.rs | 28 +- codecz-sys/Cargo.toml | 2 +- codecz-sys/src/lib.rs | 6 +- codecz/Cargo.toml | 2 +- codecz/src/encodings/mod.rs | 8 +- pyenc/src/error.rs | 22 -- pyenc/test/test_dtype.py | 9 - pyproject.toml | 2 +- {pyenc => pyvortex}/Cargo.toml | 18 +- {pyenc => pyvortex}/pyproject.toml | 4 +- .../python/vortex}/__init__.py | 0 {pyenc => pyvortex}/src/array.rs | 50 +-- {pyenc => pyvortex}/src/compress.rs | 4 +- {pyenc => pyvortex}/src/dtype.rs | 10 +- {pyenc => pyvortex}/src/encode.rs | 18 +- pyvortex/src/error.rs | 22 ++ {pyenc => pyvortex}/src/lib.rs | 6 +- {pyenc => pyvortex}/src/serde.rs | 2 +- .../src/vortex_arrow.rs | 2 +- {pyenc => pyvortex}/test/conftest.py | 0 {pyenc => pyvortex}/test/test_array.py | 16 +- {pyenc => pyvortex}/test/test_compress.py | 36 +-- pyvortex/test/test_dtype.py | 9 + {pyenc => pyvortex}/test/test_serde.py | 12 +- requirements-dev.lock | 2 +- requirements.lock | 2 +- {spiral-alloc => vortex-alloc}/Cargo.toml | 2 +- {spiral-alloc => vortex-alloc}/src/alloc.rs | 10 +- {spiral-alloc => vortex-alloc}/src/lib.rs | 2 +- {enc-alp => vortex-alp}/Cargo.toml | 4 +- {enc-alp => vortex-alp}/src/alp.rs | 32 +- {enc-alp => vortex-alp}/src/compress.rs | 12 +- {enc-alp => vortex-alp}/src/downcast.rs | 2 +- {enc-alp => vortex-alp}/src/lib.rs | 2 +- {enc-alp => vortex-alp}/src/serde.rs | 14 +- {enc-alp => vortex-alp}/src/stats.rs | 2 +- {enc-dict => vortex-dict}/Cargo.toml | 4 +- {enc-dict => vortex-dict}/src/compress.rs | 24 +- {enc-dict => vortex-dict}/src/dict.rs | 26 +- {enc-dict => vortex-dict}/src/downcast.rs | 2 +- {enc-dict => vortex-dict}/src/lib.rs | 2 +- {enc-dict => vortex-dict}/src/serde.rs | 12 +- {enc-dict => vortex-dict}/src/stats.rs | 2 +- {enc-ffor => vortex-ffor}/Cargo.toml | 4 +- {enc-ffor => vortex-ffor}/src/compress.rs | 18 +- {enc-ffor => vortex-ffor}/src/downcast.rs | 2 +- {enc-ffor => vortex-ffor}/src/ffor.rs | 40 +-- {enc-ffor => vortex-ffor}/src/lib.rs | 2 +- {enc-ffor => vortex-ffor}/src/serde.rs | 12 +- {enc-ffor => vortex-ffor}/src/stats.rs | 2 +- {enc-ree => vortex-ree}/Cargo.toml | 4 +- {enc-ree => vortex-ree}/src/compress.rs | 20 +- {enc-ree => vortex-ree}/src/downcast.rs | 2 +- {enc-ree => vortex-ree}/src/lib.rs | 2 +- {enc-ree => vortex-ree}/src/ree.rs | 48 +-- {enc-ree => vortex-ree}/src/serde.rs | 12 +- {enc-ree => vortex-ree}/src/stats.rs | 2 +- {enc-roaring => vortex-roaring}/Cargo.toml | 4 +- .../src/boolean/compress.rs | 12 +- .../src/boolean/mod.rs | 38 +-- .../src/boolean/serde.rs | 4 +- .../src/boolean/stats.rs | 18 +- .../src/downcast.rs | 2 +- .../src/integer/compress.rs | 18 +- .../src/integer/mod.rs | 38 +-- .../src/integer/serde.rs | 8 +- .../src/integer/stats.rs | 2 +- {enc-roaring => vortex-roaring}/src/lib.rs | 2 +- .../src/serde_tests.rs | 4 +- {enc-zigzag => vortex-zigzag}/Cargo.toml | 6 +- {enc-zigzag => vortex-zigzag}/src/compress.rs | 18 +- {enc-zigzag => vortex-zigzag}/src/downcast.rs | 2 +- {enc-zigzag => vortex-zigzag}/src/lib.rs | 2 +- {enc-zigzag => vortex-zigzag}/src/serde.rs | 14 +- {enc-zigzag => vortex-zigzag}/src/stats.rs | 2 +- {enc-zigzag => vortex-zigzag}/src/zigzag.rs | 32 +- {enc => vortex}/Cargo.toml | 6 +- {enc => vortex}/README.md | 0 {enc => vortex}/src/array/bool/compress.rs | 0 {enc => vortex}/src/array/bool/mod.rs | 10 +- {enc => vortex}/src/array/bool/serde.rs | 0 {enc => vortex}/src/array/bool/stats.rs | 0 {enc => vortex}/src/array/chunked/compress.rs | 0 {enc => vortex}/src/array/chunked/mod.rs | 12 +- {enc => vortex}/src/array/chunked/serde.rs | 0 {enc => vortex}/src/array/chunked/stats.rs | 0 .../src/array/constant/compress.rs | 0 {enc => vortex}/src/array/constant/compute.rs | 0 {enc => vortex}/src/array/constant/mod.rs | 8 +- {enc => vortex}/src/array/constant/serde.rs | 0 {enc => vortex}/src/array/constant/stats.rs | 0 {enc => vortex}/src/array/constant/take.rs | 4 +- {enc => vortex}/src/array/downcast.rs | 0 {enc => vortex}/src/array/mod.rs | 22 +- .../src/array/primitive/compress.rs | 0 {enc => vortex}/src/array/primitive/mod.rs | 10 +- {enc => vortex}/src/array/primitive/serde.rs | 0 {enc => vortex}/src/array/primitive/stats.rs | 0 {enc => vortex}/src/array/sparse/compress.rs | 0 {enc => vortex}/src/array/sparse/mod.rs | 24 +- {enc => vortex}/src/array/sparse/serde.rs | 0 {enc => vortex}/src/array/sparse/stats.rs | 0 {enc => vortex}/src/array/struct_/compress.rs | 0 {enc => vortex}/src/array/struct_/mod.rs | 8 +- {enc => vortex}/src/array/struct_/serde.rs | 0 {enc => vortex}/src/array/struct_/stats.rs | 0 {enc => vortex}/src/array/typed/compress.rs | 0 {enc => vortex}/src/array/typed/mod.rs | 8 +- {enc => vortex}/src/array/typed/serde.rs | 0 {enc => vortex}/src/array/typed/stats.rs | 0 {enc => vortex}/src/array/varbin/compress.rs | 0 {enc => vortex}/src/array/varbin/mod.rs | 20 +- {enc => vortex}/src/array/varbin/serde.rs | 0 {enc => vortex}/src/array/varbin/stats.rs | 8 +- .../src/array/varbinview/compress.rs | 0 {enc => vortex}/src/array/varbinview/mod.rs | 18 +- {enc => vortex}/src/array/varbinview/serde.rs | 0 {enc => vortex}/src/arrow/aligned_iter.rs | 0 {enc => vortex}/src/arrow/compute/mod.rs | 0 {enc => vortex}/src/arrow/compute/repeat.rs | 0 {enc => vortex}/src/arrow/convert.rs | 28 +- {enc => vortex}/src/arrow/mod.rs | 0 {enc => vortex}/src/compress.rs | 0 {enc => vortex}/src/compute/add.rs | 10 +- {enc => vortex}/src/compute/as_contiguous.rs | 18 +- {enc => vortex}/src/compute/cast.rs | 0 {enc => vortex}/src/compute/mod.rs | 0 {enc => vortex}/src/compute/repeat.rs | 0 {enc => vortex}/src/compute/search_sorted.rs | 4 +- {enc => vortex}/src/compute/take.rs | 8 +- {enc => vortex}/src/dtype.rs | 0 {enc => vortex}/src/encode.rs | 0 {enc => vortex}/src/error.rs | 12 +- {enc => vortex}/src/formatter.rs | 2 +- {enc => vortex}/src/lib.rs | 0 {enc => vortex}/src/polars.rs | 0 {enc => vortex}/src/ptype.rs | 10 +- {enc => vortex}/src/sampling.rs | 0 {enc => vortex}/src/scalar/arrow.rs | 0 {enc => vortex}/src/scalar/binary.rs | 12 +- {enc => vortex}/src/scalar/bool.rs | 16 +- {enc => vortex}/src/scalar/equal.rs | 0 {enc => vortex}/src/scalar/list.rs | 20 +- {enc => vortex}/src/scalar/localtime.rs | 4 +- {enc => vortex}/src/scalar/mod.rs | 4 +- {enc => vortex}/src/scalar/null.rs | 4 +- {enc => vortex}/src/scalar/nullable.rs | 16 +- {enc => vortex}/src/scalar/ord.rs | 0 {enc => vortex}/src/scalar/primitive.rs | 72 ++--- {enc => vortex}/src/scalar/serde.rs | 0 {enc => vortex}/src/scalar/struct_.rs | 8 +- {enc => vortex}/src/scalar/utf8.rs | 12 +- {enc => vortex}/src/serde/dtype.rs | 0 {enc => vortex}/src/serde/mod.rs | 0 {enc => vortex}/src/stats.rs | 15 +- zig/c-abi/include/prelude.h | 6 +- zig/c-abi/test_wrapper.zig | 24 +- zig/c-abi/types.zig | 2 +- zig/c-abi/wrapper.zig | 2 +- 164 files changed, 842 insertions(+), 831 deletions(-) rename {enc-bench => bench-vortex}/Cargo.toml (54%) rename enc-bench/benches/enc_compress_benchmark.rs => bench-vortex/benches/compress_benchmark.rs (82%) rename {enc-bench => bench-vortex}/src/lib.rs (81%) delete mode 100644 pyenc/src/error.rs delete mode 100644 pyenc/test/test_dtype.py rename {pyenc => pyvortex}/Cargo.toml (62%) rename {pyenc => pyvortex}/pyproject.toml (91%) rename {pyenc/python/enc => pyvortex/python/vortex}/__init__.py (100%) rename {pyenc => pyvortex}/src/array.rs (84%) rename {pyenc => pyvortex}/src/compress.rs (88%) rename {pyenc => pyvortex}/src/dtype.rs (81%) rename {pyenc => pyvortex}/src/encode.rs (84%) create mode 100644 pyvortex/src/error.rs rename {pyenc => pyvortex}/src/lib.rs (97%) rename {pyenc => pyvortex}/src/serde.rs (98%) rename pyenc/src/enc_arrow.rs => pyvortex/src/vortex_arrow.rs (97%) rename {pyenc => pyvortex}/test/conftest.py (100%) rename {pyenc => pyvortex}/test/test_array.py (68%) rename {pyenc => pyvortex}/test/test_compress.py (56%) create mode 100644 pyvortex/test/test_dtype.py rename {pyenc => pyvortex}/test/test_serde.py (59%) rename {spiral-alloc => vortex-alloc}/Cargo.toml (90%) rename {spiral-alloc => vortex-alloc}/src/alloc.rs (86%) rename {spiral-alloc => vortex-alloc}/src/lib.rs (75%) rename {enc-alp => vortex-alp}/Cargo.toml (85%) rename {enc-alp => vortex-alp}/src/alp.rs (81%) rename {enc-alp => vortex-alp}/src/compress.rs (93%) rename {enc-alp => vortex-alp}/src/downcast.rs (93%) rename {enc-alp => vortex-alp}/src/lib.rs (80%) rename {enc-alp => vortex-alp}/src/serde.rs (88%) rename {enc-alp => vortex-alp}/src/stats.rs (82%) rename {enc-dict => vortex-dict}/Cargo.toml (79%) rename {enc-dict => vortex-dict}/src/compress.rs (95%) rename {enc-dict => vortex-dict}/src/dict.rs (79%) rename {enc-dict => vortex-dict}/src/downcast.rs (93%) rename {enc-dict => vortex-dict}/src/lib.rs (82%) rename {enc-dict => vortex-dict}/src/serde.rs (86%) rename {enc-dict => vortex-dict}/src/stats.rs (96%) rename {enc-ffor => vortex-ffor}/Cargo.toml (80%) rename {enc-ffor => vortex-ffor}/src/compress.rs (91%) rename {enc-ffor => vortex-ffor}/src/downcast.rs (93%) rename {enc-ffor => vortex-ffor}/src/ffor.rs (82%) rename {enc-ffor => vortex-ffor}/src/lib.rs (81%) rename {enc-ffor => vortex-ffor}/src/serde.rs (90%) rename {enc-ffor => vortex-ffor}/src/stats.rs (82%) rename {enc-ree => vortex-ree}/Cargo.toml (79%) rename {enc-ree => vortex-ree}/src/compress.rs (89%) rename {enc-ree => vortex-ree}/src/downcast.rs (93%) rename {enc-ree => vortex-ree}/src/lib.rs (80%) rename {enc-ree => vortex-ree}/src/ree.rs (87%) rename {enc-ree => vortex-ree}/src/serde.rs (87%) rename {enc-ree => vortex-ree}/src/stats.rs (71%) rename {enc-roaring => vortex-roaring}/Cargo.toml (73%) rename {enc-roaring => vortex-roaring}/src/boolean/compress.rs (80%) rename {enc-roaring => vortex-roaring}/src/boolean/mod.rs (81%) rename {enc-roaring => vortex-roaring}/src/boolean/serde.rs (92%) rename {enc-roaring => vortex-roaring}/src/boolean/stats.rs (88%) rename {enc-roaring => vortex-roaring}/src/downcast.rs (96%) rename {enc-roaring => vortex-roaring}/src/integer/compress.rs (84%) rename {enc-roaring => vortex-roaring}/src/integer/mod.rs (79%) rename {enc-roaring => vortex-roaring}/src/integer/serde.rs (89%) rename {enc-roaring => vortex-roaring}/src/integer/stats.rs (92%) rename {enc-roaring => vortex-roaring}/src/lib.rs (87%) rename {enc-roaring => vortex-roaring}/src/serde_tests.rs (81%) rename {enc-zigzag => vortex-zigzag}/Cargo.toml (50%) rename {enc-zigzag => vortex-zigzag}/src/compress.rs (88%) rename {enc-zigzag => vortex-zigzag}/src/downcast.rs (94%) rename {enc-zigzag => vortex-zigzag}/src/lib.rs (81%) rename {enc-zigzag => vortex-zigzag}/src/serde.rs (83%) rename {enc-zigzag => vortex-zigzag}/src/stats.rs (83%) rename {enc-zigzag => vortex-zigzag}/src/zigzag.rs (78%) rename {enc => vortex}/Cargo.toml (87%) rename {enc => vortex}/README.md (100%) rename {enc => vortex}/src/array/bool/compress.rs (100%) rename {enc => vortex}/src/array/bool/mod.rs (95%) rename {enc => vortex}/src/array/bool/serde.rs (100%) rename {enc => vortex}/src/array/bool/stats.rs (100%) rename {enc => vortex}/src/array/chunked/compress.rs (100%) rename {enc => vortex}/src/array/chunked/mod.rs (95%) rename {enc => vortex}/src/array/chunked/serde.rs (100%) rename {enc => vortex}/src/array/chunked/stats.rs (100%) rename {enc => vortex}/src/array/constant/compress.rs (100%) rename {enc => vortex}/src/array/constant/compute.rs (100%) rename {enc => vortex}/src/array/constant/mod.rs (91%) rename {enc => vortex}/src/array/constant/serde.rs (100%) rename {enc => vortex}/src/array/constant/stats.rs (100%) rename {enc => vortex}/src/array/constant/take.rs (71%) rename {enc => vortex}/src/array/downcast.rs (100%) rename {enc => vortex}/src/array/mod.rs (88%) rename {enc => vortex}/src/array/primitive/compress.rs (100%) rename {enc => vortex}/src/array/primitive/mod.rs (96%) rename {enc => vortex}/src/array/primitive/serde.rs (100%) rename {enc => vortex}/src/array/primitive/stats.rs (100%) rename {enc => vortex}/src/array/sparse/compress.rs (100%) rename {enc => vortex}/src/array/sparse/mod.rs (94%) rename {enc => vortex}/src/array/sparse/serde.rs (100%) rename {enc => vortex}/src/array/sparse/stats.rs (100%) rename {enc => vortex}/src/array/struct_/compress.rs (100%) rename {enc => vortex}/src/array/struct_/mod.rs (96%) rename {enc => vortex}/src/array/struct_/serde.rs (100%) rename {enc => vortex}/src/array/struct_/stats.rs (100%) rename {enc => vortex}/src/array/typed/compress.rs (100%) rename {enc => vortex}/src/array/typed/mod.rs (95%) rename {enc => vortex}/src/array/typed/serde.rs (100%) rename {enc => vortex}/src/array/typed/stats.rs (100%) rename {enc => vortex}/src/array/varbin/compress.rs (100%) rename {enc => vortex}/src/array/varbin/mod.rs (95%) rename {enc => vortex}/src/array/varbin/serde.rs (100%) rename {enc => vortex}/src/array/varbin/stats.rs (95%) rename {enc => vortex}/src/array/varbinview/compress.rs (100%) rename {enc => vortex}/src/array/varbinview/mod.rs (94%) rename {enc => vortex}/src/array/varbinview/serde.rs (100%) rename {enc => vortex}/src/arrow/aligned_iter.rs (100%) rename {enc => vortex}/src/arrow/compute/mod.rs (100%) rename {enc => vortex}/src/arrow/compute/repeat.rs (100%) rename {enc => vortex}/src/arrow/convert.rs (92%) rename {enc => vortex}/src/arrow/mod.rs (100%) rename {enc => vortex}/src/compress.rs (100%) rename {enc => vortex}/src/compute/add.rs (80%) rename {enc => vortex}/src/compute/as_contiguous.rs (83%) rename {enc => vortex}/src/compute/cast.rs (100%) rename {enc => vortex}/src/compute/mod.rs (100%) rename {enc => vortex}/src/compute/repeat.rs (100%) rename {enc => vortex}/src/compute/search_sorted.rs (96%) rename {enc => vortex}/src/compute/take.rs (61%) rename {enc => vortex}/src/dtype.rs (100%) rename {enc => vortex}/src/encode.rs (100%) rename {enc => vortex}/src/error.rs (90%) rename {enc => vortex}/src/formatter.rs (93%) rename {enc => vortex}/src/lib.rs (100%) rename {enc => vortex}/src/polars.rs (100%) rename {enc => vortex}/src/ptype.rs (94%) rename {enc => vortex}/src/sampling.rs (100%) rename {enc => vortex}/src/scalar/arrow.rs (100%) rename {enc => vortex}/src/scalar/binary.rs (85%) rename {enc => vortex}/src/scalar/bool.rs (82%) rename {enc => vortex}/src/scalar/equal.rs (100%) rename {enc => vortex}/src/scalar/list.rs (85%) rename {enc => vortex}/src/scalar/localtime.rs (94%) rename {enc => vortex}/src/scalar/mod.rs (94%) rename {enc => vortex}/src/scalar/null.rs (91%) rename {enc => vortex}/src/scalar/nullable.rs (87%) rename {enc => vortex}/src/scalar/ord.rs (100%) rename {enc => vortex}/src/scalar/primitive.rs (78%) rename {enc => vortex}/src/scalar/serde.rs (100%) rename {enc => vortex}/src/scalar/struct_.rs (90%) rename {enc => vortex}/src/scalar/utf8.rs (86%) rename {enc => vortex}/src/serde/dtype.rs (100%) rename {enc => vortex}/src/serde/mod.rs (100%) rename {enc => vortex}/src/stats.rs (96%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3eab6a16a..9c7dcad81e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: - name: Zig Test run: zig build test - - name: Pytest - PyEnc + - name: Pytest - PyVortex run: rye run pytest --benchmark-disable test/ - working-directory: pyenc/ + working-directory: pyvortex/ diff --git a/Cargo.lock b/Cargo.lock index f36cab5e5d..49fafdbf0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -566,8 +566,8 @@ dependencies = [ "rand", "rand_chacha", "safe-transmute", - "spiral-alloc", "thiserror", + "vortex-alloc", ] [[package]] @@ -576,7 +576,7 @@ version = "0.1.0" dependencies = [ "bindgen", "safe-transmute", - "spiral-alloc", + "vortex-alloc", "walkdir", ] @@ -762,124 +762,6 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" -[[package]] -name = "enc" -version = "0.1.0" -dependencies = [ - "allocator-api2", - "arrow", - "dyn-clone", - "half", - "humansize", - "itertools 0.12.1", - "leb128", - "linkme", - "log", - "num-traits", - "num_enum", - "once_cell", - "polars-arrow", - "polars-core", - "polars-ops", - "rand", - "rayon", - "roaring", - "spiral-alloc", - "thiserror", -] - -[[package]] -name = "enc-alp" -version = "0.1.0" -dependencies = [ - "arrow", - "codecz", - "enc", - "itertools 0.12.1", - "linkme", - "log", -] - -[[package]] -name = "enc-bench" -version = "0.1.0" -dependencies = [ - "arrow-array", - "criterion", - "enc", - "enc-alp", - "enc-dict", - "enc-ffor", - "enc-ree", - "enc-roaring", - "enc-zigzag", - "itertools 0.12.1", - "log", - "parquet", - "rand", - "reqwest", - "simplelog", -] - -[[package]] -name = "enc-dict" -version = "0.1.0" -dependencies = [ - "ahash", - "enc", - "half", - "hashbrown", - "linkme", - "log", - "num-traits", -] - -[[package]] -name = "enc-ffor" -version = "0.1.0" -dependencies = [ - "arrow", - "codecz", - "enc", - "itertools 0.12.1", - "linkme", - "log", -] - -[[package]] -name = "enc-ree" -version = "0.1.0" -dependencies = [ - "arrow", - "codecz", - "enc", - "half", - "itertools 0.10.5", - "linkme", - "num-traits", -] - -[[package]] -name = "enc-roaring" -version = "0.1.0" -dependencies = [ - "croaring", - "enc", - "linkme", - "log", - "num-traits", -] - -[[package]] -name = "enc-zigzag" -version = "0.1.0" -dependencies = [ - "enc", - "linkme", - "spiral-alloc", - "zigzag", -] - [[package]] name = "encoding_rs" version = "0.8.33" @@ -2069,25 +1951,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "pyenc" -version = "0.1.0" -dependencies = [ - "arrow", - "enc", - "enc-alp", - "enc-dict", - "enc-ffor", - "enc-ree", - "enc-roaring", - "enc-zigzag", - "itertools 0.12.1", - "log", - "paste", - "pyo3", - "pyo3-log", -] - [[package]] name = "pyo3" version = "0.20.3" @@ -2162,6 +2025,25 @@ dependencies = [ "syn 2.0.51", ] +[[package]] +name = "pyvortex" +version = "0.1.0" +dependencies = [ + "arrow", + "itertools 0.12.1", + "log", + "paste", + "pyo3", + "pyo3-log", + "vortex", + "vortex-alp", + "vortex-dict", + "vortex-ffor", + "vortex-ree", + "vortex-roaring", + "vortex-zigzag", +] + [[package]] name = "quote" version = "1.0.35" @@ -2532,13 +2414,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "spiral-alloc" -version = "0.1.0" -dependencies = [ - "allocator-api2", -] - [[package]] name = "static_assertions" version = "1.1.0" @@ -2891,6 +2766,131 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "vortex" +version = "0.1.0" +dependencies = [ + "allocator-api2", + "arrow", + "dyn-clone", + "half", + "humansize", + "itertools 0.12.1", + "leb128", + "linkme", + "log", + "num-traits", + "num_enum", + "once_cell", + "polars-arrow", + "polars-core", + "polars-ops", + "rand", + "rayon", + "roaring", + "thiserror", + "vortex-alloc", +] + +[[package]] +name = "vortex-alloc" +version = "0.1.0" +dependencies = [ + "allocator-api2", +] + +[[package]] +name = "vortex-alp" +version = "0.1.0" +dependencies = [ + "arrow", + "codecz", + "itertools 0.12.1", + "linkme", + "log", + "vortex", +] + +[[package]] +name = "vortex-bench" +version = "0.1.0" +dependencies = [ + "arrow-array", + "criterion", + "itertools 0.12.1", + "log", + "parquet", + "rand", + "reqwest", + "simplelog", + "vortex", + "vortex-alp", + "vortex-dict", + "vortex-ffor", + "vortex-ree", + "vortex-roaring", + "vortex-zigzag", +] + +[[package]] +name = "vortex-dict" +version = "0.1.0" +dependencies = [ + "ahash", + "half", + "hashbrown", + "linkme", + "log", + "num-traits", + "vortex", +] + +[[package]] +name = "vortex-ffor" +version = "0.1.0" +dependencies = [ + "arrow", + "codecz", + "itertools 0.12.1", + "linkme", + "log", + "vortex", +] + +[[package]] +name = "vortex-ree" +version = "0.1.0" +dependencies = [ + "arrow", + "codecz", + "half", + "itertools 0.10.5", + "linkme", + "num-traits", + "vortex", +] + +[[package]] +name = "vortex-roaring" +version = "0.1.0" +dependencies = [ + "croaring", + "linkme", + "log", + "num-traits", + "vortex", +] + +[[package]] +name = "vortex-zigzag" +version = "0.1.0" +dependencies = [ + "linkme", + "vortex", + "vortex-alloc", + "zigzag", +] + [[package]] name = "walkdir" version = "2.4.0" diff --git a/Cargo.toml b/Cargo.toml index 777d71795c..62b7edd0bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,17 +1,17 @@ [workspace] members = [ - "pyenc", + "bench-vortex", "codecz", "codecz-sys", - "enc", - "enc-alp", - "enc-bench", - "enc-dict", - "enc-ffor", - "enc-ree", - "enc-roaring", - "enc-zigzag", - "spiral-alloc", + "pyvortex", + "vortex", + "vortex-alloc", + "vortex-alp", + "vortex-dict", + "vortex-ffor", + "vortex-ree", + "vortex-roaring", + "vortex-zigzag", ] resolver = "2" diff --git a/enc-bench/Cargo.toml b/bench-vortex/Cargo.toml similarity index 54% rename from enc-bench/Cargo.toml rename to bench-vortex/Cargo.toml index 476a7bf771..70939d3d62 100644 --- a/enc-bench/Cargo.toml +++ b/bench-vortex/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "enc-bench" +name = "vortex-bench" version = "0.1.0" edition = "2021" @@ -8,13 +8,13 @@ workspace = true [dependencies] arrow-array = "50.0.0" -enc = { path = "../enc" } -enc-alp = { path = "../enc-alp" } -enc-dict = { path = "../enc-dict" } -enc-ffor = { path = "../enc-ffor"} -enc-ree = { path = "../enc-ree" } -enc-roaring = { path = "../enc-roaring" } -enc-zigzag = { path = "../enc-zigzag" } +vortex = { path = "../vortex" } +vortex-alp = { path = "../vortex-alp" } +vortex-dict = { path = "../vortex-dict" } +vortex-ffor = { path = "../vortex-ffor" } +vortex-ree = { path = "../vortex-ree" } +vortex-roaring = { path = "../vortex-roaring" } +vortex-zigzag = { path = "../vortex-zigzag" } itertools = "0.12.1" [dev-dependencies] @@ -26,5 +26,5 @@ reqwest = { version = "0.11.24", features = ["blocking"] } simplelog = { version = "0.12.1", features = ["paris"] } [[bench]] -name = "enc_compress_benchmark" +name = "compress_benchmark" harness = false \ No newline at end of file diff --git a/enc-bench/benches/enc_compress_benchmark.rs b/bench-vortex/benches/compress_benchmark.rs similarity index 82% rename from enc-bench/benches/enc_compress_benchmark.rs rename to bench-vortex/benches/compress_benchmark.rs index ccbb27b4aa..05d947cafb 100644 --- a/enc-bench/benches/enc_compress_benchmark.rs +++ b/bench-vortex/benches/compress_benchmark.rs @@ -8,15 +8,15 @@ use rand::distributions::{Alphanumeric, Uniform}; use rand::prelude::SliceRandom; use rand::{thread_rng, Rng}; -use enc::array::chunked::ChunkedArray; -use enc::array::primitive::PrimitiveArray; -use enc::array::varbin::VarBinArray; -use enc::array::{Array, ArrayRef}; -use enc::compress::CompressCtx; -use enc::dtype::DType; -use enc::error::{EncError, EncResult}; +use vortex::array::chunked::ChunkedArray; +use vortex::array::primitive::PrimitiveArray; +use vortex::array::varbin::VarBinArray; +use vortex::array::{Array, ArrayRef}; +use vortex::compress::CompressCtx; +use vortex::dtype::DType; +use vortex::error::{VortexError, VortexResult}; -use enc_bench::enumerate_arrays; +use vortex_bench::enumerate_arrays; fn download_taxi_data() -> &'static Path { let download_path = Path::new("../../pyspiral/bench/.data/https-d37ci6vzurychx-cloudfront-net-trip-data-yellow-tripdata-2023-11.parquet"); @@ -55,9 +55,9 @@ fn enc_compress(c: &mut Criterion) { let schema = reader.schema(); let dtype: DType = schema.try_into().unwrap(); let chunks = reader - .map(|batch_result| batch_result.map_err(EncError::from)) + .map(|batch_result| batch_result.map_err(VortexError::from)) .map(|batch| batch.map(|b| b.into())) - .collect::>>() + .collect::>>() .unwrap(); let chunked = ChunkedArray::new(chunks, dtype); println!( @@ -67,7 +67,7 @@ fn enc_compress(c: &mut Criterion) { ); let array = chunked.boxed(); - c.bench_function("enc.compress", |b| { + c.bench_function("compress", |b| { b.iter(|| black_box(compress(array.clone()))) }); } @@ -100,12 +100,12 @@ fn gen_varbin_dict(len: usize, uniqueness: f64) -> VarBinArray { } fn dict_encode_primitive(arr: &PrimitiveArray) -> usize { - let (codes, values) = enc_dict::dict_encode_primitive(arr); + let (codes, values) = vortex_dict::dict_encode_primitive(arr); (codes.nbytes() + values.nbytes()) / arr.nbytes() } fn dict_encode_varbin(arr: &VarBinArray) -> usize { - let (codes, values) = enc_dict::dict_encode_varbin(arr); + let (codes, values) = vortex_dict::dict_encode_varbin(arr); (codes.nbytes() + values.nbytes()) / arr.nbytes() } @@ -113,10 +113,10 @@ fn dict_encode(c: &mut Criterion) { let primitive_arr = gen_primitive_dict(1_000_000, 0.05); let varbin_arr = gen_varbin_dict(1_000_000, 0.05); - c.bench_function("enc.dict_encode_primitives", |b| { + c.bench_function("dict_encode_primitives", |b| { b.iter(|| black_box(dict_encode_primitive(&primitive_arr))); }); - c.bench_function("enc.dict_encode_varbin", |b| { + c.bench_function("dict_encode_varbin", |b| { b.iter(|| black_box(dict_encode_varbin(&varbin_arr))); }); } diff --git a/enc-bench/src/lib.rs b/bench-vortex/src/lib.rs similarity index 81% rename from enc-bench/src/lib.rs rename to bench-vortex/src/lib.rs index 147a915638..78e1da74a2 100644 --- a/enc-bench/src/lib.rs +++ b/bench-vortex/src/lib.rs @@ -1,12 +1,12 @@ use itertools::Itertools; -use enc::array::Encoding; -use enc_alp::ALPEncoding; -use enc_dict::DictEncoding; -use enc_ffor::FFoREncoding; -use enc_ree::REEEncoding; -use enc_roaring::{RoaringBoolEncoding, RoaringIntEncoding}; -use enc_zigzag::ZigZagEncoding; +use vortex::array::Encoding; +use vortex_alp::ALPEncoding; +use vortex_dict::DictEncoding; +use vortex_ffor::FFoREncoding; +use vortex_ree::REEEncoding; +use vortex_roaring::{RoaringBoolEncoding, RoaringIntEncoding}; +use vortex_zigzag::ZigZagEncoding; pub fn enumerate_arrays() { let encodings: Vec<&dyn Encoding> = vec![ @@ -33,11 +33,11 @@ mod test { use simplelog::{ColorChoice, Config, TermLogger, TerminalMode}; use crate::enumerate_arrays; - use enc::array::chunked::ChunkedArray; - use enc::array::{Array, ArrayRef}; - use enc::compress::CompressCtx; - use enc::dtype::DType; - use enc::error::{EncError, EncResult}; + use vortex::array::chunked::ChunkedArray; + use vortex::array::{Array, ArrayRef}; + use vortex::compress::CompressCtx; + use vortex::dtype::DType; + use vortex::error::{VortexError, VortexResult}; pub fn download_taxi_data() -> &'static Path { let download_path = Path::new("../../pyspiral/bench/.data/https-d37ci6vzurychx-cloudfront-net-trip-data-yellow-tripdata-2023-11.parquet"); @@ -79,9 +79,9 @@ mod test { let schema = reader.schema(); let dtype: DType = schema.try_into().unwrap(); let chunks = reader - .map(|batch_result| batch_result.map_err(EncError::from)) + .map(|batch_result| batch_result.map_err(VortexError::from)) .map(|batch| batch.map(|b| b.into())) - .collect::>>() + .collect::>>() .unwrap(); let chunked = ChunkedArray::new(chunks, dtype); println!( diff --git a/codecz-sys/Cargo.toml b/codecz-sys/Cargo.toml index 23be8516cc..96e029ada9 100644 --- a/codecz-sys/Cargo.toml +++ b/codecz-sys/Cargo.toml @@ -9,7 +9,7 @@ workspace = true [dependencies] safe-transmute = "0.11.2" -spiral-alloc = { version = "0.1.0", path = "../spiral-alloc" } +vortex-alloc = { version = "0.1.0", path = "../vortex-alloc" } [build-dependencies] bindgen = "0.69.1" diff --git a/codecz-sys/src/lib.rs b/codecz-sys/src/lib.rs index e464897cc6..3ddf164259 100644 --- a/codecz-sys/src/lib.rs +++ b/codecz-sys/src/lib.rs @@ -4,7 +4,7 @@ use core::fmt::{Display, Formatter}; use core::mem::size_of; -use spiral_alloc::AlignedVec; +use vortex_alloc::AlignedVec; include!(concat!(env!("OUT_DIR"), "/bindings.rs")); @@ -34,8 +34,8 @@ where T: Sized, { fn from(slice: &[T]) -> Self { - if slice.as_ptr().align_offset(SPIRAL_ALIGNMENT as usize) != 0 { - panic!("slice must be aligned to {} bytes", SPIRAL_ALIGNMENT); + if slice.as_ptr().align_offset(VORTEX_ALIGNMENT as usize) != 0 { + panic!("slice must be aligned to {} bytes", VORTEX_ALIGNMENT); } Self { ptr: slice.as_ptr() as *mut u8, diff --git a/codecz/Cargo.toml b/codecz/Cargo.toml index 4200925248..72e858bb85 100644 --- a/codecz/Cargo.toml +++ b/codecz/Cargo.toml @@ -17,7 +17,7 @@ codecz-sys = { version = "0.1.0", path = "../codecz-sys" } half = "2.3.1" arrow-buffer = "50.0.0" itertools = "0.12.1" -spiral-alloc = { version = "0.1.0", path = "../spiral-alloc" } +vortex-alloc = { version = "0.1.0", path = "../vortex-alloc" } [dependencies.num-traits] version = "0.2" diff --git a/codecz/src/encodings/mod.rs b/codecz/src/encodings/mod.rs index 05de3c3c10..5a38976ced 100644 --- a/codecz/src/encodings/mod.rs +++ b/codecz/src/encodings/mod.rs @@ -6,7 +6,7 @@ pub mod alp; pub mod ffor; pub mod ree; -pub use spiral_alloc::{AlignedAllocator, AlignedVec, ALIGNED_ALLOCATOR, SPIRAL_ALIGNMENT}; +pub use vortex_alloc::{AlignedAllocator, AlignedVec, ALIGNED_ALLOCATOR, VORTEX_ALIGNMENT}; pub(crate) type ByteBuffer = codecz_sys::ByteBuffer_t; pub(crate) type WrittenBuffer = codecz_sys::WrittenBuffer_t; @@ -36,11 +36,11 @@ mod test { #[allow(clippy::assertions_on_constants)] fn test_alignment() { assert_eq!( - codecz_sys::SPIRAL_ALIGNMENT as usize, - spiral_alloc::SPIRAL_ALIGNMENT, + codecz_sys::VORTEX_ALIGNMENT as usize, + vortex_alloc::VORTEX_ALIGNMENT, ); assert_eq!( - codecz_sys::SPIRAL_ALIGNMENT as usize, + codecz_sys::VORTEX_ALIGNMENT as usize, super::ALIGNED_ALLOCATOR.min_alignment(), ); assert!(arrow_buffer::alloc::ALIGNMENT >= 64); diff --git a/pyenc/src/error.rs b/pyenc/src/error.rs deleted file mode 100644 index 21c5009d67..0000000000 --- a/pyenc/src/error.rs +++ /dev/null @@ -1,22 +0,0 @@ -use pyo3::exceptions::PyValueError; -use pyo3::PyErr; - -use enc::error::EncError; - -pub struct PyEncError(EncError); - -impl PyEncError { - pub fn new(error: EncError) -> Self { - Self(error) - } - - pub fn map_err(error: EncError) -> PyErr { - PyEncError::new(error).into() - } -} - -impl From for PyErr { - fn from(value: PyEncError) -> Self { - PyValueError::new_err(value.0.to_string()) - } -} diff --git a/pyenc/test/test_dtype.py b/pyenc/test/test_dtype.py deleted file mode 100644 index 8e15a3c4ff..0000000000 --- a/pyenc/test/test_dtype.py +++ /dev/null @@ -1,9 +0,0 @@ -import enc - - -def test_int(): - assert str(enc.int()) == "signed_int(_)" - assert str(enc.int(32)) == "signed_int(32)" - assert str(enc.int(32, signed=False)) == "unsigned_int(32)" - assert str(enc.float(16)) == "float(16)" - assert str(enc.bool(nullable=True)) == "bool?" diff --git a/pyproject.toml b/pyproject.toml index cffda950cc..2398bb5dae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ serve-docs = "mkdocs serve" [tool.rye.workspace] members = [ - "pyenc", + "pyvortex", ] [tool.ruff] diff --git a/pyenc/Cargo.toml b/pyvortex/Cargo.toml similarity index 62% rename from pyenc/Cargo.toml rename to pyvortex/Cargo.toml index d9d207be60..3f648d72bc 100644 --- a/pyenc/Cargo.toml +++ b/pyvortex/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "pyenc" +name = "pyvortex" version = "0.1.0" edition = "2021" @@ -7,18 +7,18 @@ edition = "2021" workspace = true [lib] -name = "pyenc" +name = "pyvortex" crate-type = ["rlib", "cdylib"] [dependencies] arrow = { version = "50.0.0", features = ["ffi"] } -enc = { path = "../enc" } -enc-alp = { path = "../enc-alp" } -enc-dict = { path = "../enc-dict" } -enc-ffor = { path = "../enc-ffor" } -enc-ree = { path = "../enc-ree" } -enc-roaring = { path = "../enc-roaring" } -enc-zigzag = { path = "../enc-zigzag" } +vortex = { path = "../vortex" } +vortex-alp = { path = "../vortex-alp" } +vortex-dict = { path = "../vortex-dict" } +vortex-ffor = { path = "../vortex-ffor" } +vortex-ree = { path = "../vortex-ree" } +vortex-roaring = { path = "../vortex-roaring" } +vortex-zigzag = { path = "../vortex-zigzag" } itertools = "0.12.1" log = "0.4.20" paste = "1.0.14" diff --git a/pyenc/pyproject.toml b/pyvortex/pyproject.toml similarity index 91% rename from pyenc/pyproject.toml rename to pyvortex/pyproject.toml index 824255a89d..f472a8dbb7 100644 --- a/pyenc/pyproject.toml +++ b/pyvortex/pyproject.toml @@ -1,5 +1,5 @@ [project] -name = "enc" +name = "vortex" version = "0.1.0" description = "Add your description here" authors = [ @@ -22,5 +22,5 @@ dev-dependencies = [ [tool.maturin] python-source = "python" -module-name = "enc._lib" +module-name = "vortex._lib" features = ["pyo3/extension-module"] diff --git a/pyenc/python/enc/__init__.py b/pyvortex/python/vortex/__init__.py similarity index 100% rename from pyenc/python/enc/__init__.py rename to pyvortex/python/vortex/__init__.py diff --git a/pyenc/src/array.rs b/pyvortex/src/array.rs similarity index 84% rename from pyenc/src/array.rs rename to pyvortex/src/array.rs index 1912ca2e54..e04d6060ea 100644 --- a/pyenc/src/array.rs +++ b/pyvortex/src/array.rs @@ -2,28 +2,30 @@ use paste::paste; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; -use enc::array::bool::BoolArray; -use enc::array::chunked::ChunkedArray; -use enc::array::constant::ConstantArray; -use enc::array::primitive::PrimitiveArray; -use enc::array::sparse::SparseArray; -use enc::array::struct_::StructArray; -use enc::array::typed::TypedArray; -use enc::array::varbin::VarBinArray; -use enc::array::varbinview::VarBinViewArray; -use enc::array::{Array, ArrayKind, ArrayRef}; -use enc_alp::{ALPArray, ALP_ENCODING}; -use enc_dict::{DictArray, DICT_ENCODING}; -use enc_ffor::{FFORArray, FFOR_ENCODING}; -use enc_ree::{REEArray, REE_ENCODING}; -use enc_roaring::{RoaringBoolArray, RoaringIntArray, ROARING_BOOL_ENCODING, ROARING_INT_ENCODING}; -use enc_zigzag::{ZigZagArray, ZIGZAG_ENCODING}; +use vortex::array::bool::BoolArray; +use vortex::array::chunked::ChunkedArray; +use vortex::array::constant::ConstantArray; +use vortex::array::primitive::PrimitiveArray; +use vortex::array::sparse::SparseArray; +use vortex::array::struct_::StructArray; +use vortex::array::typed::TypedArray; +use vortex::array::varbin::VarBinArray; +use vortex::array::varbinview::VarBinViewArray; +use vortex::array::{Array, ArrayKind, ArrayRef}; +use vortex_alp::{ALPArray, ALP_ENCODING}; +use vortex_dict::{DictArray, DICT_ENCODING}; +use vortex_ffor::{FFORArray, FFOR_ENCODING}; +use vortex_ree::{REEArray, REE_ENCODING}; +use vortex_roaring::{ + RoaringBoolArray, RoaringIntArray, ROARING_BOOL_ENCODING, ROARING_INT_ENCODING, +}; +use vortex_zigzag::{ZigZagArray, ZIGZAG_ENCODING}; use crate::dtype::PyDType; -use crate::enc_arrow; -use crate::error::PyEncError; +use crate::error::PyVortexError; +use crate::vortex_arrow; -#[pyclass(name = "Array", module = "enc", sequence, subclass)] +#[pyclass(name = "Array", module = "vortex", sequence, subclass)] pub struct PyArray { inner: ArrayRef, } @@ -31,7 +33,7 @@ pub struct PyArray { macro_rules! pyarray { ($T:ident, $TName:tt) => { paste! { - #[pyclass(name = $TName, module = "enc", extends = PyArray, sequence, subclass)] + #[pyclass(name = $TName, module = "vortex", extends = PyArray, sequence, subclass)] pub struct [] { inner: $T, } @@ -159,7 +161,7 @@ impl PyArray { #[pymethods] impl PyArray { fn to_pyarrow(self_: PyRef<'_, Self>) -> PyResult<&PyAny> { - enc_arrow::export_array(self_.py(), &self_.inner) + vortex_arrow::export_array(self_.py(), &self_.inner) } fn __len__(&self) -> usize { @@ -191,7 +193,7 @@ impl PyRoaringBoolArray { #[staticmethod] fn encode(array: PyRef<'_, PyArray>) -> PyResult> { RoaringBoolArray::encode(array.unwrap()) - .map_err(PyEncError::map_err) + .map_err(PyVortexError::map_err) .and_then(|zarray| PyArray::wrap(array.py(), zarray.boxed())) } } @@ -201,7 +203,7 @@ impl PyRoaringIntArray { #[staticmethod] fn encode(array: PyRef<'_, PyArray>) -> PyResult> { RoaringIntArray::encode(array.unwrap()) - .map_err(PyEncError::map_err) + .map_err(PyVortexError::map_err) .and_then(|zarray| PyArray::wrap(array.py(), zarray.boxed())) } } @@ -211,7 +213,7 @@ impl PyZigZagArray { #[staticmethod] fn encode(array: PyRef<'_, PyArray>) -> PyResult> { ZigZagArray::encode(array.unwrap()) - .map_err(PyEncError::map_err) + .map_err(PyVortexError::map_err) .and_then(|zarray| PyArray::wrap(array.py(), zarray)) } } diff --git a/pyenc/src/compress.rs b/pyvortex/src/compress.rs similarity index 88% rename from pyenc/src/compress.rs rename to pyvortex/src/compress.rs index 0ecb8061e1..46833c9d5a 100644 --- a/pyenc/src/compress.rs +++ b/pyvortex/src/compress.rs @@ -1,12 +1,12 @@ use pyo3::types::PyType; use pyo3::{pyclass, pyfunction, pymethods, Py, PyResult, Python}; -use enc::compress::{CompressConfig, CompressCtx}; +use vortex::compress::{CompressConfig, CompressCtx}; use crate::array::PyArray; #[derive(Clone)] -#[pyclass(name = "CompressConfig", module = "enc")] +#[pyclass(name = "CompressConfig", module = "vortex")] pub struct PyCompressConfig { inner: CompressConfig, } diff --git a/pyenc/src/dtype.rs b/pyvortex/src/dtype.rs similarity index 81% rename from pyenc/src/dtype.rs rename to pyvortex/src/dtype.rs index 7f724e0d52..14f76243fc 100644 --- a/pyenc/src/dtype.rs +++ b/pyvortex/src/dtype.rs @@ -1,14 +1,14 @@ use arrow::datatypes::DataType; use arrow::pyarrow::FromPyArrow; -use enc::arrow::convert::TryIntoDType; use pyo3::types::PyType; use pyo3::{pyclass, pymethods, Py, PyAny, PyResult, Python}; +use vortex::arrow::convert::TryIntoDType; -use enc::dtype::DType; +use vortex::dtype::DType; -use crate::error::PyEncError; +use crate::error::PyVortexError; -#[pyclass(name = "DType", module = "enc", subclass)] +#[pyclass(name = "DType", module = "vortex", subclass)] pub struct PyDType { inner: DType, } @@ -39,7 +39,7 @@ impl PyDType { cls.py(), arrow_dtype .try_into_dtype(nullable) - .map_err(PyEncError::new)?, + .map_err(PyVortexError::new)?, ) } } diff --git a/pyenc/src/encode.rs b/pyvortex/src/encode.rs similarity index 84% rename from pyenc/src/encode.rs rename to pyvortex/src/encode.rs index a16ade5c90..2604cec8dd 100644 --- a/pyenc/src/encode.rs +++ b/pyvortex/src/encode.rs @@ -6,14 +6,14 @@ use arrow::record_batch::RecordBatchReader; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; -use enc::array::chunked::ChunkedArray; -use enc::array::{Array, ArrayRef}; -use enc::arrow::convert::TryIntoDType; -use enc::dtype::DType; +use vortex::array::chunked::ChunkedArray; +use vortex::array::{Array, ArrayRef}; +use vortex::arrow::convert::TryIntoDType; +use vortex::dtype::DType; use crate::array::PyArray; -use crate::enc_arrow::map_arrow_err; -use crate::error::PyEncError; +use crate::error::PyVortexError; +use crate::vortex_arrow::map_arrow_err; /// The main entry point for creating enc arrays from other Python objects. /// @@ -26,7 +26,7 @@ pub fn encode(obj: &PyAny) -> PyResult> { if obj.is_instance(pa_array)? { let arrow_array = ArrayData::from_pyarrow(obj).map(make_array)?; - let enc_array: enc::array::ArrayRef = arrow_array.into(); + let enc_array: vortex::array::ArrayRef = arrow_array.into(); PyArray::wrap(obj.py(), enc_array) } else if obj.is_instance(chunked_array)? { let chunks: Vec<&PyAny> = obj.getattr("chunks")?.extract()?; @@ -43,11 +43,11 @@ pub fn encode(obj: &PyAny) -> PyResult> { .getattr("type") .and_then(DataType::from_pyarrow)? .try_into_dtype(null_count > 0) - .map_err(PyEncError::map_err)?; + .map_err(PyVortexError::map_err)?; PyArray::wrap(obj.py(), ChunkedArray::new(encoded_chunks, dtype).boxed()) } else if obj.is_instance(table)? { let array_stream = ArrowArrayStreamReader::from_pyarrow(obj)?; - let dtype = DType::try_from(array_stream.schema()).map_err(PyEncError::map_err)?; + let dtype = DType::try_from(array_stream.schema()).map_err(PyVortexError::map_err)?; let chunks = array_stream .into_iter() .map(|b| b.map(ArrayRef::from).map_err(map_arrow_err)) diff --git a/pyvortex/src/error.rs b/pyvortex/src/error.rs new file mode 100644 index 0000000000..39daec11fa --- /dev/null +++ b/pyvortex/src/error.rs @@ -0,0 +1,22 @@ +use pyo3::exceptions::PyValueError; +use pyo3::PyErr; + +use vortex::error::VortexError; + +pub struct PyVortexError(VortexError); + +impl PyVortexError { + pub fn new(error: VortexError) -> Self { + Self(error) + } + + pub fn map_err(error: VortexError) -> PyErr { + PyVortexError::new(error).into() + } +} + +impl From for PyErr { + fn from(value: PyVortexError) -> Self { + PyValueError::new_err(value.0.to_string()) + } +} diff --git a/pyenc/src/lib.rs b/pyvortex/src/lib.rs similarity index 97% rename from pyenc/src/lib.rs rename to pyvortex/src/lib.rs index 78db0f31d1..c57674f93a 100644 --- a/pyenc/src/lib.rs +++ b/pyvortex/src/lib.rs @@ -2,7 +2,7 @@ use log::debug; use pyo3::prelude::*; use dtype::PyDType; -use enc::dtype::DType; +use vortex::dtype::DType; use crate::array::*; use crate::compress::PyCompressConfig; @@ -10,10 +10,10 @@ use crate::compress::PyCompressConfig; mod array; mod compress; mod dtype; -mod enc_arrow; mod encode; mod error; mod serde; +mod vortex_arrow; /// A Python module implemented in Rust. #[pymodule] @@ -22,7 +22,7 @@ fn _lib(_py: Python, m: &PyModule) -> PyResult<()> { debug!( "Discovered encodings: {:?}", - enc::array::ENCODINGS + vortex::array::ENCODINGS .iter() .map(|e| e.id().to_string()) .collect::>() diff --git a/pyenc/src/serde.rs b/pyvortex/src/serde.rs similarity index 98% rename from pyenc/src/serde.rs rename to pyvortex/src/serde.rs index 00e5d9540c..d5f47bc165 100644 --- a/pyenc/src/serde.rs +++ b/pyvortex/src/serde.rs @@ -4,7 +4,7 @@ use std::io::{ErrorKind, Read, Write}; use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::{ffi, pyfunction, FromPyPointer, IntoPy, Py, PyAny, PyResult, Python}; -use enc::serde::{ReadCtx, WriteCtx}; +use vortex::serde::{ReadCtx, WriteCtx}; use crate::array::PyArray; use crate::dtype::PyDType; diff --git a/pyenc/src/enc_arrow.rs b/pyvortex/src/vortex_arrow.rs similarity index 97% rename from pyenc/src/enc_arrow.rs rename to pyvortex/src/vortex_arrow.rs index 1fb3a50c4c..4f9fb44031 100644 --- a/pyenc/src/enc_arrow.rs +++ b/pyvortex/src/vortex_arrow.rs @@ -6,7 +6,7 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::types::{IntoPyDict, PyList}; -use enc::array::Array; +use vortex::array::Array; pub fn map_arrow_err(error: ArrowError) -> PyErr { PyValueError::new_err(error.to_string()) diff --git a/pyenc/test/conftest.py b/pyvortex/test/conftest.py similarity index 100% rename from pyenc/test/conftest.py rename to pyvortex/test/conftest.py diff --git a/pyenc/test/test_array.py b/pyvortex/test/test_array.py similarity index 68% rename from pyenc/test/test_array.py rename to pyvortex/test/test_array.py index 5686b4ac0c..9c24d217e9 100644 --- a/pyenc/test/test_array.py +++ b/pyvortex/test/test_array.py @@ -1,32 +1,32 @@ import pyarrow as pa import pytest -import enc +import vortex def test_primitive_array_round_trip(): a = pa.array([0, 1, 2, 3]) - arr = enc.encode(a) - assert isinstance(arr, enc.PrimitiveArray) + arr = vortex.encode(a) + assert isinstance(arr, vortex.PrimitiveArray) assert arr.to_pyarrow().combine_chunks() == a def test_varbin_array_round_trip(): a = pa.array(["a", "b", "c"]) - arr = enc.encode(a) - assert isinstance(arr, enc.VarBinArray) + arr = vortex.encode(a) + assert isinstance(arr, vortex.VarBinArray) assert arr.to_pyarrow().combine_chunks() == a @pytest.mark.xfail(strict=True) def test_varbin_array_doesnt_round_trip(): a = pa.array(["a", "b", "c"], type=pa.large_utf8()) - arr = enc.encode(a) - assert isinstance(arr, enc.VarBinArray) + arr = vortex.encode(a) + assert isinstance(arr, vortex.VarBinArray) assert arr.to_pyarrow().combine_chunks() == a def test_empty_array(): a = pa.array([], type=pa.uint8()) - primitive = enc.encode(a) + primitive = vortex.encode(a) assert primitive.to_pyarrow().type == pa.uint8() diff --git a/pyenc/test/test_compress.py b/pyvortex/test/test_compress.py similarity index 56% rename from pyenc/test/test_compress.py rename to pyvortex/test/test_compress.py index fb94d95a6b..272b1abfc7 100644 --- a/pyenc/test/test_compress.py +++ b/pyvortex/test/test_compress.py @@ -1,48 +1,48 @@ import numpy as np import pyarrow as pa -import enc +import vortex def test_primitive_compress(): a = pa.array([0, 0, 0, 0, 9, 9, 9, 9, 1, 5]) - arr_compressed = enc.compress(enc.encode(a)) - assert not isinstance(arr_compressed, enc.PrimitiveArray) + arr_compressed = vortex.compress(vortex.encode(a)) + assert not isinstance(arr_compressed, vortex.PrimitiveArray) assert arr_compressed.nbytes < a.nbytes def test_bool_compress(): - a = enc.encode(pa.array([False] * 10_000 + [True] * 10_000)) - arr_compressed = enc.compress(a) + a = vortex.encode(pa.array([False] * 10_000 + [True] * 10_000)) + arr_compressed = vortex.compress(a) assert len(arr_compressed) == 20_000 - assert isinstance(arr_compressed, enc.RoaringBoolArray) + assert isinstance(arr_compressed, vortex.RoaringBoolArray) assert arr_compressed.nbytes < a.nbytes def test_roaring_bool_encode(): - a = enc.encode(pa.array([True] * 10_000)) - rarr = enc.RoaringBoolArray.encode(a) - assert isinstance(rarr, enc.RoaringBoolArray) + a = vortex.encode(pa.array([True] * 10_000)) + rarr = vortex.RoaringBoolArray.encode(a) + assert isinstance(rarr, vortex.RoaringBoolArray) assert rarr.nbytes < a.nbytes def test_roaring_int_encode(): - a = enc.encode(pa.array(np.arange(10_000), type=pa.uint32())) - compressed = enc.compress(a) + a = vortex.encode(pa.array(np.arange(10_000), type=pa.uint32())) + compressed = vortex.compress(a) assert compressed.encoding == "roaring.int" def test_zigzag_encode(): - a = enc.encode(pa.array([-1, -1, 0, -1, 1, -1])) - zarr = enc.ZigZagArray.encode(a) - assert isinstance(zarr, enc.ZigZagArray) + a = vortex.encode(pa.array([-1, -1, 0, -1, 1, -1])) + zarr = vortex.ZigZagArray.encode(a) + assert isinstance(zarr, vortex.ZigZagArray) # TODO(ngates): support decoding once we have decompressor. def test_chunked_encode(): chunked = pa.chunked_array([pa.array([0, 1, 2]), pa.array([3, 4, 5])]) - encoded = enc.encode(chunked) - assert isinstance(encoded, enc.ChunkedArray) + encoded = vortex.encode(chunked) + assert isinstance(encoded, vortex.ChunkedArray) assert encoded.to_pyarrow().combine_chunks() == pa.array([0, 1, 2, 3, 4, 5]) @@ -53,8 +53,8 @@ def test_table_encode(): "string": pa.chunked_array([pa.array(["a", "b", "c"]), pa.array(["d", "e", "f"])]), } ) - encoded = enc.encode(table) - assert isinstance(encoded, enc.ChunkedArray) + encoded = vortex.encode(table) + assert isinstance(encoded, vortex.ChunkedArray) assert encoded.to_pyarrow().combine_chunks() == pa.StructArray.from_arrays( [pa.array([0, 1, 2, 3, 4, 5]), pa.array(["a", "b", "c", "d", "e", "f"])], names=["number", "string"] ) diff --git a/pyvortex/test/test_dtype.py b/pyvortex/test/test_dtype.py new file mode 100644 index 0000000000..e8761f34b4 --- /dev/null +++ b/pyvortex/test/test_dtype.py @@ -0,0 +1,9 @@ +import vortex + + +def test_int(): + assert str(vortex.int()) == "signed_int(_)" + assert str(vortex.int(32)) == "signed_int(32)" + assert str(vortex.int(32, signed=False)) == "unsigned_int(32)" + assert str(vortex.float(16)) == "float(16)" + assert str(vortex.bool(nullable=True)) == "bool?" diff --git a/pyenc/test/test_serde.py b/pyvortex/test/test_serde.py similarity index 59% rename from pyenc/test/test_serde.py rename to pyvortex/test/test_serde.py index c30a5dd208..9a05a67a4a 100644 --- a/pyenc/test/test_serde.py +++ b/pyvortex/test/test_serde.py @@ -1,19 +1,19 @@ import pyarrow as pa from pyarrow import fs -import enc +import vortex local = fs.LocalFileSystem() def test_serde(tmp_path): a = pa.array([0, 1, 2, 3]) - arr = enc.encode(a) - assert isinstance(arr, enc.PrimitiveArray) + arr = vortex.encode(a) + assert isinstance(arr, vortex.PrimitiveArray) subfs = fs.SubTreeFileSystem(str(tmp_path), local) with subfs.open_output_stream("array.enc", buffer_size=8192) as nf: - enc.write(arr, nf) + vortex.write(arr, nf) with subfs.open_input_stream("array.enc", buffer_size=8192) as nf: - read_array = enc.read(arr.dtype, nf) - assert isinstance(read_array, enc.PrimitiveArray) + read_array = vortex.read(arr.dtype, nf) + assert isinstance(read_array, vortex.PrimitiveArray) diff --git a/requirements-dev.lock b/requirements-dev.lock index 10fa1792c2..ccda876e09 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -7,7 +7,7 @@ # all-features: false # with-sources: false --e file:pyenc +-e file:pyvortex -e file:. babel==2.14.0 bracex==2.4 diff --git a/requirements.lock b/requirements.lock index 19c3f6aeb7..8f59ff5e9c 100644 --- a/requirements.lock +++ b/requirements.lock @@ -7,5 +7,5 @@ # all-features: false # with-sources: false --e file:pyenc +-e file:pyvortex -e file:. diff --git a/spiral-alloc/Cargo.toml b/vortex-alloc/Cargo.toml similarity index 90% rename from spiral-alloc/Cargo.toml rename to vortex-alloc/Cargo.toml index 50bec61c3d..9f7e2f70ba 100644 --- a/spiral-alloc/Cargo.toml +++ b/vortex-alloc/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "spiral-alloc" +name = "vortex-alloc" version = "0.1.0" edition = "2021" diff --git a/spiral-alloc/src/alloc.rs b/vortex-alloc/src/alloc.rs similarity index 86% rename from spiral-alloc/src/alloc.rs rename to vortex-alloc/src/alloc.rs index 7d018c926f..9c319e7908 100644 --- a/spiral-alloc/src/alloc.rs +++ b/vortex-alloc/src/alloc.rs @@ -1,6 +1,6 @@ use core::ptr::NonNull; -use super::SPIRAL_ALIGNMENT; +use super::VORTEX_ALIGNMENT; use allocator_api2::alloc::*; use allocator_api2::vec::Vec; @@ -11,9 +11,9 @@ pub struct AlignedAllocator { impl AlignedAllocator { pub const fn with_default_alignment() -> Self { - assert!(SPIRAL_ALIGNMENT.is_power_of_two()); + assert!(VORTEX_ALIGNMENT.is_power_of_two()); Self { - min_alignment: SPIRAL_ALIGNMENT, + min_alignment: VORTEX_ALIGNMENT, } } @@ -55,8 +55,8 @@ mod test { #[test] fn test_alignment() { let alloc = AlignedAllocator::with_default_alignment(); - assert_eq!(SPIRAL_ALIGNMENT, 128); - assert_eq!(alloc.min_alignment, SPIRAL_ALIGNMENT); + assert_eq!(VORTEX_ALIGNMENT, 128); + assert_eq!(alloc.min_alignment, VORTEX_ALIGNMENT); let ptr = alloc .allocate(core::alloc::Layout::new::()) diff --git a/spiral-alloc/src/lib.rs b/vortex-alloc/src/lib.rs similarity index 75% rename from spiral-alloc/src/lib.rs rename to vortex-alloc/src/lib.rs index 22a044c723..623d7a7ec5 100644 --- a/spiral-alloc/src/lib.rs +++ b/vortex-alloc/src/lib.rs @@ -2,5 +2,5 @@ mod alloc; pub use alloc::*; -pub const SPIRAL_ALIGNMENT: usize = 128; +pub const VORTEX_ALIGNMENT: usize = 128; pub const ALIGNED_ALLOCATOR: AlignedAllocator = AlignedAllocator::with_default_alignment(); diff --git a/enc-alp/Cargo.toml b/vortex-alp/Cargo.toml similarity index 85% rename from enc-alp/Cargo.toml rename to vortex-alp/Cargo.toml index a40a674d07..45a42bc1db 100644 --- a/enc-alp/Cargo.toml +++ b/vortex-alp/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "enc-alp" +name = "vortex-alp" version = "0.1.0" edition = "2021" @@ -7,7 +7,7 @@ edition = "2021" [dependencies] arrow = { version = "50.0.0" } -enc = { "path" = "../enc" } +vortex = { "path" = "../vortex" } linkme = "0.3.22" itertools = "0.12.1" codecz = { version = "0.1.0", path = "../codecz" } diff --git a/enc-alp/src/alp.rs b/vortex-alp/src/alp.rs similarity index 81% rename from enc-alp/src/alp.rs rename to vortex-alp/src/alp.rs index 1a3d5ff34a..86e32fceda 100644 --- a/enc-alp/src/alp.rs +++ b/vortex-alp/src/alp.rs @@ -3,14 +3,14 @@ use std::sync::{Arc, RwLock}; use codecz::alp; pub use codecz::alp::ALPExponents; -use enc::array::{Array, ArrayKind, ArrayRef, ArrowIterator, Encoding, EncodingId, EncodingRef}; -use enc::compress::EncodingCompression; -use enc::dtype::{DType, FloatWidth, IntWidth, Signedness}; -use enc::error::{EncError, EncResult}; -use enc::formatter::{ArrayDisplay, ArrayFormatter}; -use enc::scalar::{NullableScalar, Scalar}; -use enc::serde::{ArraySerde, EncodingSerde}; -use enc::stats::{Stats, StatsSet}; +use vortex::array::{Array, ArrayKind, ArrayRef, ArrowIterator, Encoding, EncodingId, EncodingRef}; +use vortex::compress::EncodingCompression; +use vortex::dtype::{DType, FloatWidth, IntWidth, Signedness}; +use vortex::error::{VortexError, VortexResult}; +use vortex::formatter::{ArrayDisplay, ArrayFormatter}; +use vortex::scalar::{NullableScalar, Scalar}; +use vortex::serde::{ArraySerde, EncodingSerde}; +use vortex::stats::{Stats, StatsSet}; use crate::compress::alp_encode; @@ -32,14 +32,14 @@ impl ALPArray { encoded: ArrayRef, exponents: ALPExponents, patches: Option, - ) -> EncResult { + ) -> VortexResult { let dtype = match encoded.dtype() { d @ DType::Int(width, Signedness::Signed, nullability) => match width { IntWidth::_32 => DType::Float(32.into(), *nullability), IntWidth::_64 => DType::Float(64.into(), *nullability), - _ => return Err(EncError::InvalidDType(d.clone())), + _ => return Err(VortexError::InvalidDType(d.clone())), }, - d => return Err(EncError::InvalidDType(d.clone())), + d => return Err(VortexError::InvalidDType(d.clone())), }; Ok(Self { encoded, @@ -50,10 +50,10 @@ impl ALPArray { }) } - pub fn encode(array: &dyn Array) -> EncResult { + pub fn encode(array: &dyn Array) -> VortexResult { match ArrayKind::from(array) { ArrayKind::Primitive(p) => Ok(alp_encode(p).boxed()), - _ => Err(EncError::InvalidEncoding(array.encoding().id().clone())), + _ => Err(VortexError::InvalidEncoding(array.encoding().id().clone())), } } @@ -106,7 +106,7 @@ impl Array for ALPArray { Stats::new(&self.stats, self) } - fn scalar_at(&self, index: usize) -> EncResult> { + fn scalar_at(&self, index: usize) -> VortexResult> { if let Some(patch) = self .patches() .and_then(|p| p.scalar_at(index).ok()) @@ -141,7 +141,7 @@ impl Array for ALPArray { todo!() } - fn slice(&self, start: usize, stop: usize) -> EncResult { + fn slice(&self, start: usize, stop: usize) -> VortexResult { Ok(Self::try_new( self.encoded().slice(start, stop)?, self.exponents(), @@ -185,7 +185,7 @@ impl ArrayDisplay for ALPArray { #[derive(Debug)] pub struct ALPEncoding; -pub const ALP_ENCODING: EncodingId = EncodingId::new("enc.alp"); +pub const ALP_ENCODING: EncodingId = EncodingId::new("vortex.alp"); impl Encoding for ALPEncoding { fn id(&self) -> &EncodingId { diff --git a/enc-alp/src/compress.rs b/vortex-alp/src/compress.rs similarity index 93% rename from enc-alp/src/compress.rs rename to vortex-alp/src/compress.rs index 47865f1a47..182cef2a4a 100644 --- a/enc-alp/src/compress.rs +++ b/vortex-alp/src/compress.rs @@ -2,12 +2,12 @@ use log::debug; use codecz::alp; use codecz::alp::{ALPEncoded, ALPExponents, SupportsALP}; -use enc::array::downcast::DowncastArrayBuiltin; -use enc::array::primitive::PrimitiveArray; -use enc::array::sparse::SparseArray; -use enc::array::{Array, ArrayRef}; -use enc::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression}; -use enc::ptype::{NativePType, PType}; +use vortex::array::downcast::DowncastArrayBuiltin; +use vortex::array::primitive::PrimitiveArray; +use vortex::array::sparse::SparseArray; +use vortex::array::{Array, ArrayRef}; +use vortex::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression}; +use vortex::ptype::{NativePType, PType}; use crate::alp::{ALPArray, ALPEncoding}; use crate::downcast::DowncastALP; diff --git a/enc-alp/src/downcast.rs b/vortex-alp/src/downcast.rs similarity index 93% rename from enc-alp/src/downcast.rs rename to vortex-alp/src/downcast.rs index 0e74899eba..4bbf6b01bd 100644 --- a/enc-alp/src/downcast.rs +++ b/vortex-alp/src/downcast.rs @@ -1,4 +1,4 @@ -use enc::array::{Array, ArrayRef}; +use vortex::array::{Array, ArrayRef}; use crate::ALPArray; diff --git a/enc-alp/src/lib.rs b/vortex-alp/src/lib.rs similarity index 80% rename from enc-alp/src/lib.rs rename to vortex-alp/src/lib.rs index aa3620ce24..6f0fe3ae42 100644 --- a/enc-alp/src/lib.rs +++ b/vortex-alp/src/lib.rs @@ -1,6 +1,6 @@ pub use alp::*; -use enc::array::{EncodingRef, ENCODINGS}; use linkme::distributed_slice; +use vortex::array::{EncodingRef, ENCODINGS}; mod alp; mod compress; diff --git a/enc-alp/src/serde.rs b/vortex-alp/src/serde.rs similarity index 88% rename from enc-alp/src/serde.rs rename to vortex-alp/src/serde.rs index 3f3af01957..58ef3e090e 100644 --- a/enc-alp/src/serde.rs +++ b/vortex-alp/src/serde.rs @@ -2,9 +2,9 @@ use std::io; use std::io::ErrorKind; use codecz::alp::ALPExponents; -use enc::array::{Array, ArrayRef}; -use enc::dtype::{DType, FloatWidth, Signedness}; -use enc::serde::{ArraySerde, EncodingSerde, ReadCtx, WriteCtx}; +use vortex::array::{Array, ArrayRef}; +use vortex::dtype::{DType, FloatWidth, Signedness}; +use vortex::serde::{ArraySerde, EncodingSerde, ReadCtx, WriteCtx}; use crate::{ALPArray, ALPEncoding}; @@ -53,10 +53,10 @@ impl EncodingSerde for ALPEncoding { mod test { use std::io; - use enc::array::downcast::DowncastArrayBuiltin; - use enc::array::primitive::PrimitiveArray; - use enc::array::{Array, ArrayRef}; - use enc::serde::{ReadCtx, WriteCtx}; + use vortex::array::downcast::DowncastArrayBuiltin; + use vortex::array::primitive::PrimitiveArray; + use vortex::array::{Array, ArrayRef}; + use vortex::serde::{ReadCtx, WriteCtx}; use crate::compress::alp_encode; use crate::downcast::DowncastALP; diff --git a/enc-alp/src/stats.rs b/vortex-alp/src/stats.rs similarity index 82% rename from enc-alp/src/stats.rs rename to vortex-alp/src/stats.rs index ee170557f1..f19d5a6e81 100644 --- a/enc-alp/src/stats.rs +++ b/vortex-alp/src/stats.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use crate::ALPArray; -use enc::stats::{Stat, StatsCompute, StatsSet}; +use vortex::stats::{Stat, StatsCompute, StatsSet}; impl StatsCompute for ALPArray { fn compute(&self, _stat: &Stat) -> StatsSet { diff --git a/enc-dict/Cargo.toml b/vortex-dict/Cargo.toml similarity index 79% rename from enc-dict/Cargo.toml rename to vortex-dict/Cargo.toml index a7517d32cd..f57a8c780c 100644 --- a/enc-dict/Cargo.toml +++ b/vortex-dict/Cargo.toml @@ -1,11 +1,11 @@ [package] -name = "enc-dict" +name = "vortex-dict" version = "0.1.0" edition = "2021" [dependencies] ahash = "0.8.7" -enc = { "path" = "../enc" } +vortex = { "path" = "../vortex" } half = { version = "2.3.1", features = ["std", "num-traits"] } hashbrown = "0.14.3" linkme = "0.3.22" diff --git a/enc-dict/src/compress.rs b/vortex-dict/src/compress.rs similarity index 95% rename from enc-dict/src/compress.rs rename to vortex-dict/src/compress.rs index 2072d8d479..257c9df053 100644 --- a/enc-dict/src/compress.rs +++ b/vortex-dict/src/compress.rs @@ -6,15 +6,15 @@ use hashbrown::HashMap; use log::debug; use num_traits::{AsPrimitive, FromPrimitive, Unsigned}; -use enc::array::downcast::DowncastArrayBuiltin; -use enc::array::primitive::PrimitiveArray; -use enc::array::varbin::VarBinArray; -use enc::array::{Array, ArrayKind, ArrayRef}; -use enc::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression}; -use enc::dtype::DType; -use enc::match_each_native_ptype; -use enc::ptype::NativePType; -use enc::scalar::AsBytes; +use vortex::array::downcast::DowncastArrayBuiltin; +use vortex::array::primitive::PrimitiveArray; +use vortex::array::varbin::VarBinArray; +use vortex::array::{Array, ArrayKind, ArrayRef}; +use vortex::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression}; +use vortex::dtype::DType; +use vortex::match_each_native_ptype; +use vortex::ptype::NativePType; +use vortex::scalar::AsBytes; use crate::dict::{DictArray, DictEncoding}; use crate::downcast::DowncastDict; @@ -266,9 +266,9 @@ where #[cfg(test)] mod test { - use enc::array::primitive::PrimitiveArray; - use enc::array::varbin::VarBinArray; - use enc::array::Array; + use vortex::array::primitive::PrimitiveArray; + use vortex::array::varbin::VarBinArray; + use vortex::array::Array; use crate::compress::{dict_encode_typed_primitive, dict_encode_varbin}; diff --git a/enc-dict/src/dict.rs b/vortex-dict/src/dict.rs similarity index 79% rename from enc-dict/src/dict.rs rename to vortex-dict/src/dict.rs index 85128d7178..0a96198833 100644 --- a/enc-dict/src/dict.rs +++ b/vortex-dict/src/dict.rs @@ -1,16 +1,16 @@ use std::any::Any; use std::sync::{Arc, RwLock}; -use enc::array::{ +use vortex::array::{ check_index_bounds, check_slice_bounds, Array, ArrayRef, ArrowIterator, Encoding, EncodingId, }; -use enc::compress::EncodingCompression; -use enc::dtype::{DType, Signedness}; -use enc::error::{EncError, EncResult}; -use enc::formatter::{ArrayDisplay, ArrayFormatter}; -use enc::scalar::Scalar; -use enc::serde::{ArraySerde, EncodingSerde}; -use enc::stats::{Stats, StatsSet}; +use vortex::compress::EncodingCompression; +use vortex::dtype::{DType, Signedness}; +use vortex::error::{VortexError, VortexResult}; +use vortex::formatter::{ArrayDisplay, ArrayFormatter}; +use vortex::scalar::Scalar; +use vortex::serde::{ArraySerde, EncodingSerde}; +use vortex::stats::{Stats, StatsSet}; #[derive(Debug, Clone)] pub struct DictArray { @@ -24,9 +24,9 @@ impl DictArray { Self::try_new(codes, dict).unwrap() } - pub fn try_new(codes: ArrayRef, dict: ArrayRef) -> EncResult { + pub fn try_new(codes: ArrayRef, dict: ArrayRef) -> VortexResult { if !matches!(codes.dtype(), DType::Int(_, Signedness::Unsigned, _)) { - return Err(EncError::InvalidDType(codes.dtype().clone())); + return Err(VortexError::InvalidDType(codes.dtype().clone())); } Ok(Self { codes, @@ -75,7 +75,7 @@ impl Array for DictArray { Stats::new(&self.stats, self) } - fn scalar_at(&self, index: usize) -> EncResult> { + fn scalar_at(&self, index: usize) -> VortexResult> { check_index_bounds(self, index)?; let dict_index: usize = self.codes().scalar_at(index)?.try_into()?; self.dict().scalar_at(dict_index) @@ -86,7 +86,7 @@ impl Array for DictArray { } // TODO(robert): Add function to trim the dictionary - fn slice(&self, start: usize, stop: usize) -> EncResult { + fn slice(&self, start: usize, stop: usize) -> VortexResult { check_slice_bounds(self, start, stop)?; Ok(Self::new(self.codes().slice(start, stop)?, self.dict.clone()).boxed()) } @@ -122,7 +122,7 @@ impl ArrayDisplay for DictArray { #[derive(Debug)] pub struct DictEncoding; -pub const DICT_ENCODING: EncodingId = EncodingId::new("enc.dict"); +pub const DICT_ENCODING: EncodingId = EncodingId::new("vortex.dict"); impl Encoding for DictEncoding { fn id(&self) -> &EncodingId { diff --git a/enc-dict/src/downcast.rs b/vortex-dict/src/downcast.rs similarity index 93% rename from enc-dict/src/downcast.rs rename to vortex-dict/src/downcast.rs index a66202cf7b..5e9614e455 100644 --- a/enc-dict/src/downcast.rs +++ b/vortex-dict/src/downcast.rs @@ -1,4 +1,4 @@ -use enc::array::{Array, ArrayRef}; +use vortex::array::{Array, ArrayRef}; use crate::DictArray; diff --git a/enc-dict/src/lib.rs b/vortex-dict/src/lib.rs similarity index 82% rename from enc-dict/src/lib.rs rename to vortex-dict/src/lib.rs index 248a049c5c..afef63b7da 100644 --- a/enc-dict/src/lib.rs +++ b/vortex-dict/src/lib.rs @@ -1,5 +1,5 @@ -use enc::array::{EncodingRef, ENCODINGS}; use linkme::distributed_slice; +use vortex::array::{EncodingRef, ENCODINGS}; pub use compress::*; pub use dict::*; diff --git a/enc-dict/src/serde.rs b/vortex-dict/src/serde.rs similarity index 86% rename from enc-dict/src/serde.rs rename to vortex-dict/src/serde.rs index 6900b72169..59ff2aec3d 100644 --- a/enc-dict/src/serde.rs +++ b/vortex-dict/src/serde.rs @@ -1,7 +1,7 @@ use std::io; -use enc::array::{Array, ArrayRef}; -use enc::serde::{ArraySerde, EncodingSerde, ReadCtx, WriteCtx}; +use vortex::array::{Array, ArrayRef}; +use vortex::serde::{ArraySerde, EncodingSerde, ReadCtx, WriteCtx}; use crate::{DictArray, DictEncoding}; @@ -27,10 +27,10 @@ impl EncodingSerde for DictEncoding { mod test { use std::io; - use enc::array::downcast::DowncastArrayBuiltin; - use enc::array::primitive::PrimitiveArray; - use enc::array::{Array, ArrayRef}; - use enc::serde::{ReadCtx, WriteCtx}; + use vortex::array::downcast::DowncastArrayBuiltin; + use vortex::array::primitive::PrimitiveArray; + use vortex::array::{Array, ArrayRef}; + use vortex::serde::{ReadCtx, WriteCtx}; use crate::downcast::DowncastDict; use crate::DictArray; diff --git a/enc-dict/src/stats.rs b/vortex-dict/src/stats.rs similarity index 96% rename from enc-dict/src/stats.rs rename to vortex-dict/src/stats.rs index 7270a5fb15..e8b647b070 100644 --- a/enc-dict/src/stats.rs +++ b/vortex-dict/src/stats.rs @@ -1,4 +1,4 @@ -use enc::stats::{Stat, StatsCompute, StatsSet}; +use vortex::stats::{Stat, StatsCompute, StatsSet}; use crate::dict::DictArray; diff --git a/enc-ffor/Cargo.toml b/vortex-ffor/Cargo.toml similarity index 80% rename from enc-ffor/Cargo.toml rename to vortex-ffor/Cargo.toml index 14ed3b4183..03412b922b 100644 --- a/enc-ffor/Cargo.toml +++ b/vortex-ffor/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "enc-ffor" +name = "vortex-ffor" version = "0.1.0" edition = "2021" @@ -8,7 +8,7 @@ workspace = true [dependencies] arrow = { version = "50.0.0" } -enc = { "path" = "../enc" } +vortex = { "path" = "../vortex" } linkme = "0.3.22" itertools = "0.12.1" codecz = { path = "../codecz" } diff --git a/enc-ffor/src/compress.rs b/vortex-ffor/src/compress.rs similarity index 91% rename from enc-ffor/src/compress.rs rename to vortex-ffor/src/compress.rs index c497bbf3ee..71c6db7c2e 100644 --- a/enc-ffor/src/compress.rs +++ b/vortex-ffor/src/compress.rs @@ -3,15 +3,15 @@ use log::debug; use crate::downcast::DowncastFFOR; use codecz::ffor; use codecz::ffor::{FforEncoded, SupportsFFoR}; -use enc::array::downcast::DowncastArrayBuiltin; -use enc::array::primitive::PrimitiveArray; -use enc::array::sparse::SparseArray; -use enc::array::{Array, ArrayRef}; -use enc::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression}; -use enc::match_each_integer_ptype; -use enc::ptype::NativePType; -use enc::scalar::{ListScalarVec, NullableScalar, Scalar}; -use enc::stats::Stat; +use vortex::array::downcast::DowncastArrayBuiltin; +use vortex::array::primitive::PrimitiveArray; +use vortex::array::sparse::SparseArray; +use vortex::array::{Array, ArrayRef}; +use vortex::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression}; +use vortex::match_each_integer_ptype; +use vortex::ptype::NativePType; +use vortex::scalar::{ListScalarVec, NullableScalar, Scalar}; +use vortex::stats::Stat; use crate::ffor::{FFORArray, FFoREncoding}; diff --git a/enc-ffor/src/downcast.rs b/vortex-ffor/src/downcast.rs similarity index 93% rename from enc-ffor/src/downcast.rs rename to vortex-ffor/src/downcast.rs index 4ee00a1a3f..cb06b72390 100644 --- a/enc-ffor/src/downcast.rs +++ b/vortex-ffor/src/downcast.rs @@ -1,4 +1,4 @@ -use enc::array::{Array, ArrayRef}; +use vortex::array::{Array, ArrayRef}; use crate::FFORArray; diff --git a/enc-ffor/src/ffor.rs b/vortex-ffor/src/ffor.rs similarity index 82% rename from enc-ffor/src/ffor.rs rename to vortex-ffor/src/ffor.rs index 58e5099207..613c945bb1 100644 --- a/enc-ffor/src/ffor.rs +++ b/vortex-ffor/src/ffor.rs @@ -1,19 +1,19 @@ use std::any::Any; use std::sync::{Arc, RwLock}; -use enc::array::downcast::DowncastArrayBuiltin; -use enc::array::{ +use vortex::array::downcast::DowncastArrayBuiltin; +use vortex::array::{ check_validity_buffer, Array, ArrayKind, ArrayRef, ArrowIterator, Encoding, EncodingId, EncodingRef, }; -use enc::compress::EncodingCompression; -use enc::dtype::DType; -use enc::error::{EncError, EncResult}; -use enc::formatter::{ArrayDisplay, ArrayFormatter}; -use enc::match_each_integer_ptype; -use enc::scalar::{NullableScalar, Scalar}; -use enc::serde::{ArraySerde, EncodingSerde}; -use enc::stats::{Stats, StatsSet}; +use vortex::compress::EncodingCompression; +use vortex::dtype::DType; +use vortex::error::{VortexError, VortexResult}; +use vortex::formatter::{ArrayDisplay, ArrayFormatter}; +use vortex::match_each_integer_ptype; +use vortex::scalar::{NullableScalar, Scalar}; +use vortex::serde::{ArraySerde, EncodingSerde}; +use vortex::stats::{Stats, StatsSet}; use crate::compress::ffor_encode; @@ -47,16 +47,16 @@ impl FFORArray { min_val: Box, num_bits: u8, len: usize, - ) -> EncResult { + ) -> VortexResult { let validity = validity.filter(|v| !v.is_empty()); check_validity_buffer(validity.as_ref())?; if !matches!(min_val.dtype(), DType::Int(_, _, _)) { - return Err(EncError::InvalidDType(min_val.dtype().clone())); + return Err(VortexError::InvalidDType(min_val.dtype().clone())); } if validity.is_some() && !min_val.dtype().is_nullable() { - return Err(EncError::InvalidDType(min_val.dtype().clone())); + return Err(VortexError::InvalidDType(min_val.dtype().clone())); } Ok(Self { @@ -70,10 +70,10 @@ impl FFORArray { }) } - pub fn encode(array: &dyn Array) -> EncResult { + pub fn encode(array: &dyn Array) -> VortexResult { match ArrayKind::from(array) { ArrayKind::Primitive(p) => Ok(ffor_encode(p).boxed()), - _ => Err(EncError::InvalidEncoding(array.encoding().id().clone())), + _ => Err(VortexError::InvalidEncoding(array.encoding().id().clone())), } } @@ -145,7 +145,7 @@ impl Array for FFORArray { Stats::new(&self.stats, self) } - fn scalar_at(&self, index: usize) -> EncResult> { + fn scalar_at(&self, index: usize) -> VortexResult> { if !self.is_valid(index) { return Ok(NullableScalar::none(self.dtype().clone()).boxed()); } @@ -159,7 +159,7 @@ impl Array for FFORArray { } let Some(parray) = self.encoded().maybe_primitive() else { - return Err(EncError::InvalidEncoding( + return Err(VortexError::InvalidEncoding( self.encoded().encoding().id().clone(), )); }; @@ -177,7 +177,7 @@ impl Array for FFORArray { .into()); }) } else { - return Err(EncError::InvalidDType(self.dtype().clone())); + return Err(VortexError::InvalidDType(self.dtype().clone())); } } @@ -185,7 +185,7 @@ impl Array for FFORArray { todo!() } - fn slice(&self, _start: usize, _stop: usize) -> EncResult { + fn slice(&self, _start: usize, _stop: usize) -> VortexResult { unimplemented!("FFoRArray::slice") } @@ -228,7 +228,7 @@ impl ArrayDisplay for FFORArray { #[derive(Debug)] pub struct FFoREncoding; -pub const FFOR_ENCODING: EncodingId = EncodingId::new("enc.ffor"); +pub const FFOR_ENCODING: EncodingId = EncodingId::new("vortex.ffor"); impl Encoding for FFoREncoding { fn id(&self) -> &EncodingId { diff --git a/enc-ffor/src/lib.rs b/vortex-ffor/src/lib.rs similarity index 81% rename from enc-ffor/src/lib.rs rename to vortex-ffor/src/lib.rs index 166d213dad..66482fbe64 100644 --- a/enc-ffor/src/lib.rs +++ b/vortex-ffor/src/lib.rs @@ -1,6 +1,6 @@ -use enc::array::{EncodingRef, ENCODINGS}; pub use ffor::*; use linkme::distributed_slice; +use vortex::array::{EncodingRef, ENCODINGS}; mod compress; mod downcast; diff --git a/enc-ffor/src/serde.rs b/vortex-ffor/src/serde.rs similarity index 90% rename from enc-ffor/src/serde.rs rename to vortex-ffor/src/serde.rs index 52e4221cfa..cc55dc5889 100644 --- a/enc-ffor/src/serde.rs +++ b/vortex-ffor/src/serde.rs @@ -1,7 +1,7 @@ use std::io; -use enc::array::{Array, ArrayRef}; -use enc::serde::{ArraySerde, EncodingSerde, ReadCtx, WriteCtx}; +use vortex::array::{Array, ArrayRef}; +use vortex::serde::{ArraySerde, EncodingSerde, ReadCtx, WriteCtx}; use crate::{FFORArray, FFoREncoding}; @@ -52,10 +52,10 @@ impl EncodingSerde for FFoREncoding { mod test { use std::io; - use enc::array::downcast::DowncastArrayBuiltin; - use enc::array::primitive::PrimitiveArray; - use enc::array::{Array, ArrayRef}; - use enc::serde::{ReadCtx, WriteCtx}; + use vortex::array::downcast::DowncastArrayBuiltin; + use vortex::array::primitive::PrimitiveArray; + use vortex::array::{Array, ArrayRef}; + use vortex::serde::{ReadCtx, WriteCtx}; use crate::compress::ffor_encode; use crate::downcast::DowncastFFOR; diff --git a/enc-ffor/src/stats.rs b/vortex-ffor/src/stats.rs similarity index 82% rename from enc-ffor/src/stats.rs rename to vortex-ffor/src/stats.rs index c771107aad..c0372e99aa 100644 --- a/enc-ffor/src/stats.rs +++ b/vortex-ffor/src/stats.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use crate::FFORArray; -use enc::stats::{Stat, StatsCompute, StatsSet}; +use vortex::stats::{Stat, StatsCompute, StatsSet}; impl StatsCompute for FFORArray { fn compute(&self, _stat: &Stat) -> StatsSet { diff --git a/enc-ree/Cargo.toml b/vortex-ree/Cargo.toml similarity index 79% rename from enc-ree/Cargo.toml rename to vortex-ree/Cargo.toml index d24fe7a77a..8a3b57ed00 100644 --- a/enc-ree/Cargo.toml +++ b/vortex-ree/Cargo.toml @@ -1,12 +1,12 @@ [package] -name = "enc-ree" +name = "vortex-ree" version = "0.1.0" edition = "2021" [dependencies] arrow = { version = "50.0.0" } codecz = { path = "../codecz" } -enc = { path = "../enc" } +vortex = { path = "../vortex" } linkme = "0.3.22" half = "2.3.1" num-traits = "0.2.17" diff --git a/enc-ree/src/compress.rs b/vortex-ree/src/compress.rs similarity index 89% rename from enc-ree/src/compress.rs rename to vortex-ree/src/compress.rs index 5b1f4ff5d1..46836951af 100644 --- a/enc-ree/src/compress.rs +++ b/vortex-ree/src/compress.rs @@ -1,11 +1,11 @@ use codecz::AlignedAllocator; -use enc::array::downcast::DowncastArrayBuiltin; -use enc::array::primitive::{PrimitiveArray, PRIMITIVE_ENCODING}; -use enc::array::{Array, ArrayRef}; -use enc::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression}; -use enc::dtype::{DType, IntWidth, Nullability}; -use enc::ptype::match_each_native_ptype; -use enc::stats::Stat; +use vortex::array::downcast::DowncastArrayBuiltin; +use vortex::array::primitive::{PrimitiveArray, PRIMITIVE_ENCODING}; +use vortex::array::{Array, ArrayRef}; +use vortex::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression}; +use vortex::dtype::{DType, IntWidth, Nullability}; +use vortex::ptype::match_each_native_ptype; +use vortex::stats::Stat; use crate::downcast::DowncastREE; use crate::{REEArray, REEEncoding}; @@ -94,9 +94,9 @@ pub fn ree_decode( mod test { use arrow::buffer::BooleanBuffer; - use enc::array::bool::BoolArray; - use enc::array::downcast::DowncastArrayBuiltin; - use enc::array::Array; + use vortex::array::bool::BoolArray; + use vortex::array::downcast::DowncastArrayBuiltin; + use vortex::array::Array; use crate::compress::ree_decode; use crate::REEArray; diff --git a/enc-ree/src/downcast.rs b/vortex-ree/src/downcast.rs similarity index 93% rename from enc-ree/src/downcast.rs rename to vortex-ree/src/downcast.rs index 707b9d4a9e..22818cb18c 100644 --- a/enc-ree/src/downcast.rs +++ b/vortex-ree/src/downcast.rs @@ -1,4 +1,4 @@ -use enc::array::{Array, ArrayRef}; +use vortex::array::{Array, ArrayRef}; use crate::REEArray; diff --git a/enc-ree/src/lib.rs b/vortex-ree/src/lib.rs similarity index 80% rename from enc-ree/src/lib.rs rename to vortex-ree/src/lib.rs index 6917d9ab43..7fecbf9c23 100644 --- a/enc-ree/src/lib.rs +++ b/vortex-ree/src/lib.rs @@ -1,5 +1,5 @@ -use enc::array::{EncodingRef, ENCODINGS}; use linkme::distributed_slice; +use vortex::array::{EncodingRef, ENCODINGS}; pub use ree::*; diff --git a/enc-ree/src/ree.rs b/vortex-ree/src/ree.rs similarity index 87% rename from enc-ree/src/ree.rs rename to vortex-ree/src/ree.rs index 7da307b94f..2150a89f76 100644 --- a/enc-ree/src/ree.rs +++ b/vortex-ree/src/ree.rs @@ -8,22 +8,22 @@ use arrow::array::{Array as ArrowArray, ArrayRef as ArrowArrayRef, AsArray}; use num_traits::AsPrimitive; use codecz::ree::SupportsREE; -use enc::array::primitive::PrimitiveArray; -use enc::array::{ +use vortex::array::primitive::PrimitiveArray; +use vortex::array::{ check_index_bounds, check_slice_bounds, check_validity_buffer, Array, ArrayKind, ArrayRef, ArrowIterator, Encoding, EncodingId, EncodingRef, }; -use enc::arrow::match_arrow_numeric_type; -use enc::compress::EncodingCompression; -use enc::compute; -use enc::compute::search_sorted::SearchSortedSide; -use enc::dtype::{DType, Nullability, Signedness}; -use enc::error::{EncError, EncResult}; -use enc::formatter::{ArrayDisplay, ArrayFormatter}; -use enc::ptype::NativePType; -use enc::scalar::Scalar; -use enc::serde::{ArraySerde, EncodingSerde}; -use enc::stats::{Stat, Stats, StatsSet}; +use vortex::arrow::match_arrow_numeric_type; +use vortex::compress::EncodingCompression; +use vortex::compute; +use vortex::compute::search_sorted::SearchSortedSide; +use vortex::dtype::{DType, Nullability, Signedness}; +use vortex::error::{VortexError, VortexResult}; +use vortex::formatter::{ArrayDisplay, ArrayFormatter}; +use vortex::ptype::NativePType; +use vortex::scalar::Scalar; +use vortex::serde::{ArraySerde, EncodingSerde}; +use vortex::stats::{Stat, Stats, StatsSet}; use crate::compress::ree_encode; @@ -52,14 +52,14 @@ impl REEArray { values: ArrayRef, validity: Option, length: usize, - ) -> EncResult { + ) -> VortexResult { check_validity_buffer(validity.as_ref())?; if !matches!( ends.dtype(), DType::Int(_, Signedness::Unsigned, Nullability::NonNullable) ) { - return Err(EncError::InvalidDType(ends.dtype().clone())); + return Err(VortexError::InvalidDType(ends.dtype().clone())); } if !ends @@ -67,7 +67,7 @@ impl REEArray { .get_as::(&Stat::IsStrictSorted) .unwrap_or(true) { - return Err(EncError::IndexArrayMustBeStrictSorted); + return Err(VortexError::IndexArrayMustBeStrictSorted); } // see https://github.com/fulcrum-so/spiral/issues/873 @@ -82,7 +82,7 @@ impl REEArray { }) } - pub fn find_physical_index(&self, index: usize) -> EncResult { + pub fn find_physical_index(&self, index: usize) -> VortexResult { compute::search_sorted::search_sorted_usize( self.ends(), index + self.offset, @@ -90,7 +90,7 @@ impl REEArray { ) } - pub fn encode(array: &dyn Array) -> EncResult { + pub fn encode(array: &dyn Array) -> VortexResult { match ArrayKind::from(array) { ArrayKind::Primitive(p) => { let (ends, values) = ree_encode(p); @@ -99,7 +99,7 @@ impl REEArray { .boxed(), ) } - _ => Err(EncError::InvalidEncoding(array.encoding().id().clone())), + _ => Err(VortexError::InvalidEncoding(array.encoding().id().clone())), } } @@ -155,7 +155,7 @@ impl Array for REEArray { Stats::new(&self.stats, self) } - fn scalar_at(&self, index: usize) -> EncResult> { + fn scalar_at(&self, index: usize) -> VortexResult> { check_index_bounds(self, index)?; self.values.scalar_at(self.find_physical_index(index)?) } @@ -185,7 +185,7 @@ impl Array for REEArray { }) } - fn slice(&self, start: usize, stop: usize) -> EncResult { + fn slice(&self, start: usize, stop: usize) -> VortexResult { check_slice_bounds(self, start, stop)?; let slice_begin = self.find_physical_index(start)?; let slice_end = self.find_physical_index(stop)?; @@ -229,7 +229,7 @@ impl<'arr> AsRef<(dyn Array + 'arr)> for REEArray { #[derive(Debug)] pub struct REEEncoding; -pub const REE_ENCODING: EncodingId = EncodingId::new("enc.ree"); +pub const REE_ENCODING: EncodingId = EncodingId::new("vortex.ree"); impl Encoding for REEEncoding { fn id(&self) -> &EncodingId { @@ -314,11 +314,11 @@ fn run_ends_logical_length>(ends: &T) -> usize { mod test { use arrow::array::cast::AsArray; use arrow::array::types::Int32Type; - use enc::array::Array; use itertools::Itertools; + use vortex::array::Array; use crate::REEArray; - use enc::dtype::{DType, IntWidth, Nullability, Signedness}; + use vortex::dtype::{DType, IntWidth, Nullability, Signedness}; #[test] fn new() { diff --git a/enc-ree/src/serde.rs b/vortex-ree/src/serde.rs similarity index 87% rename from enc-ree/src/serde.rs rename to vortex-ree/src/serde.rs index b4604b9666..9134ee9060 100644 --- a/enc-ree/src/serde.rs +++ b/vortex-ree/src/serde.rs @@ -1,7 +1,7 @@ use std::io; -use enc::array::{Array, ArrayRef}; -use enc::serde::{ArraySerde, EncodingSerde, ReadCtx, WriteCtx}; +use vortex::array::{Array, ArrayRef}; +use vortex::serde::{ArraySerde, EncodingSerde, ReadCtx, WriteCtx}; use crate::{REEArray, REEEncoding}; @@ -37,10 +37,10 @@ impl EncodingSerde for REEEncoding { mod test { use std::io; - use enc::array::downcast::DowncastArrayBuiltin; - use enc::array::primitive::PrimitiveArray; - use enc::array::{Array, ArrayRef}; - use enc::serde::{ReadCtx, WriteCtx}; + use vortex::array::downcast::DowncastArrayBuiltin; + use vortex::array::primitive::PrimitiveArray; + use vortex::array::{Array, ArrayRef}; + use vortex::serde::{ReadCtx, WriteCtx}; use crate::downcast::DowncastREE; use crate::REEArray; diff --git a/enc-ree/src/stats.rs b/vortex-ree/src/stats.rs similarity index 71% rename from enc-ree/src/stats.rs rename to vortex-ree/src/stats.rs index 99fb0f63a3..b5bda96c3d 100644 --- a/enc-ree/src/stats.rs +++ b/vortex-ree/src/stats.rs @@ -1,5 +1,5 @@ use crate::REEArray; -use enc::stats::{Stat, StatsCompute, StatsSet}; +use vortex::stats::{Stat, StatsCompute, StatsSet}; impl StatsCompute for REEArray { fn compute(&self, _stat: &Stat) -> StatsSet { diff --git a/enc-roaring/Cargo.toml b/vortex-roaring/Cargo.toml similarity index 73% rename from enc-roaring/Cargo.toml rename to vortex-roaring/Cargo.toml index 79f37056e4..7efeff2227 100644 --- a/enc-roaring/Cargo.toml +++ b/vortex-roaring/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "enc-roaring" +name = "vortex-roaring" version = "0.1.0" edition = "2021" @@ -7,7 +7,7 @@ edition = "2021" workspace = true [dependencies] -enc = { "path" = "../enc" } +vortex = { "path" = "../vortex" } linkme = "0.3.22" croaring = "1.0.1" num-traits = "0.2.17" diff --git a/enc-roaring/src/boolean/compress.rs b/vortex-roaring/src/boolean/compress.rs similarity index 80% rename from enc-roaring/src/boolean/compress.rs rename to vortex-roaring/src/boolean/compress.rs index ca961c89a4..de2d8b334a 100644 --- a/enc-roaring/src/boolean/compress.rs +++ b/vortex-roaring/src/boolean/compress.rs @@ -1,11 +1,11 @@ use croaring::Bitmap; -use enc::array::bool::{BoolArray, BOOL_ENCODING}; -use enc::array::downcast::DowncastArrayBuiltin; -use enc::array::{Array, ArrayRef}; -use enc::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression}; -use enc::dtype::DType; -use enc::dtype::Nullability::NonNullable; +use vortex::array::bool::{BoolArray, BOOL_ENCODING}; +use vortex::array::downcast::DowncastArrayBuiltin; +use vortex::array::{Array, ArrayRef}; +use vortex::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression}; +use vortex::dtype::DType; +use vortex::dtype::Nullability::NonNullable; use crate::boolean::{RoaringBoolArray, RoaringBoolEncoding}; diff --git a/enc-roaring/src/boolean/mod.rs b/vortex-roaring/src/boolean/mod.rs similarity index 81% rename from enc-roaring/src/boolean/mod.rs rename to vortex-roaring/src/boolean/mod.rs index b5d68ed45b..49856c882c 100644 --- a/enc-roaring/src/boolean/mod.rs +++ b/vortex-roaring/src/boolean/mod.rs @@ -4,18 +4,18 @@ use std::sync::{Arc, RwLock}; use croaring::{Bitmap, Native}; use compress::roaring_encode; -use enc::array::{ +use vortex::array::{ check_index_bounds, check_slice_bounds, Array, ArrayKind, ArrayRef, ArrowIterator, Encoding, EncodingId, EncodingRef, }; -use enc::compress::EncodingCompression; -use enc::dtype::DType; -use enc::dtype::Nullability::NonNullable; -use enc::error::{EncError, EncResult}; -use enc::formatter::{ArrayDisplay, ArrayFormatter}; -use enc::scalar::Scalar; -use enc::serde::{ArraySerde, EncodingSerde}; -use enc::stats::{Stats, StatsSet}; +use vortex::compress::EncodingCompression; +use vortex::dtype::DType; +use vortex::dtype::Nullability::NonNullable; +use vortex::error::{VortexError, VortexResult}; +use vortex::formatter::{ArrayDisplay, ArrayFormatter}; +use vortex::scalar::Scalar; +use vortex::serde::{ArraySerde, EncodingSerde}; +use vortex::stats::{Stats, StatsSet}; mod compress; mod serde; @@ -41,10 +41,10 @@ impl RoaringBoolArray { &self.bitmap } - pub fn encode(array: &dyn Array) -> EncResult { + pub fn encode(array: &dyn Array) -> VortexResult { match ArrayKind::from(array) { ArrayKind::Bool(p) => Ok(roaring_encode(p)), - _ => Err(EncError::InvalidEncoding(array.encoding().id().clone())), + _ => Err(VortexError::InvalidEncoding(array.encoding().id().clone())), } } } @@ -84,7 +84,7 @@ impl Array for RoaringBoolArray { Stats::new(&self.stats, self) } - fn scalar_at(&self, index: usize) -> EncResult> { + fn scalar_at(&self, index: usize) -> VortexResult> { check_index_bounds(self, index)?; if self.bitmap.contains(index as u32) { @@ -98,7 +98,7 @@ impl Array for RoaringBoolArray { todo!() } - fn slice(&self, start: usize, stop: usize) -> EncResult { + fn slice(&self, start: usize, stop: usize) -> VortexResult { check_slice_bounds(self, start, stop)?; let slice_bitmap = Bitmap::from_range(start as u32..stop as u32); @@ -161,15 +161,15 @@ impl Encoding for RoaringBoolEncoding { #[cfg(test)] mod test { - use enc::array::bool::BoolArray; - use enc::array::Array; - use enc::error::EncResult; - use enc::scalar::Scalar; + use vortex::array::bool::BoolArray; + use vortex::array::Array; + use vortex::error::VortexResult; + use vortex::scalar::Scalar; use crate::RoaringBoolArray; #[test] - pub fn iter() -> EncResult<()> { + pub fn iter() -> VortexResult<()> { let bool: &dyn Array = &BoolArray::from(vec![true, false, true, true]); let array = RoaringBoolArray::encode(bool)?; @@ -180,7 +180,7 @@ mod test { } #[test] - pub fn scalar_at() -> EncResult<()> { + pub fn scalar_at() -> VortexResult<()> { let bool: &dyn Array = &BoolArray::from(vec![true, false, true, true]); let array = RoaringBoolArray::encode(bool)?; diff --git a/enc-roaring/src/boolean/serde.rs b/vortex-roaring/src/boolean/serde.rs similarity index 92% rename from enc-roaring/src/boolean/serde.rs rename to vortex-roaring/src/boolean/serde.rs index 3faeeb4f22..b624b734c4 100644 --- a/enc-roaring/src/boolean/serde.rs +++ b/vortex-roaring/src/boolean/serde.rs @@ -3,8 +3,8 @@ use std::io::ErrorKind; use croaring::{Bitmap, Portable}; -use enc::array::{Array, ArrayRef}; -use enc::serde::{ArraySerde, EncodingSerde, ReadCtx, WriteCtx}; +use vortex::array::{Array, ArrayRef}; +use vortex::serde::{ArraySerde, EncodingSerde, ReadCtx, WriteCtx}; use crate::{RoaringBoolArray, RoaringBoolEncoding}; diff --git a/enc-roaring/src/boolean/stats.rs b/vortex-roaring/src/boolean/stats.rs similarity index 88% rename from enc-roaring/src/boolean/stats.rs rename to vortex-roaring/src/boolean/stats.rs index 38c6672c71..9c7a141a0b 100644 --- a/enc-roaring/src/boolean/stats.rs +++ b/vortex-roaring/src/boolean/stats.rs @@ -1,5 +1,5 @@ -use enc::array::Array; -use enc::stats::{Stat, StatsCompute, StatsSet}; +use vortex::array::Array; +use vortex::stats::{Stat, StatsCompute, StatsSet}; use crate::boolean::RoaringBoolArray; @@ -35,15 +35,15 @@ impl StatsCompute for RoaringBoolArray { #[cfg(test)] mod test { - use enc::array::bool::BoolArray; - use enc::array::Array; - use enc::error::EncResult; - use enc::stats::Stat::*; + use vortex::array::bool::BoolArray; + use vortex::array::Array; + use vortex::error::VortexResult; + use vortex::stats::Stat::*; use crate::RoaringBoolArray; #[test] - pub fn stats_all_true() -> EncResult<()> { + pub fn stats_all_true() -> VortexResult<()> { let bool: &dyn Array = &BoolArray::from(vec![true, true]); let array = RoaringBoolArray::encode(bool)?; @@ -65,7 +65,7 @@ mod test { } #[test] - pub fn stats_all_false() -> EncResult<()> { + pub fn stats_all_false() -> VortexResult<()> { let bool: &dyn Array = &BoolArray::from(vec![false, false]); let array = RoaringBoolArray::encode(bool)?; @@ -87,7 +87,7 @@ mod test { } #[test] - pub fn stats_mixed() -> EncResult<()> { + pub fn stats_mixed() -> VortexResult<()> { let bool: &dyn Array = &BoolArray::from(vec![false, true, true]); let array = RoaringBoolArray::encode(bool)?; diff --git a/enc-roaring/src/downcast.rs b/vortex-roaring/src/downcast.rs similarity index 96% rename from enc-roaring/src/downcast.rs rename to vortex-roaring/src/downcast.rs index a0501a80d7..2ad13158e5 100644 --- a/enc-roaring/src/downcast.rs +++ b/vortex-roaring/src/downcast.rs @@ -1,4 +1,4 @@ -use enc::array::{Array, ArrayRef}; +use vortex::array::{Array, ArrayRef}; use crate::{RoaringBoolArray, RoaringIntArray}; diff --git a/enc-roaring/src/integer/compress.rs b/vortex-roaring/src/integer/compress.rs similarity index 84% rename from enc-roaring/src/integer/compress.rs rename to vortex-roaring/src/integer/compress.rs index 961571f2c8..01f28a07e3 100644 --- a/enc-roaring/src/integer/compress.rs +++ b/vortex-roaring/src/integer/compress.rs @@ -2,15 +2,15 @@ use croaring::Bitmap; use log::debug; use num_traits::NumCast; -use enc::array::downcast::DowncastArrayBuiltin; -use enc::array::primitive::{PrimitiveArray, PRIMITIVE_ENCODING}; -use enc::array::{Array, ArrayRef}; -use enc::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression}; -use enc::dtype::DType; -use enc::dtype::Nullability::NonNullable; -use enc::dtype::Signedness::Unsigned; -use enc::ptype::{NativePType, PType}; -use enc::stats::Stat; +use vortex::array::downcast::DowncastArrayBuiltin; +use vortex::array::primitive::{PrimitiveArray, PRIMITIVE_ENCODING}; +use vortex::array::{Array, ArrayRef}; +use vortex::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression}; +use vortex::dtype::DType; +use vortex::dtype::Nullability::NonNullable; +use vortex::dtype::Signedness::Unsigned; +use vortex::ptype::{NativePType, PType}; +use vortex::stats::Stat; use crate::{RoaringIntArray, RoaringIntEncoding}; diff --git a/enc-roaring/src/integer/mod.rs b/vortex-roaring/src/integer/mod.rs similarity index 79% rename from enc-roaring/src/integer/mod.rs rename to vortex-roaring/src/integer/mod.rs index 366b76a004..72262dcc65 100644 --- a/enc-roaring/src/integer/mod.rs +++ b/vortex-roaring/src/integer/mod.rs @@ -4,18 +4,18 @@ use std::sync::{Arc, RwLock}; use croaring::{Bitmap, Native}; use compress::roaring_encode; -use enc::array::{ +use vortex::array::{ check_index_bounds, check_slice_bounds, Array, ArrayKind, ArrayRef, ArrowIterator, Encoding, EncodingId, EncodingRef, }; -use enc::compress::EncodingCompression; -use enc::dtype::DType; -use enc::error::{EncError, EncResult}; -use enc::formatter::{ArrayDisplay, ArrayFormatter}; -use enc::ptype::PType; -use enc::scalar::Scalar; -use enc::serde::{ArraySerde, EncodingSerde}; -use enc::stats::{Stats, StatsSet}; +use vortex::compress::EncodingCompression; +use vortex::dtype::DType; +use vortex::error::{VortexError, VortexResult}; +use vortex::formatter::{ArrayDisplay, ArrayFormatter}; +use vortex::ptype::PType; +use vortex::scalar::Scalar; +use vortex::serde::{ArraySerde, EncodingSerde}; +use vortex::stats::{Stats, StatsSet}; mod compress; mod serde; @@ -33,9 +33,9 @@ impl RoaringIntArray { Self::try_new(bitmap, ptype).unwrap() } - pub fn try_new(bitmap: Bitmap, ptype: PType) -> EncResult { + pub fn try_new(bitmap: Bitmap, ptype: PType) -> VortexResult { if !ptype.is_unsigned_int() { - return Err(EncError::InvalidPType(ptype)); + return Err(VortexError::InvalidPType(ptype)); } Ok(Self { @@ -53,10 +53,10 @@ impl RoaringIntArray { self.ptype } - pub fn encode(array: &dyn Array) -> EncResult { + pub fn encode(array: &dyn Array) -> VortexResult { match ArrayKind::from(array) { ArrayKind::Primitive(p) => Ok(roaring_encode(p)), - _ => Err(EncError::InvalidEncoding(array.encoding().id().clone())), + _ => Err(VortexError::InvalidEncoding(array.encoding().id().clone())), } } } @@ -96,7 +96,7 @@ impl Array for RoaringIntArray { Stats::new(&self.stats, self) } - fn scalar_at(&self, index: usize) -> EncResult> { + fn scalar_at(&self, index: usize) -> VortexResult> { check_index_bounds(self, index)?; // Unwrap since we know the index is valid let bitmap_value = self.bitmap.select(index as u32).unwrap(); @@ -114,7 +114,7 @@ impl Array for RoaringIntArray { todo!() } - fn slice(&self, start: usize, stop: usize) -> EncResult { + fn slice(&self, start: usize, stop: usize) -> VortexResult { check_slice_bounds(self, start, stop)?; todo!() } @@ -167,14 +167,14 @@ impl Encoding for RoaringIntEncoding { #[cfg(test)] mod test { - use enc::array::primitive::PrimitiveArray; - use enc::array::Array; - use enc::error::EncResult; + use vortex::array::primitive::PrimitiveArray; + use vortex::array::Array; + use vortex::error::VortexResult; use crate::RoaringIntArray; #[test] - pub fn scalar_at() -> EncResult<()> { + pub fn scalar_at() -> VortexResult<()> { let ints: &dyn Array = &PrimitiveArray::from_vec::(vec![2, 12, 22, 32]); let array = RoaringIntArray::encode(ints)?; diff --git a/enc-roaring/src/integer/serde.rs b/vortex-roaring/src/integer/serde.rs similarity index 89% rename from enc-roaring/src/integer/serde.rs rename to vortex-roaring/src/integer/serde.rs index e3ac38bd23..dd99d59715 100644 --- a/enc-roaring/src/integer/serde.rs +++ b/vortex-roaring/src/integer/serde.rs @@ -3,9 +3,9 @@ use std::io::ErrorKind; use croaring::{Bitmap, Portable}; -use enc::array::{Array, ArrayRef}; -use enc::ptype::PType; -use enc::serde::{ArraySerde, EncodingSerde, ReadCtx, WriteCtx}; +use vortex::array::{Array, ArrayRef}; +use vortex::ptype::PType; +use vortex::serde::{ArraySerde, EncodingSerde, ReadCtx, WriteCtx}; use crate::{RoaringIntArray, RoaringIntEncoding}; @@ -37,7 +37,7 @@ impl EncodingSerde for RoaringIntEncoding { mod test { use croaring::Bitmap; - use enc::ptype::PType; + use vortex::ptype::PType; use crate::downcast::DowncastRoaring; use crate::serde_tests::test::roundtrip_array; diff --git a/enc-roaring/src/integer/stats.rs b/vortex-roaring/src/integer/stats.rs similarity index 92% rename from enc-roaring/src/integer/stats.rs rename to vortex-roaring/src/integer/stats.rs index 70a1c471cf..14579eeb96 100644 --- a/enc-roaring/src/integer/stats.rs +++ b/vortex-roaring/src/integer/stats.rs @@ -1,4 +1,4 @@ -use enc::stats::{Stat, StatsCompute, StatsSet}; +use vortex::stats::{Stat, StatsCompute, StatsSet}; use crate::RoaringIntArray; diff --git a/enc-roaring/src/lib.rs b/vortex-roaring/src/lib.rs similarity index 87% rename from enc-roaring/src/lib.rs rename to vortex-roaring/src/lib.rs index 2859c1b796..0e844e00c8 100644 --- a/enc-roaring/src/lib.rs +++ b/vortex-roaring/src/lib.rs @@ -1,8 +1,8 @@ use linkme::distributed_slice; pub use boolean::*; -use enc::array::{EncodingRef, ENCODINGS}; pub use integer::*; +use vortex::array::{EncodingRef, ENCODINGS}; mod boolean; mod downcast; diff --git a/enc-roaring/src/serde_tests.rs b/vortex-roaring/src/serde_tests.rs similarity index 81% rename from enc-roaring/src/serde_tests.rs rename to vortex-roaring/src/serde_tests.rs index c93fe61277..71a06a9a7a 100644 --- a/enc-roaring/src/serde_tests.rs +++ b/vortex-roaring/src/serde_tests.rs @@ -1,8 +1,8 @@ #[cfg(test)] pub mod test { - use enc::array::{Array, ArrayRef}; - use enc::serde::{ReadCtx, WriteCtx}; use std::io; + use vortex::array::{Array, ArrayRef}; + use vortex::serde::{ReadCtx, WriteCtx}; pub fn roundtrip_array(array: &dyn Array) -> io::Result { let mut buf = Vec::::new(); diff --git a/enc-zigzag/Cargo.toml b/vortex-zigzag/Cargo.toml similarity index 50% rename from enc-zigzag/Cargo.toml rename to vortex-zigzag/Cargo.toml index bab1503f3b..ab528eb314 100644 --- a/enc-zigzag/Cargo.toml +++ b/vortex-zigzag/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "enc-zigzag" +name = "vortex-zigzag" version = "0.1.0" edition = "2021" @@ -7,7 +7,7 @@ edition = "2021" workspace = true [dependencies] -enc = { "path" = "../enc" } +vortex = { "path" = "../vortex" } linkme = "0.3.22" -spiral-alloc = { version = "0.1.0", path = "../spiral-alloc" } +vortex-alloc = { version = "0.1.0", path = "../vortex-alloc" } zigzag = "0.1.0" diff --git a/enc-zigzag/src/compress.rs b/vortex-zigzag/src/compress.rs similarity index 88% rename from enc-zigzag/src/compress.rs rename to vortex-zigzag/src/compress.rs index 7a60ffe83a..f2e6f97aa8 100644 --- a/enc-zigzag/src/compress.rs +++ b/vortex-zigzag/src/compress.rs @@ -1,14 +1,14 @@ use zigzag::ZigZag; use crate::downcast::DowncastZigzag; -use enc::array::downcast::DowncastArrayBuiltin; -use enc::array::primitive::PrimitiveArray; -use enc::array::{Array, ArrayKind, ArrayRef}; -use enc::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression}; -use enc::error::EncResult; -use enc::ptype::{NativePType, PType}; -use enc::stats::Stat; -use spiral_alloc::{AlignedVec, ALIGNED_ALLOCATOR}; +use vortex::array::downcast::DowncastArrayBuiltin; +use vortex::array::primitive::PrimitiveArray; +use vortex::array::{Array, ArrayKind, ArrayRef}; +use vortex::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression}; +use vortex::error::VortexResult; +use vortex::ptype::{NativePType, PType}; +use vortex::stats::Stat; +use vortex_alloc::{AlignedVec, ALIGNED_ALLOCATOR}; use crate::zigzag::{ZigZagArray, ZigZagEncoding}; @@ -50,7 +50,7 @@ fn zigzag_compressor(array: &dyn Array, like: Option<&dyn Array>, ctx: CompressC .boxed() } -pub fn zigzag_encode(parray: &PrimitiveArray) -> EncResult { +pub fn zigzag_encode(parray: &PrimitiveArray) -> VortexResult { let encoded = match parray.ptype() { PType::I8 => zigzag_encode_primitive::(parray.buffer().typed_data(), parray.validity()), PType::I16 => { diff --git a/enc-zigzag/src/downcast.rs b/vortex-zigzag/src/downcast.rs similarity index 94% rename from enc-zigzag/src/downcast.rs rename to vortex-zigzag/src/downcast.rs index 440b86f679..5e2092170b 100644 --- a/enc-zigzag/src/downcast.rs +++ b/vortex-zigzag/src/downcast.rs @@ -1,4 +1,4 @@ -use enc::array::{Array, ArrayRef}; +use vortex::array::{Array, ArrayRef}; use crate::ZigZagArray; diff --git a/enc-zigzag/src/lib.rs b/vortex-zigzag/src/lib.rs similarity index 81% rename from enc-zigzag/src/lib.rs rename to vortex-zigzag/src/lib.rs index e47b6faf22..55da8b3bcd 100644 --- a/enc-zigzag/src/lib.rs +++ b/vortex-zigzag/src/lib.rs @@ -1,5 +1,5 @@ -use enc::array::{EncodingRef, ENCODINGS}; use linkme::distributed_slice; +use vortex::array::{EncodingRef, ENCODINGS}; pub use zigzag::*; diff --git a/enc-zigzag/src/serde.rs b/vortex-zigzag/src/serde.rs similarity index 83% rename from enc-zigzag/src/serde.rs rename to vortex-zigzag/src/serde.rs index 6354445981..2af02e2f49 100644 --- a/enc-zigzag/src/serde.rs +++ b/vortex-zigzag/src/serde.rs @@ -1,9 +1,9 @@ use std::io; use std::io::ErrorKind; -use enc::array::{Array, ArrayRef}; -use enc::dtype::{DType, Signedness}; -use enc::serde::{ArraySerde, EncodingSerde, ReadCtx, WriteCtx}; +use vortex::array::{Array, ArrayRef}; +use vortex::dtype::{DType, Signedness}; +use vortex::serde::{ArraySerde, EncodingSerde, ReadCtx, WriteCtx}; use crate::{ZigZagArray, ZigZagEncoding}; @@ -33,10 +33,10 @@ impl EncodingSerde for ZigZagEncoding { mod test { use std::io; - use enc::array::downcast::DowncastArrayBuiltin; - use enc::array::primitive::PrimitiveArray; - use enc::array::{Array, ArrayRef}; - use enc::serde::{ReadCtx, WriteCtx}; + use vortex::array::downcast::DowncastArrayBuiltin; + use vortex::array::primitive::PrimitiveArray; + use vortex::array::{Array, ArrayRef}; + use vortex::serde::{ReadCtx, WriteCtx}; use crate::compress::zigzag_encode; use crate::downcast::DowncastZigzag; diff --git a/enc-zigzag/src/stats.rs b/vortex-zigzag/src/stats.rs similarity index 83% rename from enc-zigzag/src/stats.rs rename to vortex-zigzag/src/stats.rs index 31f34e8b62..1feac0aff1 100644 --- a/enc-zigzag/src/stats.rs +++ b/vortex-zigzag/src/stats.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; -use enc::stats::{Stat, StatsCompute, StatsSet}; +use vortex::stats::{Stat, StatsCompute, StatsSet}; use crate::zigzag::ZigZagArray; diff --git a/enc-zigzag/src/zigzag.rs b/vortex-zigzag/src/zigzag.rs similarity index 78% rename from enc-zigzag/src/zigzag.rs rename to vortex-zigzag/src/zigzag.rs index 10e075cef8..d095a39bfb 100644 --- a/enc-zigzag/src/zigzag.rs +++ b/vortex-zigzag/src/zigzag.rs @@ -3,17 +3,17 @@ use std::sync::{Arc, RwLock}; use zigzag::ZigZag; -use enc::array::{ +use vortex::array::{ check_index_bounds, Array, ArrayKind, ArrayRef, ArrowIterator, Encoding, EncodingId, EncodingRef, }; -use enc::compress::EncodingCompression; -use enc::dtype::{DType, IntWidth, Signedness}; -use enc::error::{EncError, EncResult}; -use enc::formatter::{ArrayDisplay, ArrayFormatter}; -use enc::scalar::{NullableScalar, Scalar}; -use enc::serde::{ArraySerde, EncodingSerde}; -use enc::stats::{Stats, StatsSet}; +use vortex::compress::EncodingCompression; +use vortex::dtype::{DType, IntWidth, Signedness}; +use vortex::error::{VortexError, VortexResult}; +use vortex::formatter::{ArrayDisplay, ArrayFormatter}; +use vortex::scalar::{NullableScalar, Scalar}; +use vortex::serde::{ArraySerde, EncodingSerde}; +use vortex::stats::{Stats, StatsSet}; use crate::compress::zigzag_encode; @@ -29,12 +29,12 @@ impl ZigZagArray { Self::try_new(encoded).unwrap() } - pub fn try_new(encoded: ArrayRef) -> EncResult { + pub fn try_new(encoded: ArrayRef) -> VortexResult { let dtype = match encoded.dtype() { DType::Int(width, Signedness::Unsigned, nullability) => { DType::Int(*width, Signedness::Signed, *nullability) } - d => return Err(EncError::InvalidDType(d.clone())), + d => return Err(VortexError::InvalidDType(d.clone())), }; Ok(Self { encoded, @@ -43,10 +43,10 @@ impl ZigZagArray { }) } - pub fn encode(array: &dyn Array) -> EncResult { + pub fn encode(array: &dyn Array) -> VortexResult { match ArrayKind::from(array) { ArrayKind::Primitive(p) => Ok(zigzag_encode(p)?.boxed()), - _ => Err(EncError::InvalidEncoding(array.encoding().id().clone())), + _ => Err(VortexError::InvalidEncoding(array.encoding().id().clone())), } } @@ -91,7 +91,7 @@ impl Array for ZigZagArray { Stats::new(&self.stats, self) } - fn scalar_at(&self, index: usize) -> EncResult> { + fn scalar_at(&self, index: usize) -> VortexResult> { check_index_bounds(self, index)?; let scalar = self.encoded().scalar_at(index)?; @@ -111,7 +111,7 @@ impl Array for ZigZagArray { DType::Int(IntWidth::_64, Signedness::Signed, _) => { Ok(i64::decode(scalar.try_into()?).into()) } - _ => Err(EncError::InvalidDType(self.dtype().clone())), + _ => Err(VortexError::InvalidDType(self.dtype().clone())), } } @@ -119,7 +119,7 @@ impl Array for ZigZagArray { todo!() } - fn slice(&self, start: usize, stop: usize) -> EncResult { + fn slice(&self, start: usize, stop: usize) -> VortexResult { Ok(Self::try_new(self.encoded.slice(start, stop)?)?.boxed()) } @@ -154,7 +154,7 @@ impl ArrayDisplay for ZigZagArray { #[derive(Debug)] pub struct ZigZagEncoding; -pub const ZIGZAG_ENCODING: EncodingId = EncodingId::new("enc.zigzag"); +pub const ZIGZAG_ENCODING: EncodingId = EncodingId::new("vortex.zigzag"); impl Encoding for ZigZagEncoding { fn id(&self) -> &EncodingId { diff --git a/enc/Cargo.toml b/vortex/Cargo.toml similarity index 87% rename from enc/Cargo.toml rename to vortex/Cargo.toml index d5863e8aa4..9b8a132f79 100644 --- a/enc/Cargo.toml +++ b/vortex/Cargo.toml @@ -1,10 +1,10 @@ [package] -name = "enc" +name = "vortex" version = "0.1.0" edition = "2021" [lib] -name = "enc" +name = "vortex" path = "src/lib.rs" [lints] @@ -29,5 +29,5 @@ polars-ops = { version = "0.37.0", features = ["search_sorted"] } rand = { version = "0.8.5", features = [] } rayon = "1.8.1" roaring = "0.10.3" -spiral-alloc = { version = "0.1.0", path = "../spiral-alloc" } +vortex-alloc = { version = "0.1.0", path = "../vortex-alloc" } thiserror = "1.0.57" diff --git a/enc/README.md b/vortex/README.md similarity index 100% rename from enc/README.md rename to vortex/README.md diff --git a/enc/src/array/bool/compress.rs b/vortex/src/array/bool/compress.rs similarity index 100% rename from enc/src/array/bool/compress.rs rename to vortex/src/array/bool/compress.rs diff --git a/enc/src/array/bool/mod.rs b/vortex/src/array/bool/mod.rs similarity index 95% rename from enc/src/array/bool/mod.rs rename to vortex/src/array/bool/mod.rs index 677c49ecfd..e41be912e7 100644 --- a/enc/src/array/bool/mod.rs +++ b/vortex/src/array/bool/mod.rs @@ -9,7 +9,7 @@ use linkme::distributed_slice; use crate::arrow::CombineChunks; use crate::compress::EncodingCompression; use crate::dtype::{DType, Nullability}; -use crate::error::EncResult; +use crate::error::VortexResult; use crate::formatter::{ArrayDisplay, ArrayFormatter}; use crate::scalar::{NullableScalar, Scalar}; use crate::serde::{ArraySerde, EncodingSerde}; @@ -36,7 +36,7 @@ impl BoolArray { Self::try_new(buffer, validity).unwrap() } - pub fn try_new(buffer: BooleanBuffer, validity: Option) -> EncResult { + pub fn try_new(buffer: BooleanBuffer, validity: Option) -> VortexResult { let validity = validity.filter(|v| !v.is_empty()); check_validity_buffer(validity.as_ref())?; @@ -105,7 +105,7 @@ impl Array for BoolArray { Stats::new(&self.stats, self) } - fn scalar_at(&self, index: usize) -> EncResult> { + fn scalar_at(&self, index: usize) -> VortexResult> { check_index_bounds(self, index)?; if self.is_valid(index) { @@ -130,7 +130,7 @@ impl Array for BoolArray { )) as ArrowArrayRef)) } - fn slice(&self, start: usize, stop: usize) -> EncResult { + fn slice(&self, start: usize, stop: usize) -> VortexResult { check_slice_bounds(self, start, stop)?; Ok(Self { @@ -163,7 +163,7 @@ impl Array for BoolArray { #[derive(Debug)] struct BoolEncoding; -pub const BOOL_ENCODING: EncodingId = EncodingId("enc.bool"); +pub const BOOL_ENCODING: EncodingId = EncodingId::new("vortex.bool"); #[distributed_slice(ENCODINGS)] static ENCODINGS_BOOL: EncodingRef = &BoolEncoding; diff --git a/enc/src/array/bool/serde.rs b/vortex/src/array/bool/serde.rs similarity index 100% rename from enc/src/array/bool/serde.rs rename to vortex/src/array/bool/serde.rs diff --git a/enc/src/array/bool/stats.rs b/vortex/src/array/bool/stats.rs similarity index 100% rename from enc/src/array/bool/stats.rs rename to vortex/src/array/bool/stats.rs diff --git a/enc/src/array/chunked/compress.rs b/vortex/src/array/chunked/compress.rs similarity index 100% rename from enc/src/array/chunked/compress.rs rename to vortex/src/array/chunked/compress.rs diff --git a/enc/src/array/chunked/mod.rs b/vortex/src/array/chunked/mod.rs similarity index 95% rename from enc/src/array/chunked/mod.rs rename to vortex/src/array/chunked/mod.rs index 1100dcf54f..eb1fb5f837 100644 --- a/enc/src/array/chunked/mod.rs +++ b/vortex/src/array/chunked/mod.rs @@ -12,7 +12,7 @@ use crate::array::{ }; use crate::compress::EncodingCompression; use crate::dtype::DType; -use crate::error::{EncError, EncResult}; +use crate::error::{VortexError, VortexResult}; use crate::formatter::{ArrayDisplay, ArrayFormatter}; use crate::scalar::Scalar; use crate::serde::{ArraySerde, EncodingSerde}; @@ -35,7 +35,7 @@ impl ChunkedArray { Self::try_new(chunks, dtype).unwrap() } - pub fn try_new(chunks: Vec, dtype: DType) -> EncResult { + pub fn try_new(chunks: Vec, dtype: DType) -> VortexResult { chunks .iter() .map(|c| c.dtype().as_nullable()) @@ -43,7 +43,7 @@ impl ChunkedArray { .map(|_| ()) .or_else(|mismatched| match mismatched { None => Ok(()), - Some((fst, snd)) => Err(EncError::MismatchedTypes(fst, snd)), + Some((fst, snd)) => Err(VortexError::MismatchedTypes(fst, snd)), })?; let chunk_ends = chunks @@ -126,7 +126,7 @@ impl Array for ChunkedArray { Stats::new(&self.stats, self) } - fn scalar_at(&self, index: usize) -> EncResult> { + fn scalar_at(&self, index: usize) -> VortexResult> { check_index_bounds(self, index)?; let (chunk_index, chunk_offset) = self.find_physical_location(index); self.chunks[chunk_index].scalar_at(chunk_offset) @@ -136,7 +136,7 @@ impl Array for ChunkedArray { Box::new(ChunkedArrowIterator::new(self)) } - fn slice(&self, start: usize, stop: usize) -> EncResult { + fn slice(&self, start: usize, stop: usize) -> VortexResult { check_slice_bounds(self, start, stop)?; let (offset_chunk, offset_in_first_chunk) = self.find_physical_location(start); @@ -213,7 +213,7 @@ impl ArrayDisplay for ChunkedArray { #[derive(Debug)] struct ChunkedEncoding; -pub const CHUNKED_ENCODING: EncodingId = EncodingId("enc.chunked"); +pub const CHUNKED_ENCODING: EncodingId = EncodingId::new("vortex.chunked"); #[distributed_slice(ENCODINGS)] static ENCODINGS_CHUNKED: EncodingRef = &ChunkedEncoding; diff --git a/enc/src/array/chunked/serde.rs b/vortex/src/array/chunked/serde.rs similarity index 100% rename from enc/src/array/chunked/serde.rs rename to vortex/src/array/chunked/serde.rs diff --git a/enc/src/array/chunked/stats.rs b/vortex/src/array/chunked/stats.rs similarity index 100% rename from enc/src/array/chunked/stats.rs rename to vortex/src/array/chunked/stats.rs diff --git a/enc/src/array/constant/compress.rs b/vortex/src/array/constant/compress.rs similarity index 100% rename from enc/src/array/constant/compress.rs rename to vortex/src/array/constant/compress.rs diff --git a/enc/src/array/constant/compute.rs b/vortex/src/array/constant/compute.rs similarity index 100% rename from enc/src/array/constant/compute.rs rename to vortex/src/array/constant/compute.rs diff --git a/enc/src/array/constant/mod.rs b/vortex/src/array/constant/mod.rs similarity index 91% rename from enc/src/array/constant/mod.rs rename to vortex/src/array/constant/mod.rs index 0d325783fb..42527c948d 100644 --- a/enc/src/array/constant/mod.rs +++ b/vortex/src/array/constant/mod.rs @@ -11,7 +11,7 @@ use crate::array::{ use crate::arrow::compute::repeat; use crate::compress::EncodingCompression; use crate::dtype::DType; -use crate::error::EncResult; +use crate::error::VortexResult; use crate::formatter::{ArrayDisplay, ArrayFormatter}; use crate::scalar::Scalar; use crate::serde::{ArraySerde, EncodingSerde}; @@ -80,7 +80,7 @@ impl Array for ConstantArray { Stats::new(&self.stats, self) } - fn scalar_at(&self, index: usize) -> EncResult> { + fn scalar_at(&self, index: usize) -> VortexResult> { check_index_bounds(self, index)?; Ok(self.scalar.clone()) } @@ -90,7 +90,7 @@ impl Array for ConstantArray { Box::new(std::iter::once(repeat(arrow_scalar.as_ref(), self.length))) } - fn slice(&self, start: usize, stop: usize) -> EncResult { + fn slice(&self, start: usize, stop: usize) -> VortexResult { check_slice_bounds(self, start, stop)?; Ok(ConstantArray::new(self.scalar.clone(), stop - start).boxed()) @@ -126,7 +126,7 @@ impl ArrayDisplay for ConstantArray { #[derive(Debug)] pub struct ConstantEncoding; -pub const CONSTANT_ENCODING: EncodingId = EncodingId("enc.constant"); +pub const CONSTANT_ENCODING: EncodingId = EncodingId::new("vortex.constant"); #[distributed_slice(ENCODINGS)] static ENCODINGS_CONSTANT: EncodingRef = &ConstantEncoding; diff --git a/enc/src/array/constant/serde.rs b/vortex/src/array/constant/serde.rs similarity index 100% rename from enc/src/array/constant/serde.rs rename to vortex/src/array/constant/serde.rs diff --git a/enc/src/array/constant/stats.rs b/vortex/src/array/constant/stats.rs similarity index 100% rename from enc/src/array/constant/stats.rs rename to vortex/src/array/constant/stats.rs diff --git a/enc/src/array/constant/take.rs b/vortex/src/array/constant/take.rs similarity index 71% rename from enc/src/array/constant/take.rs rename to vortex/src/array/constant/take.rs index 9e20e433bb..0760e501fd 100644 --- a/enc/src/array/constant/take.rs +++ b/vortex/src/array/constant/take.rs @@ -1,10 +1,10 @@ use crate::array::constant::ConstantArray; use crate::array::{Array, ArrayRef}; use crate::compute::take::TakeFn; -use crate::error::EncResult; +use crate::error::VortexResult; impl TakeFn for ConstantArray { - fn take(&self, indices: &dyn Array) -> EncResult { + fn take(&self, indices: &dyn Array) -> VortexResult { Ok(ConstantArray::new(dyn_clone::clone_box(self.scalar()), indices.len()).boxed()) } } diff --git a/enc/src/array/downcast.rs b/vortex/src/array/downcast.rs similarity index 100% rename from enc/src/array/downcast.rs rename to vortex/src/array/downcast.rs diff --git a/enc/src/array/mod.rs b/vortex/src/array/mod.rs similarity index 88% rename from enc/src/array/mod.rs rename to vortex/src/array/mod.rs index 11d4448c2c..8be11fd2f1 100644 --- a/enc/src/array/mod.rs +++ b/vortex/src/array/mod.rs @@ -17,7 +17,7 @@ use crate::array::varbinview::{VarBinViewArray, VARBINVIEW_ENCODING}; use crate::compress::EncodingCompression; use crate::compute::ArrayCompute; use crate::dtype::{DType, Nullability}; -use crate::error::{EncError, EncResult}; +use crate::error::{VortexError, VortexResult}; use crate::formatter::{ArrayDisplay, ArrayFormatter}; use crate::scalar::Scalar; use crate::serde::{ArraySerde, EncodingSerde}; @@ -40,7 +40,7 @@ pub type ArrayRef = Box; /// An Enc Array is the base object representing all arrays in enc. /// /// Arrays have a dtype and an encoding. DTypes represent the logical type of the -/// values stored in an pyenc array. Encodings represent the physical layout of the +/// values stored in a vortex array. Encodings represent the physical layout of the /// array. /// /// This differs from Apache Arrow where logical and physical are combined in @@ -61,11 +61,11 @@ pub trait Array: ArrayDisplay + Debug + Send + Sync + dyn_clone::DynClone + 'sta /// Get statistics for the array fn stats(&self) -> Stats; /// Get scalar value at given index - fn scalar_at(&self, index: usize) -> EncResult>; + fn scalar_at(&self, index: usize) -> VortexResult>; /// Produce arrow batches from the encoding fn iter_arrow(&self) -> Box; /// Limit array to start..stop range - fn slice(&self, start: usize, stop: usize) -> EncResult; + fn slice(&self, start: usize, stop: usize) -> VortexResult; /// Encoding kind of the array fn encoding(&self) -> &'static dyn Encoding; /// Approximate size in bytes of the array. Only takes into account variable size portion of the array @@ -80,29 +80,29 @@ pub trait Array: ArrayDisplay + Debug + Send + Sync + dyn_clone::DynClone + 'sta dyn_clone::clone_trait_object!(Array); -pub fn check_slice_bounds(array: &dyn Array, start: usize, stop: usize) -> EncResult<()> { +pub fn check_slice_bounds(array: &dyn Array, start: usize, stop: usize) -> VortexResult<()> { if start > array.len() { - return Err(EncError::OutOfBounds(start, 0, array.len())); + return Err(VortexError::OutOfBounds(start, 0, array.len())); } if stop > array.len() { - return Err(EncError::OutOfBounds(stop, 0, array.len())); + return Err(VortexError::OutOfBounds(stop, 0, array.len())); } Ok(()) } -pub fn check_index_bounds(array: &dyn Array, index: usize) -> EncResult<()> { +pub fn check_index_bounds(array: &dyn Array, index: usize) -> VortexResult<()> { if index >= array.len() { - return Err(EncError::OutOfBounds(index, 0, array.len())); + return Err(VortexError::OutOfBounds(index, 0, array.len())); } Ok(()) } -pub fn check_validity_buffer(validity: Option<&ArrayRef>) -> EncResult<()> { +pub fn check_validity_buffer(validity: Option<&ArrayRef>) -> VortexResult<()> { if validity .map(|v| !matches!(v.dtype(), DType::Bool(Nullability::NonNullable))) .unwrap_or(false) { - return Err(EncError::MismatchedTypes( + return Err(VortexError::MismatchedTypes( validity.unwrap().dtype().clone(), DType::Bool(Nullability::NonNullable), )); diff --git a/enc/src/array/primitive/compress.rs b/vortex/src/array/primitive/compress.rs similarity index 100% rename from enc/src/array/primitive/compress.rs rename to vortex/src/array/primitive/compress.rs diff --git a/enc/src/array/primitive/mod.rs b/vortex/src/array/primitive/mod.rs similarity index 96% rename from enc/src/array/primitive/mod.rs rename to vortex/src/array/primitive/mod.rs index f22821a647..910433c33d 100644 --- a/enc/src/array/primitive/mod.rs +++ b/vortex/src/array/primitive/mod.rs @@ -21,7 +21,7 @@ use crate::array::{ use crate::arrow::CombineChunks; use crate::compress::EncodingCompression; use crate::dtype::DType; -use crate::error::EncResult; +use crate::error::VortexResult; use crate::formatter::{ArrayDisplay, ArrayFormatter}; use crate::ptype::{match_each_native_ptype, NativePType, PType}; use crate::scalar::{NullableScalar, Scalar}; @@ -46,7 +46,7 @@ impl PrimitiveArray { Self::try_new(ptype, buffer, validity).unwrap() } - pub fn try_new(ptype: PType, buffer: Buffer, validity: Option) -> EncResult { + pub fn try_new(ptype: PType, buffer: Buffer, validity: Option) -> VortexResult { let validity = validity.filter(|v| !v.is_empty()); check_validity_buffer(validity.as_ref())?; let dtype = if validity.is_some() { @@ -166,7 +166,7 @@ impl Array for PrimitiveArray { Stats::new(&self.stats, self) } - fn scalar_at(&self, index: usize) -> EncResult> { + fn scalar_at(&self, index: usize) -> VortexResult> { check_index_bounds(self, index)?; if self.is_valid(index) { @@ -202,7 +202,7 @@ impl Array for PrimitiveArray { ))) } - fn slice(&self, start: usize, stop: usize) -> EncResult { + fn slice(&self, start: usize, stop: usize) -> VortexResult { check_slice_bounds(self, start, stop)?; let byte_start = start * self.ptype.byte_width(); @@ -246,7 +246,7 @@ impl<'arr> AsRef<(dyn Array + 'arr)> for PrimitiveArray { #[derive(Debug)] pub struct PrimitiveEncoding; -pub const PRIMITIVE_ENCODING: EncodingId = EncodingId("enc.primitive"); +pub const PRIMITIVE_ENCODING: EncodingId = EncodingId::new("vortex.primitive"); #[distributed_slice(ENCODINGS)] static ENCODINGS_PRIMITIVE: EncodingRef = &PrimitiveEncoding; diff --git a/enc/src/array/primitive/serde.rs b/vortex/src/array/primitive/serde.rs similarity index 100% rename from enc/src/array/primitive/serde.rs rename to vortex/src/array/primitive/serde.rs diff --git a/enc/src/array/primitive/stats.rs b/vortex/src/array/primitive/stats.rs similarity index 100% rename from enc/src/array/primitive/stats.rs rename to vortex/src/array/primitive/stats.rs diff --git a/enc/src/array/sparse/compress.rs b/vortex/src/array/sparse/compress.rs similarity index 100% rename from enc/src/array/sparse/compress.rs rename to vortex/src/array/sparse/compress.rs diff --git a/enc/src/array/sparse/mod.rs b/vortex/src/array/sparse/mod.rs similarity index 94% rename from enc/src/array/sparse/mod.rs rename to vortex/src/array/sparse/mod.rs index 8ba14c74d8..95d4f5cd0e 100644 --- a/enc/src/array/sparse/mod.rs +++ b/vortex/src/array/sparse/mod.rs @@ -17,7 +17,7 @@ use crate::arrow::CombineChunks; use crate::compress::EncodingCompression; use crate::compute::search_sorted::{search_sorted_usize, SearchSortedSide}; use crate::dtype::{DType, Nullability, Signedness}; -use crate::error::{EncError, EncResult}; +use crate::error::{VortexError, VortexResult}; use crate::formatter::{ArrayDisplay, ArrayFormatter}; use crate::match_arrow_numeric_type; use crate::scalar::{NullableScalar, Scalar}; @@ -47,7 +47,7 @@ impl SparseArray { Self::try_new(indices, values, len).unwrap() } - pub fn try_new(indices: ArrayRef, values: ArrayRef, len: usize) -> EncResult { + pub fn try_new(indices: ArrayRef, values: ArrayRef, len: usize) -> VortexResult { Self::new_with_offset(indices, values, len, 0) } @@ -56,12 +56,12 @@ impl SparseArray { values: ArrayRef, len: usize, indices_offset: usize, - ) -> EncResult { + ) -> VortexResult { if !matches!( indices.dtype(), DType::Int(_, Signedness::Unsigned, Nullability::NonNullable) ) { - return Err(EncError::InvalidDType(indices.dtype().clone())); + return Err(VortexError::InvalidDType(indices.dtype().clone())); } let dtype = DType::Struct( @@ -134,7 +134,7 @@ impl Array for SparseArray { Stats::new(&self.stats, self) } - fn scalar_at(&self, index: usize) -> EncResult> { + fn scalar_at(&self, index: usize) -> VortexResult> { check_index_bounds(self, index)?; // Check whether `true_patch_index` exists in the patch index array @@ -194,7 +194,7 @@ impl Array for SparseArray { )) as Arc)) } - fn slice(&self, start: usize, stop: usize) -> EncResult { + fn slice(&self, start: usize, stop: usize) -> VortexResult { check_slice_bounds(self, start, stop)?; // Find the index of the first patch index that is greater than or equal to the offset of this array @@ -246,7 +246,7 @@ impl ArrayDisplay for SparseArray { #[derive(Debug)] pub struct SparseEncoding; -pub const SPARSE_ENCODING: EncodingId = EncodingId::new("enc.sparse"); +pub const SPARSE_ENCODING: EncodingId = EncodingId::new("vortex.sparse"); #[distributed_slice(ENCODINGS)] static ENCODINGS_SPARSE: EncodingRef = &SparseEncoding; @@ -269,7 +269,7 @@ impl Encoding for SparseEncoding { mod test { use crate::array::sparse::SparseArray; use crate::array::Array; - use crate::error::EncError; + use crate::error::VortexError; use arrow::array::AsArray; use arrow::datatypes::{Int32Type, UInt32Type}; @@ -334,7 +334,7 @@ mod test { ); assert_eq!( sparse_array().scalar_at(10).err().unwrap(), - EncError::OutOfBounds(10, 0, 10) + VortexError::OutOfBounds(10, 0, 10) ); } @@ -344,7 +344,7 @@ mod test { assert_eq!(usize::try_from(sliced.scalar_at(0).unwrap()).unwrap(), 100); assert_eq!( sliced.scalar_at(5).err().unwrap(), - EncError::OutOfBounds(5, 0, 5) + VortexError::OutOfBounds(5, 0, 5) ); } @@ -357,7 +357,7 @@ mod test { ); assert_eq!( sliced_once.scalar_at(7).err().unwrap(), - EncError::OutOfBounds(7, 0, 7) + VortexError::OutOfBounds(7, 0, 7) ); let sliced_twice = sliced_once.slice(1, 6).unwrap(); @@ -367,7 +367,7 @@ mod test { ); assert_eq!( sliced_twice.scalar_at(5).err().unwrap(), - EncError::OutOfBounds(5, 0, 5) + VortexError::OutOfBounds(5, 0, 5) ); } } diff --git a/enc/src/array/sparse/serde.rs b/vortex/src/array/sparse/serde.rs similarity index 100% rename from enc/src/array/sparse/serde.rs rename to vortex/src/array/sparse/serde.rs diff --git a/enc/src/array/sparse/stats.rs b/vortex/src/array/sparse/stats.rs similarity index 100% rename from enc/src/array/sparse/stats.rs rename to vortex/src/array/sparse/stats.rs diff --git a/enc/src/array/struct_/compress.rs b/vortex/src/array/struct_/compress.rs similarity index 100% rename from enc/src/array/struct_/compress.rs rename to vortex/src/array/struct_/compress.rs diff --git a/enc/src/array/struct_/mod.rs b/vortex/src/array/struct_/mod.rs similarity index 96% rename from enc/src/array/struct_/mod.rs rename to vortex/src/array/struct_/mod.rs index fc49de7d77..f3562d1f96 100644 --- a/enc/src/array/struct_/mod.rs +++ b/vortex/src/array/struct_/mod.rs @@ -10,7 +10,7 @@ use linkme::distributed_slice; use crate::arrow::aligned_iter::AlignedArrowArrayIterator; use crate::compress::EncodingCompression; use crate::dtype::{DType, FieldNames}; -use crate::error::EncResult; +use crate::error::VortexResult; use crate::formatter::{ArrayDisplay, ArrayFormatter}; use crate::scalar::{Scalar, StructScalar}; use crate::serde::{ArraySerde, EncodingSerde}; @@ -112,7 +112,7 @@ impl Array for StructArray { Stats::new(&self.stats, self) } - fn scalar_at(&self, index: usize) -> EncResult> { + fn scalar_at(&self, index: usize) -> VortexResult> { Ok(StructScalar::new( self.dtype.clone(), self.fields @@ -142,7 +142,7 @@ impl Array for StructArray { ) } - fn slice(&self, start: usize, stop: usize) -> EncResult { + fn slice(&self, start: usize, stop: usize) -> VortexResult { check_slice_bounds(self, start, stop)?; let fields = self @@ -181,7 +181,7 @@ impl<'arr> AsRef<(dyn Array + 'arr)> for StructArray { #[derive(Debug)] pub struct StructEncoding; -pub const STRUCT_ENCODING: EncodingId = EncodingId("enc.struct"); +pub const STRUCT_ENCODING: EncodingId = EncodingId::new("vortex.struct"); #[distributed_slice(ENCODINGS)] static ENCODINGS_STRUCT: EncodingRef = &StructEncoding; diff --git a/enc/src/array/struct_/serde.rs b/vortex/src/array/struct_/serde.rs similarity index 100% rename from enc/src/array/struct_/serde.rs rename to vortex/src/array/struct_/serde.rs diff --git a/enc/src/array/struct_/stats.rs b/vortex/src/array/struct_/stats.rs similarity index 100% rename from enc/src/array/struct_/stats.rs rename to vortex/src/array/struct_/stats.rs diff --git a/enc/src/array/typed/compress.rs b/vortex/src/array/typed/compress.rs similarity index 100% rename from enc/src/array/typed/compress.rs rename to vortex/src/array/typed/compress.rs diff --git a/enc/src/array/typed/mod.rs b/vortex/src/array/typed/mod.rs similarity index 95% rename from enc/src/array/typed/mod.rs rename to vortex/src/array/typed/mod.rs index c339679585..b9822a94c9 100644 --- a/enc/src/array/typed/mod.rs +++ b/vortex/src/array/typed/mod.rs @@ -7,7 +7,7 @@ use linkme::distributed_slice; use crate::array::{Array, ArrayRef, ArrowIterator, Encoding, EncodingId, EncodingRef, ENCODINGS}; use crate::compress::EncodingCompression; use crate::dtype::DType; -use crate::error::EncResult; +use crate::error::VortexResult; use crate::formatter::{ArrayDisplay, ArrayFormatter}; use crate::scalar::Scalar; use crate::serde::{ArraySerde, EncodingSerde}; @@ -85,7 +85,7 @@ impl Array for TypedArray { Stats::new(&self.stats, self) } - fn scalar_at(&self, index: usize) -> EncResult> { + fn scalar_at(&self, index: usize) -> VortexResult> { let underlying = self.array.scalar_at(index)?; underlying.as_ref().cast(self.dtype()) } @@ -100,7 +100,7 @@ impl Array for TypedArray { ) } - fn slice(&self, start: usize, stop: usize) -> EncResult { + fn slice(&self, start: usize, stop: usize) -> VortexResult { Ok(Self::new(self.array.slice(start, stop)?, self.dtype.clone()).boxed()) } @@ -128,7 +128,7 @@ impl<'arr> AsRef<(dyn Array + 'arr)> for TypedArray { #[derive(Debug)] struct TypedEncoding; -pub const TYPED_ENCODING: EncodingId = EncodingId("enc.typed"); +pub const TYPED_ENCODING: EncodingId = EncodingId::new("vortex.typed"); #[distributed_slice(ENCODINGS)] static ENCODINGS_TYPED: EncodingRef = &TypedEncoding; diff --git a/enc/src/array/typed/serde.rs b/vortex/src/array/typed/serde.rs similarity index 100% rename from enc/src/array/typed/serde.rs rename to vortex/src/array/typed/serde.rs diff --git a/enc/src/array/typed/stats.rs b/vortex/src/array/typed/stats.rs similarity index 100% rename from enc/src/array/typed/stats.rs rename to vortex/src/array/typed/stats.rs diff --git a/enc/src/array/varbin/compress.rs b/vortex/src/array/varbin/compress.rs similarity index 100% rename from enc/src/array/varbin/compress.rs rename to vortex/src/array/varbin/compress.rs diff --git a/enc/src/array/varbin/mod.rs b/vortex/src/array/varbin/mod.rs similarity index 95% rename from enc/src/array/varbin/mod.rs rename to vortex/src/array/varbin/mod.rs index ba1d917201..abd028ec28 100644 --- a/enc/src/array/varbin/mod.rs +++ b/vortex/src/array/varbin/mod.rs @@ -18,7 +18,7 @@ use crate::array::{ use crate::arrow::CombineChunks; use crate::compress::EncodingCompression; use crate::dtype::{DType, IntWidth, Nullability, Signedness}; -use crate::error::{EncError, EncResult}; +use crate::error::{VortexError, VortexResult}; use crate::formatter::{ArrayDisplay, ArrayFormatter}; use crate::match_each_native_ptype; use crate::ptype::NativePType; @@ -54,9 +54,9 @@ impl VarBinArray { bytes: ArrayRef, dtype: DType, validity: Option, - ) -> EncResult { + ) -> VortexResult { if !matches!(offsets.dtype(), DType::Int(_, _, Nullability::NonNullable)) { - return Err(EncError::UnsupportedOffsetsArrayDType( + return Err(VortexError::UnsupportedOffsetsArrayDType( offsets.dtype().clone(), )); } @@ -64,10 +64,12 @@ impl VarBinArray { bytes.dtype(), DType::Int(IntWidth::_8, Signedness::Unsigned, Nullability::NonNullable) ) { - return Err(EncError::UnsupportedDataArrayDType(bytes.dtype().clone())); + return Err(VortexError::UnsupportedDataArrayDType( + bytes.dtype().clone(), + )); } if !matches!(dtype, DType::Binary(_) | DType::Utf8(_)) { - return Err(EncError::InvalidDType(dtype)); + return Err(VortexError::InvalidDType(dtype)); } let validity = validity.filter(|v| !v.is_empty()); @@ -176,7 +178,7 @@ impl VarBinArray { } } - pub fn bytes_at(&self, index: usize) -> EncResult> { + pub fn bytes_at(&self, index: usize) -> VortexResult> { check_index_bounds(self, index)?; let (start, end): (usize, usize) = if let Some(p) = self.offsets.maybe_primitive() { @@ -232,7 +234,7 @@ impl Array for VarBinArray { Stats::new(&self.stats, self) } - fn scalar_at(&self, index: usize) -> EncResult> { + fn scalar_at(&self, index: usize) -> VortexResult> { if self.is_valid(index) { self.bytes_at(index).map(|bytes| { if matches!(self.dtype, DType::Utf8(_)) { @@ -269,7 +271,7 @@ impl Array for VarBinArray { Box::new(iter::once(make_array(data))) } - fn slice(&self, start: usize, stop: usize) -> EncResult { + fn slice(&self, start: usize, stop: usize) -> VortexResult { check_slice_bounds(self, start, stop)?; Ok(VarBinArray::new( @@ -308,7 +310,7 @@ impl<'arr> AsRef<(dyn Array + 'arr)> for VarBinArray { #[derive(Debug)] struct VarBinEncoding; -pub const VARBIN_ENCODING: EncodingId = EncodingId("enc.varbin"); +pub const VARBIN_ENCODING: EncodingId = EncodingId::new("vortex.varbin"); #[distributed_slice(ENCODINGS)] static ENCODINGS_VARBIN: EncodingRef = &VarBinEncoding; diff --git a/enc/src/array/varbin/serde.rs b/vortex/src/array/varbin/serde.rs similarity index 100% rename from enc/src/array/varbin/serde.rs rename to vortex/src/array/varbin/serde.rs diff --git a/enc/src/array/varbin/stats.rs b/vortex/src/array/varbin/stats.rs similarity index 95% rename from enc/src/array/varbin/stats.rs rename to vortex/src/array/varbin/stats.rs index 3c7fab78ee..12f3743a29 100644 --- a/enc/src/array/varbin/stats.rs +++ b/vortex/src/array/varbin/stats.rs @@ -5,11 +5,11 @@ use crate::array::varbin::VarBinArray; use crate::array::varbinview::VarBinViewArray; use crate::array::Array; use crate::dtype::DType; -use crate::error::EncResult; +use crate::error::VortexResult; use crate::stats::{Stat, StatsCompute, StatsSet}; pub trait BinaryArray { - fn bytes_at(&self, index: usize) -> EncResult>; + fn bytes_at(&self, index: usize) -> VortexResult>; } impl StatsCompute for T @@ -66,13 +66,13 @@ where } impl BinaryArray for VarBinArray { - fn bytes_at(&self, index: usize) -> EncResult> { + fn bytes_at(&self, index: usize) -> VortexResult> { VarBinArray::bytes_at(self, index) } } impl BinaryArray for VarBinViewArray { - fn bytes_at(&self, index: usize) -> EncResult> { + fn bytes_at(&self, index: usize) -> VortexResult> { VarBinViewArray::bytes_at(self, index) } } diff --git a/enc/src/array/varbinview/compress.rs b/vortex/src/array/varbinview/compress.rs similarity index 100% rename from enc/src/array/varbinview/compress.rs rename to vortex/src/array/varbinview/compress.rs diff --git a/enc/src/array/varbinview/mod.rs b/vortex/src/array/varbinview/mod.rs similarity index 94% rename from enc/src/array/varbinview/mod.rs rename to vortex/src/array/varbinview/mod.rs index 3a76957d1b..544e6497b5 100644 --- a/enc/src/array/varbinview/mod.rs +++ b/vortex/src/array/varbinview/mod.rs @@ -18,7 +18,7 @@ use crate::array::{ use crate::arrow::CombineChunks; use crate::compress::EncodingCompression; use crate::dtype::{DType, IntWidth, Nullability, Signedness}; -use crate::error::{EncError, EncResult}; +use crate::error::{VortexError, VortexResult}; use crate::formatter::{ArrayDisplay, ArrayFormatter}; use crate::scalar::{NullableScalar, Scalar}; use crate::serde::{ArraySerde, EncodingSerde}; @@ -103,12 +103,12 @@ impl VarBinViewArray { data: Vec, dtype: DType, validity: Option, - ) -> EncResult { + ) -> VortexResult { if !matches!( views.dtype(), DType::Int(IntWidth::_8, Signedness::Unsigned, Nullability::NonNullable) ) { - return Err(EncError::UnsupportedOffsetsArrayDType( + return Err(VortexError::UnsupportedOffsetsArrayDType( views.dtype().clone(), )); } @@ -118,12 +118,12 @@ impl VarBinViewArray { d.dtype(), DType::Int(IntWidth::_8, Signedness::Unsigned, Nullability::NonNullable) ) { - return Err(EncError::UnsupportedDataArrayDType(d.dtype().clone())); + return Err(VortexError::UnsupportedDataArrayDType(d.dtype().clone())); } } if !matches!(dtype, DType::Binary(_) | DType::Utf8(_)) { - return Err(EncError::InvalidDType(dtype)); + return Err(VortexError::InvalidDType(dtype)); } let validity = validity.filter(|v| !v.is_empty()); check_validity_buffer(validity.as_ref())?; @@ -184,7 +184,7 @@ impl VarBinViewArray { self.validity.as_ref() } - pub fn bytes_at(&self, index: usize) -> EncResult> { + pub fn bytes_at(&self, index: usize) -> VortexResult> { let view = self.view_at(index); unsafe { if view.inlined.size > 12 { @@ -246,7 +246,7 @@ impl Array for VarBinViewArray { Stats::new(&self.stats, self) } - fn scalar_at(&self, index: usize) -> EncResult> { + fn scalar_at(&self, index: usize) -> VortexResult> { if self.is_valid(index) { self.bytes_at(index).map(|bytes| { if matches!(self.dtype, DType::Utf8(_)) { @@ -289,7 +289,7 @@ impl Array for VarBinViewArray { Box::new(iter::once(data_arr)) } - fn slice(&self, start: usize, stop: usize) -> EncResult { + fn slice(&self, start: usize, stop: usize) -> VortexResult { check_slice_bounds(self, start, stop)?; Ok(Self { @@ -329,7 +329,7 @@ impl<'arr> AsRef<(dyn Array + 'arr)> for VarBinViewArray { #[derive(Debug)] struct VarBinViewEncoding; -pub const VARBINVIEW_ENCODING: EncodingId = EncodingId("enc.varbinview"); +pub const VARBINVIEW_ENCODING: EncodingId = EncodingId::new("vortex.varbinview"); #[distributed_slice(ENCODINGS)] static ENCODINGS_VARBINVIEW: EncodingRef = &VarBinViewEncoding; diff --git a/enc/src/array/varbinview/serde.rs b/vortex/src/array/varbinview/serde.rs similarity index 100% rename from enc/src/array/varbinview/serde.rs rename to vortex/src/array/varbinview/serde.rs diff --git a/enc/src/arrow/aligned_iter.rs b/vortex/src/arrow/aligned_iter.rs similarity index 100% rename from enc/src/arrow/aligned_iter.rs rename to vortex/src/arrow/aligned_iter.rs diff --git a/enc/src/arrow/compute/mod.rs b/vortex/src/arrow/compute/mod.rs similarity index 100% rename from enc/src/arrow/compute/mod.rs rename to vortex/src/arrow/compute/mod.rs diff --git a/enc/src/arrow/compute/repeat.rs b/vortex/src/arrow/compute/repeat.rs similarity index 100% rename from enc/src/arrow/compute/repeat.rs rename to vortex/src/arrow/compute/repeat.rs diff --git a/enc/src/arrow/convert.rs b/vortex/src/arrow/convert.rs similarity index 92% rename from enc/src/arrow/convert.rs rename to vortex/src/arrow/convert.rs index 0fadd4cf81..a0a032a4f2 100644 --- a/enc/src/arrow/convert.rs +++ b/vortex/src/arrow/convert.rs @@ -9,7 +9,7 @@ use crate::array::typed::TypedArray; use crate::array::{Array, ArrayRef}; use crate::dtype::DType::*; use crate::dtype::{DType, FloatWidth, IntWidth, Nullability, TimeUnit}; -use crate::error::{EncError, EncResult}; +use crate::error::{VortexError, VortexResult}; use crate::ptype::PType; impl From for ArrayRef { @@ -39,9 +39,9 @@ impl From for ArrayRef { } impl TryFrom for DType { - type Error = EncError; + type Error = VortexError; - fn try_from(value: SchemaRef) -> EncResult { + fn try_from(value: SchemaRef) -> VortexResult { Ok(Struct( value .fields() @@ -52,15 +52,15 @@ impl TryFrom for DType { .fields() .iter() .map(|f| f.data_type().try_into_dtype(f.is_nullable())) - .collect::>>()?, + .collect::>>()?, )) } } impl TryFrom<&DataType> for PType { - type Error = EncError; + type Error = VortexError; - fn try_from(value: &DataType) -> EncResult { + fn try_from(value: &DataType) -> VortexResult { match value { DataType::Int8 => Ok(PType::I8), DataType::Int16 => Ok(PType::I16), @@ -79,17 +79,17 @@ impl TryFrom<&DataType> for PType { DataType::Date32 => Ok(PType::I32), DataType::Date64 => Ok(PType::I64), DataType::Duration(_) => Ok(PType::I64), - _ => Err(EncError::InvalidArrowDataType(value.clone())), + _ => Err(VortexError::InvalidArrowDataType(value.clone())), } } } pub trait TryIntoDType { - fn try_into_dtype(self, is_nullable: bool) -> EncResult; + fn try_into_dtype(self, is_nullable: bool) -> VortexResult; } impl TryIntoDType for &DataType { - fn try_into_dtype(self, is_nullable: bool) -> EncResult { + fn try_into_dtype(self, is_nullable: bool) -> VortexResult { use crate::dtype::Nullability::*; use crate::dtype::Signedness::*; @@ -124,7 +124,7 @@ impl TryIntoDType for &DataType { f.iter().map(|f| Arc::new(f.name().clone())).collect(), f.iter() .map(|f| f.data_type().try_into_dtype(f.is_nullable())) - .collect::>>()?, + .collect::>>()?, )), DataType::Dictionary(_, v) => v.as_ref().try_into_dtype(is_nullable), DataType::Decimal128(p, s) | DataType::Decimal256(p, s) => { @@ -136,20 +136,20 @@ impl TryIntoDType for &DataType { Box::new(f.get(1).unwrap().try_into()?), Nullable, )), - _ => Err(EncError::InvalidArrowDataType(e.data_type().clone())), + _ => Err(VortexError::InvalidArrowDataType(e.data_type().clone())), }, DataType::RunEndEncoded(_, v) => v.try_into(), DataType::Duration(_) | DataType::Interval(_) | DataType::Union(_, _) => { - Err(EncError::InvalidArrowDataType(self.clone())) + Err(VortexError::InvalidArrowDataType(self.clone())) } } } } impl TryFrom<&FieldRef> for DType { - type Error = EncError; + type Error = VortexError; - fn try_from(value: &FieldRef) -> EncResult { + fn try_from(value: &FieldRef) -> VortexResult { value.data_type().try_into_dtype(value.is_nullable()) } } diff --git a/enc/src/arrow/mod.rs b/vortex/src/arrow/mod.rs similarity index 100% rename from enc/src/arrow/mod.rs rename to vortex/src/arrow/mod.rs diff --git a/enc/src/compress.rs b/vortex/src/compress.rs similarity index 100% rename from enc/src/compress.rs rename to vortex/src/compress.rs diff --git a/enc/src/compute/add.rs b/vortex/src/compute/add.rs similarity index 80% rename from enc/src/compute/add.rs rename to vortex/src/compute/add.rs index f9d3f058ea..6ef1002204 100644 --- a/enc/src/compute/add.rs +++ b/vortex/src/compute/add.rs @@ -1,14 +1,14 @@ use crate::array::constant::ConstantArray; use crate::array::{Array, ArrayKind, ArrayRef}; -use crate::error::{EncError, EncResult}; +use crate::error::{VortexError, VortexResult}; use crate::scalar::Scalar; // TODO(ngates): convert this to arithmetic operations with macro over the kernel. -pub fn add(lhs: &dyn Array, rhs: &dyn Array) -> EncResult { +pub fn add(lhs: &dyn Array, rhs: &dyn Array) -> VortexResult { // Check that the arrays are the same length. let length = lhs.len(); if rhs.len() != length { - return Err(EncError::LengthMismatch); + return Err(VortexError::LengthMismatch); } match (ArrayKind::from(lhs), ArrayKind::from(rhs)) { @@ -21,7 +21,7 @@ pub fn add(lhs: &dyn Array, rhs: &dyn Array) -> EncResult { } } -pub fn add_scalar(lhs: &dyn Array, rhs: &dyn Scalar) -> EncResult { +pub fn add_scalar(lhs: &dyn Array, rhs: &dyn Scalar) -> VortexResult { match ArrayKind::from(lhs) { ArrayKind::Constant(lhs) => { Ok(ConstantArray::new(add_scalars(lhs.scalar(), rhs)?, lhs.len()).boxed()) @@ -30,7 +30,7 @@ pub fn add_scalar(lhs: &dyn Array, rhs: &dyn Scalar) -> EncResult { } } -pub fn add_scalars(_lhs: &dyn Scalar, _rhs: &dyn Scalar) -> EncResult> { +pub fn add_scalars(_lhs: &dyn Scalar, _rhs: &dyn Scalar) -> VortexResult> { // Might need to improve this implementation... Ok(24.into()) } diff --git a/enc/src/compute/as_contiguous.rs b/vortex/src/compute/as_contiguous.rs similarity index 83% rename from enc/src/compute/as_contiguous.rs rename to vortex/src/compute/as_contiguous.rs index e55be4b32f..e1e1295e8c 100644 --- a/enc/src/compute/as_contiguous.rs +++ b/vortex/src/compute/as_contiguous.rs @@ -5,16 +5,16 @@ use crate::array::bool::{BoolArray, BOOL_ENCODING}; use crate::array::downcast::DowncastArrayBuiltin; use crate::array::primitive::{PrimitiveArray, PRIMITIVE_ENCODING}; use crate::array::{Array, ArrayRef}; -use crate::error::{EncError, EncResult}; +use crate::error::{VortexError, VortexResult}; use crate::ptype::{match_each_native_ptype, NativePType}; -use spiral_alloc::{AlignedVec, ALIGNED_ALLOCATOR}; +use vortex_alloc::{AlignedVec, ALIGNED_ALLOCATOR}; -pub fn as_contiguous(arrays: Vec) -> EncResult { +pub fn as_contiguous(arrays: Vec) -> VortexResult { if arrays.is_empty() { - return Err(EncError::ComputeError("No arrays to concatenate".into())); + return Err(VortexError::ComputeError("No arrays to concatenate".into())); } if !arrays.iter().map(|chunk| chunk.encoding().id()).all_equal() { - return Err(EncError::ComputeError( + return Err(VortexError::ComputeError( "Chunks have differing encodings".into(), )); } @@ -26,13 +26,13 @@ pub fn as_contiguous(arrays: Vec) -> EncResult { PRIMITIVE_ENCODING => { Ok(primitive_as_contiguous(arrays.iter().map(|a| a.as_primitive()).collect())?.boxed()) } - _ => Err(EncError::ComputeError( + _ => Err(VortexError::ComputeError( format!("as_contiguous not supported for {:?}", arrays[0].encoding()).into(), ))?, } } -fn bool_as_contiguous(arrays: Vec<&BoolArray>) -> EncResult { +fn bool_as_contiguous(arrays: Vec<&BoolArray>) -> VortexResult { // TODO(ngates): implement a HasValidity trait to avoid this duplicate code. let validity = if arrays.iter().all(|a| a.validity().is_none()) { None @@ -60,9 +60,9 @@ fn bool_as_contiguous(arrays: Vec<&BoolArray>) -> EncResult { )) } -fn primitive_as_contiguous(arrays: Vec<&PrimitiveArray>) -> EncResult { +fn primitive_as_contiguous(arrays: Vec<&PrimitiveArray>) -> VortexResult { if !arrays.iter().map(|chunk| (*chunk).ptype()).all_equal() { - return Err(EncError::ComputeError( + return Err(VortexError::ComputeError( "Chunks have differing ptypes".into(), )); } diff --git a/enc/src/compute/cast.rs b/vortex/src/compute/cast.rs similarity index 100% rename from enc/src/compute/cast.rs rename to vortex/src/compute/cast.rs diff --git a/enc/src/compute/mod.rs b/vortex/src/compute/mod.rs similarity index 100% rename from enc/src/compute/mod.rs rename to vortex/src/compute/mod.rs diff --git a/enc/src/compute/repeat.rs b/vortex/src/compute/repeat.rs similarity index 100% rename from enc/src/compute/repeat.rs rename to vortex/src/compute/repeat.rs diff --git a/enc/src/compute/search_sorted.rs b/vortex/src/compute/search_sorted.rs similarity index 96% rename from enc/src/compute/search_sorted.rs rename to vortex/src/compute/search_sorted.rs index 4d902ddf3d..42677dd9e7 100644 --- a/enc/src/compute/search_sorted.rs +++ b/vortex/src/compute/search_sorted.rs @@ -1,5 +1,5 @@ use crate::array::Array; -use crate::error::EncResult; +use crate::error::VortexResult; use crate::polars::IntoPolarsSeries; use crate::polars::IntoPolarsValue; use crate::scalar::Scalar; @@ -24,7 +24,7 @@ pub fn search_sorted_usize( indices: &dyn Array, index: usize, side: SearchSortedSide, -) -> EncResult { +) -> VortexResult { let enc_scalar: Box = index.into(); // Convert index into correctly typed Arrow scalar. let enc_scalar = enc_scalar.cast(indices.dtype())?; diff --git a/enc/src/compute/take.rs b/vortex/src/compute/take.rs similarity index 61% rename from enc/src/compute/take.rs rename to vortex/src/compute/take.rs index 0ca937a178..cbcaa458f1 100644 --- a/enc/src/compute/take.rs +++ b/vortex/src/compute/take.rs @@ -1,18 +1,18 @@ use crate::array::{Array, ArrayRef}; -use crate::error::{EncError, EncResult}; +use crate::error::{VortexError, VortexResult}; pub trait TakeFn { - fn take(&self, indices: &dyn Array) -> EncResult; + fn take(&self, indices: &dyn Array) -> VortexResult; } -pub fn take(array: &dyn Array, indices: &dyn Array) -> EncResult { +pub fn take(array: &dyn Array, indices: &dyn Array) -> VortexResult { array .compute() .and_then(|c| c.take()) .map(|t| t.take(indices)) .unwrap_or_else(|| { // TODO(ngates): default implementation of decode and then try again - Err(EncError::ComputeError( + Err(VortexError::ComputeError( format!("take not implemented for {}", &array.encoding().id()).into(), )) }) diff --git a/enc/src/dtype.rs b/vortex/src/dtype.rs similarity index 100% rename from enc/src/dtype.rs rename to vortex/src/dtype.rs diff --git a/enc/src/encode.rs b/vortex/src/encode.rs similarity index 100% rename from enc/src/encode.rs rename to vortex/src/encode.rs diff --git a/enc/src/error.rs b/vortex/src/error.rs similarity index 90% rename from enc/src/error.rs rename to vortex/src/error.rs index c767dbf17d..9fa652a6f9 100644 --- a/enc/src/error.rs +++ b/vortex/src/error.rs @@ -44,7 +44,7 @@ impl Display for ErrString { } #[derive(Debug, thiserror::Error, PartialEq)] -pub enum EncError { +pub enum VortexError { #[error("index {0} out of bounds from {1} to {2}")] OutOfBounds(usize, usize, usize), #[error("arguments have different lengths")] @@ -77,7 +77,7 @@ pub enum EncError { IndexArrayMustBeStrictSorted, } -pub type EncResult = Result; +pub type VortexResult = Result; // Wrap up external errors so that we can implement a dumb PartialEq #[derive(Debug)] @@ -89,9 +89,9 @@ impl PartialEq for ArrowError { } } -impl From for EncError { +impl From for VortexError { fn from(err: arrow::error::ArrowError) -> Self { - EncError::ArrowError(ArrowError(err)) + VortexError::ArrowError(ArrowError(err)) } } @@ -105,8 +105,8 @@ impl PartialEq for PolarsError { } } -impl From for EncError { +impl From for VortexError { fn from(err: polars_core::error::PolarsError) -> Self { - EncError::PolarsError(PolarsError(err)) + VortexError::PolarsError(PolarsError(err)) } } diff --git a/enc/src/formatter.rs b/vortex/src/formatter.rs similarity index 93% rename from enc/src/formatter.rs rename to vortex/src/formatter.rs index 3941479869..495476d520 100644 --- a/enc/src/formatter.rs +++ b/vortex/src/formatter.rs @@ -74,6 +74,6 @@ mod test { #[test] fn primitive_array() { let arr = PrimitiveArray::from_vec((0..100).collect()).boxed(); - assert_eq!(format!("{}", arr), "enc.primitive(signed_int(32)), len=100, nbytes=400 B (100.00%)\n[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]...\n") + assert_eq!(format!("{}", arr), "vortex.primitive(signed_int(32)), len=100, nbytes=400 B (100.00%)\n[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]...\n") } } diff --git a/enc/src/lib.rs b/vortex/src/lib.rs similarity index 100% rename from enc/src/lib.rs rename to vortex/src/lib.rs diff --git a/enc/src/polars.rs b/vortex/src/polars.rs similarity index 100% rename from enc/src/polars.rs rename to vortex/src/polars.rs diff --git a/enc/src/ptype.rs b/vortex/src/ptype.rs similarity index 94% rename from enc/src/ptype.rs rename to vortex/src/ptype.rs index 2be958cbcd..4755955b20 100644 --- a/enc/src/ptype.rs +++ b/vortex/src/ptype.rs @@ -5,7 +5,7 @@ use arrow::datatypes::ArrowNativeType; use half::f16; use crate::dtype::{DType, FloatWidth, IntWidth, Signedness}; -use crate::error::{EncError, EncResult}; +use crate::error::{VortexError, VortexResult}; use crate::scalar::Scalar; #[derive(Debug, Clone, Copy, PartialEq, PartialOrd, Eq, Hash)] @@ -33,7 +33,7 @@ pub trait NativePType: + Default + ArrowNativeType + RefUnwindSafe - + TryFrom, Error = EncError> + + TryFrom, Error = VortexError> { const PTYPE: PType; } @@ -124,9 +124,9 @@ impl PType { } impl TryFrom<&DType> for PType { - type Error = EncError; + type Error = VortexError; - fn try_from(value: &DType) -> EncResult { + fn try_from(value: &DType) -> VortexResult { use Signedness::*; match value { DType::Int(w, s, _) => match w { @@ -162,7 +162,7 @@ impl TryFrom<&DType> for PType { FloatWidth::_32 => Ok(PType::F32), FloatWidth::_64 => Ok(PType::F64), }, - _ => Err(EncError::InvalidDType(value.clone())), + _ => Err(VortexError::InvalidDType(value.clone())), } } } diff --git a/enc/src/sampling.rs b/vortex/src/sampling.rs similarity index 100% rename from enc/src/sampling.rs rename to vortex/src/sampling.rs diff --git a/enc/src/scalar/arrow.rs b/vortex/src/scalar/arrow.rs similarity index 100% rename from enc/src/scalar/arrow.rs rename to vortex/src/scalar/arrow.rs diff --git a/enc/src/scalar/binary.rs b/vortex/src/scalar/binary.rs similarity index 85% rename from enc/src/scalar/binary.rs rename to vortex/src/scalar/binary.rs index 0a9d73879b..ecaded22fc 100644 --- a/enc/src/scalar/binary.rs +++ b/vortex/src/scalar/binary.rs @@ -1,5 +1,5 @@ use crate::dtype::{DType, Nullability}; -use crate::error::{EncError, EncResult}; +use crate::error::{VortexError, VortexResult}; use crate::scalar::Scalar; use std::any::Any; use std::fmt::{Display, Formatter}; @@ -50,7 +50,7 @@ impl Scalar for BinaryScalar { &DType::Binary(Nullability::NonNullable) } - fn cast(&self, _dtype: &DType) -> EncResult> { + fn cast(&self, _dtype: &DType) -> VortexResult> { todo!() } @@ -66,26 +66,26 @@ impl From> for Box { } impl TryFrom> for Vec { - type Error = EncError; + type Error = VortexError; fn try_from(value: Box) -> Result { let dtype = value.dtype().clone(); let scalar = value .into_any() .downcast::() - .map_err(|_| EncError::InvalidDType(dtype))?; + .map_err(|_| VortexError::InvalidDType(dtype))?; Ok(scalar.value) } } impl TryFrom<&dyn Scalar> for Vec { - type Error = EncError; + type Error = VortexError; fn try_from(value: &dyn Scalar) -> Result { if let Some(scalar) = value.as_any().downcast_ref::() { Ok(scalar.value.clone()) } else { - Err(EncError::InvalidDType(value.dtype().clone())) + Err(VortexError::InvalidDType(value.dtype().clone())) } } } diff --git a/enc/src/scalar/bool.rs b/vortex/src/scalar/bool.rs similarity index 82% rename from enc/src/scalar/bool.rs rename to vortex/src/scalar/bool.rs index 97b5183ac8..5a5d1be662 100644 --- a/enc/src/scalar/bool.rs +++ b/vortex/src/scalar/bool.rs @@ -2,7 +2,7 @@ use std::any::Any; use std::fmt::{Display, Formatter}; use crate::dtype::{DType, Nullability}; -use crate::error::{EncError, EncResult}; +use crate::error::{VortexError, VortexResult}; use crate::scalar::{NullableScalar, Scalar}; #[derive(Debug, Clone, PartialEq, PartialOrd)] @@ -51,13 +51,13 @@ impl Scalar for BoolScalar { &DType::Bool(Nullability::NonNullable) } - fn cast(&self, dtype: &DType) -> EncResult> { + fn cast(&self, dtype: &DType) -> VortexResult> { match dtype { DType::Bool(Nullability::NonNullable) => Ok(self.clone().boxed()), DType::Bool(Nullability::Nullable) => { Ok(NullableScalar::some(self.clone().boxed()).boxed()) } - _ => Err(EncError::InvalidDType(dtype.clone())), + _ => Err(VortexError::InvalidDType(dtype.clone())), } } @@ -74,25 +74,25 @@ impl From for Box { } impl TryFrom> for bool { - type Error = EncError; + type Error = VortexError; #[inline] - fn try_from(value: Box) -> EncResult { + fn try_from(value: Box) -> VortexResult { value.as_ref().try_into() } } impl TryFrom<&dyn Scalar> for bool { - type Error = EncError; + type Error = VortexError; - fn try_from(value: &dyn Scalar) -> EncResult { + fn try_from(value: &dyn Scalar) -> VortexResult { if let Some(bool_scalar) = value .as_nonnull() .and_then(|v| v.as_any().downcast_ref::()) { Ok(bool_scalar.value()) } else { - Err(EncError::InvalidDType(value.dtype().clone())) + Err(VortexError::InvalidDType(value.dtype().clone())) } } } diff --git a/enc/src/scalar/equal.rs b/vortex/src/scalar/equal.rs similarity index 100% rename from enc/src/scalar/equal.rs rename to vortex/src/scalar/equal.rs diff --git a/enc/src/scalar/list.rs b/vortex/src/scalar/list.rs similarity index 85% rename from enc/src/scalar/list.rs rename to vortex/src/scalar/list.rs index 57cd58c9a8..8636ae6926 100644 --- a/enc/src/scalar/list.rs +++ b/vortex/src/scalar/list.rs @@ -4,7 +4,7 @@ use std::fmt::{Display, Formatter}; use itertools::Itertools; use crate::dtype::{DType, Nullability}; -use crate::error::{EncError, EncResult}; +use crate::error::{VortexError, VortexResult}; use crate::scalar::{NullableScalar, Scalar}; #[derive(Debug, Clone, PartialEq)] @@ -54,7 +54,7 @@ impl Scalar for ListScalar { &self.dtype } - fn cast(&self, dtype: &DType) -> EncResult> { + fn cast(&self, dtype: &DType) -> VortexResult> { match dtype { DType::List(field_dtype, n) => { let new_fields: Vec> = self @@ -74,7 +74,7 @@ impl Scalar for ListScalar { Nullability::Nullable => Ok(NullableScalar::some(list_scalar).boxed()), } } - _ => Err(EncError::InvalidDType(dtype.clone())), + _ => Err(VortexError::InvalidDType(dtype.clone())), } } @@ -96,8 +96,8 @@ impl>> From> for Box { } } -impl, Error = EncError>> TryFrom<&dyn Scalar> for ListScalarVec { - type Error = EncError; +impl, Error = VortexError>> TryFrom<&dyn Scalar> for ListScalarVec { + type Error = VortexError; fn try_from(value: &dyn Scalar) -> Result { if let Some(list_s) = value.as_any().downcast_ref::() { @@ -110,20 +110,22 @@ impl, Error = EncError>> TryFrom<&dyn Scalar> for Lis .try_collect()?, )) } else { - Err(EncError::InvalidDType(value.dtype().clone())) + Err(VortexError::InvalidDType(value.dtype().clone())) } } } -impl, Error = EncError>> TryFrom> for ListScalarVec { - type Error = EncError; +impl, Error = VortexError>> TryFrom> + for ListScalarVec +{ + type Error = VortexError; fn try_from(value: Box) -> Result { let value_dtype = value.dtype().clone(); let list_s = value .into_any() .downcast::() - .map_err(|_| EncError::InvalidDType(value_dtype))?; + .map_err(|_| VortexError::InvalidDType(value_dtype))?; Ok(ListScalarVec( list_s diff --git a/enc/src/scalar/localtime.rs b/vortex/src/scalar/localtime.rs similarity index 94% rename from enc/src/scalar/localtime.rs rename to vortex/src/scalar/localtime.rs index c7874d91b8..9a4ef52cf1 100644 --- a/enc/src/scalar/localtime.rs +++ b/vortex/src/scalar/localtime.rs @@ -1,5 +1,5 @@ use crate::dtype::{DType, Nullability, TimeUnit}; -use crate::error::EncResult; +use crate::error::VortexResult; use crate::scalar::{PScalar, Scalar}; use std::any::Any; use std::cmp::Ordering; @@ -62,7 +62,7 @@ impl Scalar for LocalTimeScalar { &self.dtype } - fn cast(&self, _dtype: &DType) -> EncResult> { + fn cast(&self, _dtype: &DType) -> VortexResult> { todo!() } diff --git a/enc/src/scalar/mod.rs b/vortex/src/scalar/mod.rs similarity index 94% rename from enc/src/scalar/mod.rs rename to vortex/src/scalar/mod.rs index 75e3bd30a4..0f7e287aa5 100644 --- a/enc/src/scalar/mod.rs +++ b/vortex/src/scalar/mod.rs @@ -13,7 +13,7 @@ pub use struct_::*; pub use utf8::*; use crate::dtype::DType; -use crate::error::EncResult; +use crate::error::VortexResult; use crate::ptype::NativePType; mod arrow; @@ -44,7 +44,7 @@ pub trait Scalar: Display + Debug + dyn_clone::DynClone + Send + Sync + 'static /// the logical type. fn dtype(&self) -> &DType; - fn cast(&self, dtype: &DType) -> EncResult>; + fn cast(&self, dtype: &DType) -> VortexResult>; fn nbytes(&self) -> usize; } diff --git a/enc/src/scalar/null.rs b/vortex/src/scalar/null.rs similarity index 91% rename from enc/src/scalar/null.rs rename to vortex/src/scalar/null.rs index cde103f400..8d0ea4418d 100644 --- a/enc/src/scalar/null.rs +++ b/vortex/src/scalar/null.rs @@ -2,7 +2,7 @@ use std::any::Any; use std::fmt::{Display, Formatter}; use crate::dtype::DType; -use crate::error::EncResult; +use crate::error::VortexResult; use crate::scalar::{NullableScalar, Scalar}; #[derive(Debug, Clone, PartialEq)] @@ -52,7 +52,7 @@ impl Scalar for NullScalar { &DType::Null } - fn cast(&self, dtype: &DType) -> EncResult> { + fn cast(&self, dtype: &DType) -> VortexResult> { Ok(NullableScalar::none(dtype.clone()).boxed()) } diff --git a/enc/src/scalar/nullable.rs b/vortex/src/scalar/nullable.rs similarity index 87% rename from enc/src/scalar/nullable.rs rename to vortex/src/scalar/nullable.rs index db878e7e26..61ffb6a5d8 100644 --- a/enc/src/scalar/nullable.rs +++ b/vortex/src/scalar/nullable.rs @@ -3,7 +3,7 @@ use std::fmt::{Display, Formatter}; use std::mem::size_of; use crate::dtype::DType; -use crate::error::{EncError, EncResult}; +use crate::error::{VortexError, VortexResult}; use crate::scalar::{NullScalar, Scalar}; #[derive(Debug, Clone, PartialEq, PartialOrd)] @@ -63,7 +63,7 @@ impl Scalar for NullableScalar { } } - fn cast(&self, _dtype: &DType) -> EncResult> { + fn cast(&self, _dtype: &DType) -> VortexResult> { todo!() } @@ -99,14 +99,14 @@ impl>> From> for Box } } -impl, Error = EncError>> TryFrom<&dyn Scalar> +impl, Error = VortexError>> TryFrom<&dyn Scalar> for NullableScalarOption { - type Error = EncError; + type Error = VortexError; fn try_from(value: &dyn Scalar) -> Result { let Some(ns) = value.as_any().downcast_ref::() else { - return Err(EncError::InvalidDType(value.dtype().clone())); + return Err(VortexError::InvalidDType(value.dtype().clone())); }; Ok(NullableScalarOption(match ns { @@ -116,17 +116,17 @@ impl, Error = EncError>> TryFrom<&dyn Scalar> } } -impl, Error = EncError>> TryFrom> +impl, Error = VortexError>> TryFrom> for NullableScalarOption { - type Error = EncError; + type Error = VortexError; fn try_from(value: Box) -> Result { let dtype = value.dtype().clone(); let ns = value .into_any() .downcast::() - .map_err(|_| EncError::InvalidDType(dtype))?; + .map_err(|_| VortexError::InvalidDType(dtype))?; Ok(NullableScalarOption(match *ns { NullableScalar::None(_) => None, diff --git a/enc/src/scalar/ord.rs b/vortex/src/scalar/ord.rs similarity index 100% rename from enc/src/scalar/ord.rs rename to vortex/src/scalar/ord.rs diff --git a/enc/src/scalar/primitive.rs b/vortex/src/scalar/primitive.rs similarity index 78% rename from enc/src/scalar/primitive.rs rename to vortex/src/scalar/primitive.rs index 9d61969f67..088f4de22c 100644 --- a/enc/src/scalar/primitive.rs +++ b/vortex/src/scalar/primitive.rs @@ -5,7 +5,7 @@ use std::mem::size_of; use half::f16; use crate::dtype::{DType, Nullability}; -use crate::error::{EncError, EncResult}; +use crate::error::{VortexError, VortexResult}; use crate::ptype::PType; use crate::scalar::{LocalTimeScalar, Scalar}; @@ -43,14 +43,14 @@ impl PScalar { // General conversion function that handles casting primitive scalar to non primitive. // If target dtype can be converted to ptype you should use cast_ptype. - pub fn cast_dtype(&self, dtype: DType) -> EncResult> { + pub fn cast_dtype(&self, dtype: DType) -> VortexResult> { macro_rules! from_int { ($dtype:ident , $ps:ident) => { match $dtype { DType::LocalTime(w, Nullability::NonNullable) => { Ok(LocalTimeScalar::new($ps.clone(), w.clone()).boxed()) } - _ => Err(EncError::InvalidDType($dtype.clone())), + _ => Err(VortexError::InvalidDType($dtype.clone())), } }; } @@ -60,11 +60,11 @@ impl PScalar { | p @ PScalar::U64(_) | p @ PScalar::I32(_) | p @ PScalar::I64(_) => from_int!(dtype, p), - _ => Err(EncError::InvalidDType(dtype.clone())), + _ => Err(VortexError::InvalidDType(dtype.clone())), } } - pub fn cast_ptype(&self, ptype: PType) -> EncResult> { + pub fn cast_ptype(&self, ptype: PType) -> VortexResult> { macro_rules! from_int { ($ptype:ident , $v:ident) => { match $ptype { @@ -89,7 +89,7 @@ impl PScalar { PType::F16 => Ok((f16::from_f32(*$v as f32)).into()), PType::F32 => Ok((*$v as f32).into()), PType::F64 => Ok((*$v as f64).into()), - _ => Err(EncError::InvalidDType(ptype.into())), + _ => Err(VortexError::InvalidDType(ptype.into())), } }; } @@ -107,7 +107,7 @@ impl PScalar { PType::F16 => Ok((*v).into()), PType::F32 => Ok(v.to_f32().into()), PType::F64 => Ok(v.to_f64().into()), - _ => Err(EncError::InvalidDType(ptype.into())), + _ => Err(VortexError::InvalidDType(ptype.into())), }, PScalar::F32(v) => from_floating!(ptype, v), PScalar::F64(v) => from_floating!(ptype, v), @@ -151,8 +151,8 @@ impl Scalar for PScalar { self.ptype().into() } - fn cast(&self, dtype: &DType) -> EncResult> { - let ptype: EncResult = dtype.try_into(); + fn cast(&self, dtype: &DType) -> VortexResult> { + let ptype: VortexResult = dtype.try_into(); ptype .and_then(|p| self.cast_ptype(p)) .or_else(|_| self.cast_dtype(dtype.clone())) @@ -173,28 +173,28 @@ macro_rules! pscalar { } impl TryFrom> for $T { - type Error = EncError; + type Error = VortexError; #[inline] - fn try_from(value: Box) -> EncResult { + fn try_from(value: Box) -> VortexResult { value.as_ref().try_into() } } impl TryFrom<&dyn Scalar> for $T { - type Error = EncError; + type Error = VortexError; - fn try_from(value: &dyn Scalar) -> EncResult { + fn try_from(value: &dyn Scalar) -> VortexResult { if let Some(pscalar) = value .as_nonnull() .and_then(|v| v.as_any().downcast_ref::()) { match pscalar { PScalar::$ptype(v) => Ok(*v), - _ => Err(EncError::InvalidDType(pscalar.ptype().into())), + _ => Err(VortexError::InvalidDType(pscalar.ptype().into())), } } else { - Err(EncError::InvalidDType(value.dtype().clone())) + Err(VortexError::InvalidDType(value.dtype().clone())) } } } @@ -221,17 +221,17 @@ impl From for Box { } impl TryFrom> for usize { - type Error = EncError; + type Error = VortexError; - fn try_from(value: Box) -> EncResult { + fn try_from(value: Box) -> VortexResult { value.as_ref().try_into() } } impl TryFrom<&dyn Scalar> for usize { - type Error = EncError; + type Error = VortexError; - fn try_from(value: &dyn Scalar) -> EncResult { + fn try_from(value: &dyn Scalar) -> VortexResult { macro_rules! match_each_pscalar_integer { ($self:expr, | $_:tt $pscalar:ident | $($body:tt)*) => ({ macro_rules! __with_pscalar__ {( $_ $pscalar:ident ) => ( $($body)* )} @@ -244,7 +244,7 @@ impl TryFrom<&dyn Scalar> for usize { PScalar::I16(v) => __with_pscalar__! { v }, PScalar::I32(v) => __with_pscalar__! { v }, PScalar::I64(v) => __with_pscalar__! { v }, - _ => Err(EncError::InvalidDType($self.ptype().into())), + _ => Err(VortexError::InvalidDType($self.ptype().into())), } }) } @@ -255,12 +255,12 @@ impl TryFrom<&dyn Scalar> for usize { { match_each_pscalar_integer!(pscalar, |$V| { if is_negative(*$V) { - return Err(EncError::ComputeError("required positive integer".into())); + return Err(VortexError::ComputeError("required positive integer".into())); } Ok(*$V as usize) }) } else { - Err(EncError::InvalidDType(value.dtype().clone())) + Err(VortexError::InvalidDType(value.dtype().clone())) } } } @@ -268,17 +268,17 @@ impl TryFrom<&dyn Scalar> for usize { impl Display for PScalar { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { - PScalar::U8(p) => write!(f, "{}", p), - PScalar::U16(p) => write!(f, "{}", p), - PScalar::U32(p) => write!(f, "{}", p), - PScalar::U64(p) => write!(f, "{}", p), - PScalar::I8(p) => write!(f, "{}", p), - PScalar::I16(p) => write!(f, "{}", p), - PScalar::I32(p) => write!(f, "{}", p), - PScalar::I64(p) => write!(f, "{}", p), - PScalar::F16(p) => write!(f, "{}", p), - PScalar::F32(p) => write!(f, "{}", p), - PScalar::F64(p) => write!(f, "{}", p), + PScalar::U8(p) => Display::fmt(p, f), + PScalar::U16(p) => Display::fmt(p, f), + PScalar::U32(p) => Display::fmt(p, f), + PScalar::U64(p) => Display::fmt(p, f), + PScalar::I8(p) => Display::fmt(p, f), + PScalar::I16(p) => Display::fmt(p, f), + PScalar::I32(p) => Display::fmt(p, f), + PScalar::I64(p) => Display::fmt(p, f), + PScalar::F16(p) => Display::fmt(p, f), + PScalar::F32(p) => Display::fmt(p, f), + PScalar::F64(p) => Display::fmt(p, f), } } } @@ -286,7 +286,7 @@ impl Display for PScalar { #[cfg(test)] mod test { use crate::dtype::{DType, IntWidth, Nullability, Signedness}; - use crate::error::EncError; + use crate::error::VortexError; use crate::ptype::PType; use crate::scalar::Scalar; @@ -300,7 +300,9 @@ mod test { let scalar: Box = (-10i16).into(); assert_eq!( scalar.as_ref().try_into(), - Err::(EncError::ComputeError("required positive integer".into())) + Err::(VortexError::ComputeError( + "required positive integer".into() + )) ); } diff --git a/enc/src/scalar/serde.rs b/vortex/src/scalar/serde.rs similarity index 100% rename from enc/src/scalar/serde.rs rename to vortex/src/scalar/serde.rs diff --git a/enc/src/scalar/struct_.rs b/vortex/src/scalar/struct_.rs similarity index 90% rename from enc/src/scalar/struct_.rs rename to vortex/src/scalar/struct_.rs index 0e9a73f01c..f0761e3303 100644 --- a/enc/src/scalar/struct_.rs +++ b/vortex/src/scalar/struct_.rs @@ -5,7 +5,7 @@ use std::fmt::{Display, Formatter}; use itertools::Itertools; use crate::dtype::DType; -use crate::error::{EncError, EncResult}; +use crate::error::{VortexError, VortexResult}; use crate::scalar::Scalar; #[derive(Debug, Clone, PartialEq)] @@ -57,11 +57,11 @@ impl Scalar for StructScalar { &self.dtype } - fn cast(&self, dtype: &DType) -> EncResult> { + fn cast(&self, dtype: &DType) -> VortexResult> { match dtype { DType::Struct(names, field_dtypes) => { if field_dtypes.len() != self.values.len() { - return Err(EncError::InvalidDType(dtype.clone())); + return Err(VortexError::InvalidDType(dtype.clone())); } let new_fields: Vec> = self @@ -77,7 +77,7 @@ impl Scalar for StructScalar { ); Ok(StructScalar::new(new_type, new_fields).boxed()) } - _ => Err(EncError::InvalidDType(dtype.clone())), + _ => Err(VortexError::InvalidDType(dtype.clone())), } } diff --git a/enc/src/scalar/utf8.rs b/vortex/src/scalar/utf8.rs similarity index 86% rename from enc/src/scalar/utf8.rs rename to vortex/src/scalar/utf8.rs index 7b8ea7b32b..975b5ebdaf 100644 --- a/enc/src/scalar/utf8.rs +++ b/vortex/src/scalar/utf8.rs @@ -2,7 +2,7 @@ use std::any::Any; use std::fmt::{Display, Formatter}; use crate::dtype::{DType, Nullability}; -use crate::error::{EncError, EncResult}; +use crate::error::{VortexError, VortexResult}; use crate::scalar::Scalar; #[derive(Debug, Clone, PartialEq, PartialOrd)] @@ -50,7 +50,7 @@ impl Scalar for Utf8Scalar { &DType::Utf8(Nullability::NonNullable) } - fn cast(&self, _dtype: &DType) -> EncResult> { + fn cast(&self, _dtype: &DType) -> VortexResult> { todo!() } @@ -72,26 +72,26 @@ impl From<&str> for Box { } impl TryFrom> for String { - type Error = EncError; + type Error = VortexError; fn try_from(value: Box) -> Result { let dtype = value.dtype().clone(); let scalar = value .into_any() .downcast::() - .map_err(|_| EncError::InvalidDType(dtype))?; + .map_err(|_| VortexError::InvalidDType(dtype))?; Ok(scalar.value) } } impl TryFrom<&dyn Scalar> for String { - type Error = EncError; + type Error = VortexError; fn try_from(value: &dyn Scalar) -> Result { if let Some(scalar) = value.as_any().downcast_ref::() { Ok(scalar.value().to_string()) } else { - Err(EncError::InvalidDType(value.dtype().clone())) + Err(VortexError::InvalidDType(value.dtype().clone())) } } } diff --git a/enc/src/serde/dtype.rs b/vortex/src/serde/dtype.rs similarity index 100% rename from enc/src/serde/dtype.rs rename to vortex/src/serde/dtype.rs diff --git a/enc/src/serde/mod.rs b/vortex/src/serde/mod.rs similarity index 100% rename from enc/src/serde/mod.rs rename to vortex/src/serde/mod.rs diff --git a/enc/src/stats.rs b/vortex/src/stats.rs similarity index 96% rename from enc/src/stats.rs rename to vortex/src/stats.rs index bdc2ce0856..e559f2d41c 100644 --- a/enc/src/stats.rs +++ b/vortex/src/stats.rs @@ -5,7 +5,7 @@ use std::sync::RwLock; use itertools::Itertools; -use crate::error::{EncError, EncResult}; +use crate::error::{VortexError, VortexResult}; use crate::ptype::NativePType; use crate::scalar::{ListScalarVec, Scalar}; @@ -38,10 +38,10 @@ impl StatsSet { StatsSet(HashMap::from([(stat, value)])) } - fn get_as, Error = EncError>>( + fn get_as, Error = VortexError>>( &self, stat: &Stat, - ) -> EncResult> { + ) -> VortexResult> { self.0.get(stat).map(|v| T::try_from(v.clone())).transpose() } @@ -241,7 +241,10 @@ impl<'a> Stats<'a> { self.cache.read().unwrap().0.get(stat).cloned() } - pub fn get_as, Error = EncError>>(&self, stat: &Stat) -> Option { + pub fn get_as, Error = VortexError>>( + &self, + stat: &Stat, + ) -> Option { self.get(stat).map(|v| T::try_from(v).unwrap()) } @@ -263,14 +266,14 @@ impl<'a> Stats<'a> { .map(|v| T::try_from(v.cast(T::PTYPE.into()).unwrap()).unwrap()) } - pub fn get_or_compute_as, Error = EncError>>( + pub fn get_or_compute_as, Error = VortexError>>( &self, stat: &Stat, ) -> Option { self.get_or_compute(stat).map(|v| T::try_from(v).unwrap()) } - pub fn get_or_compute_or, Error = EncError>>( + pub fn get_or_compute_or, Error = VortexError>>( &self, default: T, stat: &Stat, diff --git a/zig/c-abi/include/prelude.h b/zig/c-abi/include/prelude.h index 50e430e34d..311864e68d 100644 --- a/zig/c-abi/include/prelude.h +++ b/zig/c-abi/include/prelude.h @@ -5,9 +5,9 @@ #include "float.h" #include "assert.h" -#ifndef SPIRAL_ALIGNMENT -#define SPIRAL_ALIGNMENT 128 -#endif // SPIRAL_ALIGNMENT +#ifndef VORTEX_ALIGNMENT +#define VORTEX_ALIGNMENT 128 +#endif // VORTEX_ALIGNMENT #ifndef FL_MIN_ALIGNMENT #define FL_MIN_ALIGNMENT 64 diff --git a/zig/c-abi/test_wrapper.zig b/zig/c-abi/test_wrapper.zig index 39eb49e560..b7862e9684 100644 --- a/zig/c-abi/test_wrapper.zig +++ b/zig/c-abi/test_wrapper.zig @@ -7,7 +7,7 @@ const c = @cImport({ }); test "alignment 128" { - try std.testing.expectEqual(c.SPIRAL_ALIGNMENT, 128); + try std.testing.expectEqual(c.VORTEX_ALIGNMENT, 128); } test "result status" { @@ -95,11 +95,11 @@ test "run end encoding" { const ints = [_]V{ 1, 1, 1, 2, 3, 4, 4, 5 }; const numRuns = 5; - const valuesOut: []align(128) V = try gpa.alignedAlloc(V, c.SPIRAL_ALIGNMENT, 5); + const valuesOut: []align(128) V = try gpa.alignedAlloc(V, c.VORTEX_ALIGNMENT, 5); defer gpa.free(valuesOut); const valuesBuf = abi.ByteBuffer.initFromSlice(valuesOut); - const runEndsOut: []align(128) u32 = try gpa.alignedAlloc(u32, c.SPIRAL_ALIGNMENT, 5); + const runEndsOut: []align(128) u32 = try gpa.alignedAlloc(u32, c.VORTEX_ALIGNMENT, 5); defer gpa.free(runEndsOut); const runEndsBuf = abi.ByteBuffer.initFromSlice(runEndsOut); @@ -126,7 +126,7 @@ test "run end encoding" { const runEnds = [_]u32{ 3, 4, 5, 7, 8 }; try std.testing.expectEqualSlices(u32, &runEnds, runEndsOut); - const decodeOut: []align(128) V = try gpa.alignedAlloc(V, c.SPIRAL_ALIGNMENT, ints.len); + const decodeOut: []align(128) V = try gpa.alignedAlloc(V, c.VORTEX_ALIGNMENT, ints.len); defer gpa.free(decodeOut); const decodeBuf = abi.ByteBuffer.initFromSlice(decodeOut); @@ -161,11 +161,11 @@ test "alp encoding" { 4.123457612347956123084712340569871234, // this will be an exception that needs patching }; - const valuesOut: []align(128) i64 = try gpa.alignedAlloc(i64, c.SPIRAL_ALIGNMENT, floats.len); + const valuesOut: []align(128) i64 = try gpa.alignedAlloc(i64, c.VORTEX_ALIGNMENT, floats.len); defer gpa.free(valuesOut); const valuesBuf = abi.ByteBuffer.initFromSlice(valuesOut); - const bitsetOut: []align(128) u8 = try gpa.alignedAlloc(u8, c.SPIRAL_ALIGNMENT, (floats.len + 7) / 8); + const bitsetOut: []align(128) u8 = try gpa.alignedAlloc(u8, c.VORTEX_ALIGNMENT, (floats.len + 7) / 8); defer gpa.free(bitsetOut); const bitsetBuf = abi.ByteBuffer.initFromSlice(bitsetOut); @@ -213,7 +213,7 @@ test "alp encoding" { } try std.testing.expectEqual(bitset.get(floats.len - 1), 1); - const decodeOut: []align(128) f64 = try gpa.alignedAlloc(f64, c.SPIRAL_ALIGNMENT, floats.len); + const decodeOut: []align(128) f64 = try gpa.alignedAlloc(f64, c.VORTEX_ALIGNMENT, floats.len); defer gpa.free(decodeOut); const decodeBuf = abi.ByteBuffer.initFromSlice(decodeOut); @@ -240,7 +240,7 @@ test "fastlanes ffor encoding" { const gpa = std.testing.allocator; const ints = blk: { const stack = [_]i32{ 1, -2, 3, -4, 5, 100_000 }; - const aligned = try gpa.alignedAlloc(i32, c.SPIRAL_ALIGNMENT, 6); + const aligned = try gpa.alignedAlloc(i32, c.VORTEX_ALIGNMENT, 6); @memcpy(aligned, &stack); break :blk aligned; }; @@ -249,7 +249,7 @@ test "fastlanes ffor encoding" { const numBits = 4; const bytesNeeded = c.codecz_flbp_encodedSizeInBytes_u32(ints.len, numBits); - const encodedOut: []align(128) u8 = try gpa.alignedAlloc(u8, c.SPIRAL_ALIGNMENT, bytesNeeded); + const encodedOut: []align(128) u8 = try gpa.alignedAlloc(u8, c.VORTEX_ALIGNMENT, bytesNeeded); defer gpa.free(encodedOut); const encodedBuf = abi.ByteBuffer.initFromSlice(encodedOut); @@ -271,12 +271,12 @@ test "fastlanes ffor encoding" { const exceptionsOut: []align(128) u32 = try gpa.alignedAlloc( u32, - c.SPIRAL_ALIGNMENT, + c.VORTEX_ALIGNMENT, encodeResult.num_exceptions + 1, ); defer gpa.free(exceptionsOut); const exceptionsBuf = abi.ByteBuffer.initFromSlice(exceptionsOut); - const bitsetOut: []align(128) u8 = try gpa.alignedAlloc(u8, c.SPIRAL_ALIGNMENT, (ints.len + 7) / 8); + const bitsetOut: []align(128) u8 = try gpa.alignedAlloc(u8, c.VORTEX_ALIGNMENT, (ints.len + 7) / 8); defer gpa.free(bitsetOut); const bitsetBuf = abi.ByteBuffer.initFromSlice(bitsetOut); @@ -309,7 +309,7 @@ test "fastlanes ffor encoding" { } try std.testing.expectEqual(bitset.get(ints.len - 1), 1); - const decodeOut: []align(128) i32 = try gpa.alignedAlloc(i32, c.SPIRAL_ALIGNMENT, ints.len); + const decodeOut: []align(128) i32 = try gpa.alignedAlloc(i32, c.VORTEX_ALIGNMENT, ints.len); defer gpa.free(decodeOut); const decodeBuf = abi.ByteBuffer.initFromSlice(decodeOut); diff --git a/zig/c-abi/types.zig b/zig/c-abi/types.zig index 5be6ce797c..ecf4ab2108 100644 --- a/zig/c-abi/types.zig +++ b/zig/c-abi/types.zig @@ -5,7 +5,7 @@ const c = @cImport({ @cInclude("wrapper.h"); }); -pub const Alignment: u29 = c.SPIRAL_ALIGNMENT; +pub const Alignment: u29 = c.VORTEX_ALIGNMENT; pub const FastLanesMinAlignment: u29 = c.FL_MIN_ALIGNMENT; pub const CodecError = error{ diff --git a/zig/c-abi/wrapper.zig b/zig/c-abi/wrapper.zig index 43012f1966..8c5187cee0 100644 --- a/zig/c-abi/wrapper.zig +++ b/zig/c-abi/wrapper.zig @@ -10,7 +10,7 @@ const c = @cImport({ const abi = @import("abi"); // aliases -const OutputAlign: u29 = c.SPIRAL_ALIGNMENT; +const OutputAlign: u29 = c.VORTEX_ALIGNMENT; const AlpExponents = abi.AlpExponents; const ByteBuffer = abi.ByteBuffer; const ResultStatus = abi.ResultStatus;