Remove unused dependencies (and bump lance) (#286)

spiraldb · May 1, 2024 · 60168b9 · 60168b9
1 parent 687a5ca
commit 60168b9
Show file tree

Hide file tree

Showing 20 changed files with 416 additions and 826 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -4,7 +4,6 @@ members = [
     "fastlanez",
     "fastlanez-sys",
     "pyvortex",
-    "vortex-alloc",
     "vortex-alp",
     "vortex-array",
     "vortex-datetime-parts",

diff --git a/bench-vortex/Cargo.toml b/bench-vortex/Cargo.toml
@@ -15,23 +15,18 @@ rust-version = { workspace = true }
 workspace = true
 
 [dependencies]
-#vortex-alp = { path = "../vortex-alp" }
-vortex-roaring = { path = "../vortex-roaring" }
-#vortex-zigzag = { path = "../vortex-zigzag" }
-arrow = { workspace = true }
 arrow-array = { workspace = true }
-arrow-csv = { workspace = true }
-arrow-data = { workspace = true }
 arrow-select = { workspace = true }
 bzip2 = { workspace = true }
 csv = { workspace = true }
 enum-iterator = { workspace = true }
 fs_extra = { workspace = true }
 humansize = { workspace = true }
 itertools = { workspace = true }
-lance = { version = "0.10.5", features = [] }
-lance-arrow-array = { package = "arrow-array", version = "50.0" }
-lance-parquet = { package = "parquet", version = "50.0", features = [] }
+lance = { version = "0.10.16", features = [] }
+#TODO(robert): Add this back once lance is behind our version of arrow again
+#lance-arrow-array = { package = "arrow-array", version = "51.0" }
+#lance-parquet = { package = "parquet", version = "51.0", features = [] }
 lazy_static = { workspace = true }
 log = { workspace = true }
 parquet = { workspace = true, features = [] }
@@ -43,12 +38,12 @@ vortex-alp = { path = "../vortex-alp" }
 vortex-array = { path = "../vortex-array" }
 vortex-datetime-parts = { path = "../vortex-datetime-parts" }
 vortex-dict = { path = "../vortex-dict" }
+vortex-dtype = { path = "../vortex-dtype" }
 vortex-error = { path = "../vortex-error", features = ["parquet"] }
 vortex-fastlanes = { path = "../vortex-fastlanes" }
 vortex-ipc = { path = "../vortex-ipc" }
 vortex-ree = { path = "../vortex-ree" }
-vortex-dtype = { path = "../vortex-dtype" }
-vortex-zigzag = { path = "../vortex-zigzag" }
+vortex-roaring = { path = "../vortex-roaring" }
 
 [dev-dependencies]
 criterion = { workspace = true }

diff --git a/bench-vortex/src/data_downloads.rs b/bench-vortex/src/data_downloads.rs
@@ -7,7 +7,6 @@ use arrow_array::RecordBatchReader;
 use bzip2::read::BzDecoder;
 use lance::dataset::WriteParams;
 use lance::Dataset;
-use lance_parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder as LanceParquetRecordBatchReaderBuilder;
 use log::info;
 use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
 use tokio::runtime::Runtime;
@@ -36,7 +35,7 @@ pub fn download_data(fname: PathBuf, data_url: &str) -> PathBuf {
 pub fn parquet_to_lance(lance_fname: &Path, parquet_file: &Path) -> VortexResult<PathBuf> {
     let write_params = WriteParams::default();
     let read = File::open(parquet_file)?;
-    let reader = LanceParquetRecordBatchReaderBuilder::try_new(read)
+    let reader = ParquetRecordBatchReaderBuilder::try_new(read)
         .unwrap()
         .build()
         .unwrap();

diff --git a/bench-vortex/src/reader.rs b/bench-vortex/src/reader.rs
@@ -14,7 +14,6 @@ use arrow_select::concat::concat_batches;
 use arrow_select::take::take_record_batch;
 use itertools::Itertools;
 use lance::Dataset;
-use lance_arrow_array::RecordBatch as LanceRecordBatch;
 use log::info;
 use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
 use tokio::runtime::Runtime;
@@ -160,13 +159,13 @@ pub fn take_parquet(path: &Path, indices: &[u64]) -> VortexResult<RecordBatch> {
     Ok(concat_batches(&schema, &batches)?)
 }
 
-pub fn take_lance(path: &Path, indices: &[u64]) -> LanceRecordBatch {
+pub fn take_lance(path: &Path, indices: &[u64]) -> RecordBatch {
     Runtime::new()
         .unwrap()
         .block_on(async_take_lance(path, indices))
 }
 
-async fn async_take_lance(path: &Path, indices: &[u64]) -> LanceRecordBatch {
+async fn async_take_lance(path: &Path, indices: &[u64]) -> RecordBatch {
     let dataset = Dataset::open(path.to_str().unwrap()).await.unwrap();
     dataset.take(indices, dataset.schema()).await.unwrap()
 }
diff --git a/pyvortex/Cargo.toml b/pyvortex/Cargo.toml
@@ -29,7 +29,6 @@ vortex-ree = { path = "../vortex-ree" }
 vortex-roaring = { path = "../vortex-roaring" }
 vortex-dtype = { path = "../vortex-dtype" }
 vortex-zigzag = { path = "../vortex-zigzag" }
-itertools = { workspace = true }
 log = { workspace = true }
 paste = { workspace = true }
 pyo3 = { workspace = true }

diff --git a/vortex-alloc/Cargo.toml b/vortex-alloc/Cargo.toml
diff --git a/vortex-alloc/src/alloc.rs b/vortex-alloc/src/alloc.rs
diff --git a/vortex-alloc/src/lib.rs b/vortex-alloc/src/lib.rs
diff --git a/vortex-alp/Cargo.toml b/vortex-alp/Cargo.toml
@@ -17,9 +17,7 @@ workspace = true
 [dependencies]
 itertools = { workspace = true }
 linkme = { workspace = true }
-log = { workspace = true }
 num-traits = { workspace = true }
-paste = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 vortex-array = { path = "../vortex-array" }
 vortex-error = { path = "../vortex-error" }

diff --git a/vortex-array/Cargo.toml b/vortex-array/Cargo.toml
@@ -19,7 +19,6 @@ path = "src/lib.rs"
 workspace = true
 
 [dependencies]
-allocator-api2 = { workspace = true }
 arrow-array = { workspace = true }
 arrow-buffer = { workspace = true }
 arrow-schema = { workspace = true }
@@ -29,15 +28,12 @@ flexbuffers = { workspace = true }
 humansize = { workspace = true }
 itertools = { workspace = true }
 lazy_static = { workspace = true }
-leb128 = { workspace = true }
 linkme = { workspace = true }
 log = { workspace = true }
 num-traits = { workspace = true }
 num_enum = { workspace = true }
 paste = { workspace = true }
 rand = { workspace = true }
-thiserror = { workspace = true }
-vortex-alloc = { path = "../vortex-alloc" }
 vortex-dtype = { path = "../vortex-dtype", features = ["serde"] }
 vortex-error = { path = "../vortex-error", features = ["flexbuffers"] }
 vortex-flatbuffers = { path = "../vortex-flatbuffers" }

diff --git a/vortex-dict/Cargo.toml b/vortex-dict/Cargo.toml
@@ -15,9 +15,7 @@ rust-version = { workspace = true }
 ahash = { workspace = true }
 hashbrown = { workspace = true }
 linkme = { workspace = true }
-log = { workspace = true }
 num-traits = { workspace = true }
-paste = { workspace = true }
 serde = { workspace = true }
 vortex-array = { path = "../vortex-array" }
 vortex-error = { path = "../vortex-error" }

diff --git a/vortex-dtype/Cargo.toml b/vortex-dtype/Cargo.toml
@@ -19,10 +19,8 @@ path = "src/lib.rs"
 flatbuffers = { workspace = true }
 half = { workspace = true }
 itertools = { workspace = true }
-linkme = { workspace = true }
 num-traits = { workspace = true }
 serde = { workspace = true, optional = true, features = ["rc"] }
-thiserror = { workspace = true }
 vortex-error = { path = "../vortex-error" }
 vortex-flatbuffers = { path = "../vortex-flatbuffers" }
 

diff --git a/vortex-fastlanes/Cargo.toml b/vortex-fastlanes/Cargo.toml
@@ -19,9 +19,7 @@ arrayref = { workspace = true }
 fastlanez = { path = "../fastlanez" }
 itertools = { workspace = true }
 linkme = { workspace = true }
-log = { workspace = true }
 num-traits = { workspace = true }
-paste = { workspace = true }
 serde = { workspace = true }
 vortex-array = { path = "../vortex-array" }
 vortex-error = { path = "../vortex-error" }

diff --git a/vortex-flatbuffers/Cargo.toml b/vortex-flatbuffers/Cargo.toml
@@ -12,7 +12,6 @@ rust-version.workspace = true
 
 [dependencies]
 flatbuffers = { workspace = true }
-vortex-error = { path = "../vortex-error" }
 
 [lints]
 workspace = true
diff --git a/vortex-ipc/Cargo.toml b/vortex-ipc/Cargo.toml
@@ -15,9 +15,7 @@ rust-version = { workspace = true }
 arrow-buffer = { workspace = true }
 flatbuffers = { workspace = true }
 itertools = { workspace = true }
-lending-iterator = "0.1.7"
 nougat = "0.2.4"
-streaming-iterator = "0.1.9"
 vortex-array = { path = "../vortex-array" }
 vortex-error = { path = "../vortex-error" }
 vortex-flatbuffers = { path = "../vortex-flatbuffers" }

diff --git a/vortex-ree/Cargo.toml b/vortex-ree/Cargo.toml
@@ -12,12 +12,9 @@ edition = { workspace = true }
 rust-version = { workspace = true }
 
 [dependencies]
-arrow-array = { workspace = true }
-arrow-buffer = { workspace = true }
 itertools = { workspace = true }
 linkme = { workspace = true }
 num-traits = { workspace = true }
-paste = { workspace = true }
 serde = { workspace = true }
 vortex-array = { path = "../vortex-array" }
 vortex-error = { path = "../vortex-error" }

diff --git a/vortex-roaring/Cargo.toml b/vortex-roaring/Cargo.toml
@@ -22,7 +22,6 @@ croaring = { workspace = true }
 num-traits = { workspace = true }
 log = { workspace = true }
 serde = { workspace = true }
-paste = { workspace = true }
 
 [lints]
 workspace = true
diff --git a/vortex-zigzag/Cargo.toml b/vortex-zigzag/Cargo.toml
@@ -13,14 +13,15 @@ rust-version = { workspace = true }
 
 [dependencies]
 linkme = { workspace = true }
-vortex-alloc = { path = "../vortex-alloc" }
 vortex-array = { path = "../vortex-array" }
 vortex-error = { path = "../vortex-error" }
-vortex-fastlanes = { path = "../vortex-fastlanes" }
 vortex-dtype = { path = "../vortex-dtype" }
 vortex-scalar = { path = "../vortex-scalar" }
 zigzag = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 
+[dev-dependencies]
+vortex-fastlanes = { path = "../vortex-fastlanes" }
+
 [lints]
 workspace = true
diff --git a/vortex-zigzag/src/compress.rs b/vortex-zigzag/src/compress.rs
@@ -3,7 +3,6 @@ use vortex::compress::{CompressConfig, CompressCtx, EncodingCompression};
 use vortex::stats::{ArrayStatistics, Stat};
 use vortex::validity::Validity;
 use vortex::{Array, IntoArray, OwnedArray};
-use vortex_alloc::{AlignedVec, ALIGNED_ALLOCATOR};
 use vortex_dtype::{NativePType, PType};
 use vortex_error::VortexResult;
 use zigzag::ZigZag as ExternalZigZag;
@@ -99,9 +98,9 @@ fn zigzag_decode_primitive<'a, T: ExternalZigZag + NativePType>(
 where
     <T as ExternalZigZag>::UInt: NativePType,
 {
-    let mut encoded: AlignedVec<T> = AlignedVec::with_capacity_in(values.len(), ALIGNED_ALLOCATOR);
+    let mut encoded = Vec::with_capacity(values.len());
     encoded.extend(values.iter().map(|v| T::decode(*v)));
-    PrimitiveArray::from_vec(encoded.to_vec(), validity)
+    PrimitiveArray::from_vec(encoded, validity)
 }
 
 #[cfg(test)]