From 55f52f69251053a7ab8a43ca9004d3667aab3359 Mon Sep 17 00:00:00 2001 From: Andrew Duffy Date: Wed, 7 Aug 2024 23:40:05 -0400 Subject: [PATCH] revert --- bench-vortex/benches/datafusion_benchmark.rs | 6 +++--- bench-vortex/benches/random_access.rs | 5 +++-- bench-vortex/src/data_downloads.rs | 6 +++--- bench-vortex/src/lib.rs | 12 ++++++------ bench-vortex/src/reader.rs | 15 +++++++-------- bench-vortex/src/tpch/mod.rs | 9 +++++---- bench-vortex/src/vortex_utils.rs | 3 +-- 7 files changed, 28 insertions(+), 28 deletions(-) diff --git a/bench-vortex/benches/datafusion_benchmark.rs b/bench-vortex/benches/datafusion_benchmark.rs index 5429781d36..5e81802bcb 100644 --- a/bench-vortex/benches/datafusion_benchmark.rs +++ b/bench-vortex/benches/datafusion_benchmark.rs @@ -15,8 +15,8 @@ use datafusion::prelude::{col, DataFrame, SessionContext}; use lazy_static::lazy_static; use vortex::compress::CompressionStrategy; use vortex::encoding::EncodingRef; -use vortex::{Array, Context, IntoArray, ToArrayData}; -use vortex_datafusion::{VortexMemTable, VortexMemTableOptions}; +use vortex::{Array, Context}; +use vortex_datafusion::memory::{VortexMemTable, VortexMemTableOptions}; use vortex_dict::DictEncoding; use vortex_fastlanes::{BitPackedEncoding, DeltaEncoding, FoREncoding}; use vortex_sampling_compressor::compressors::bitpacked::BitPackedCompressor; @@ -81,7 +81,7 @@ fn toy_dataset_arrow() -> RecordBatch { } fn toy_dataset_vortex(compress: bool) -> Array { - let uncompressed = toy_dataset_arrow().to_array_data().into_array(); + let uncompressed = toy_dataset_arrow().into(); if !compress { return uncompressed; diff --git a/bench-vortex/benches/random_access.rs b/bench-vortex/benches/random_access.rs index 7d856ecc26..3d69ad9c2c 100644 --- a/bench-vortex/benches/random_access.rs +++ b/bench-vortex/benches/random_access.rs @@ -8,6 +8,7 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; use mimalloc::MiMalloc; use object_store::aws::AmazonS3Builder; use object_store::local::LocalFileSystem; +use object_store::ObjectStore; use tokio::runtime::Runtime; #[global_allocator] @@ -31,7 +32,7 @@ fn random_access_vortex(c: &mut Criterion) { .iter(|| async { black_box(take_vortex_tokio(&taxi_vortex, &INDICES).await.unwrap()) }) }); - let local_fs = LocalFileSystem::new(); + let local_fs = Arc::new(LocalFileSystem::new()) as Arc; let local_fs_path = object_store::path::Path::from_filesystem_path(&taxi_vortex).unwrap(); group.bench_function("localfs", |b| { b.to_async(Runtime::new().unwrap()).iter(|| async { @@ -43,7 +44,7 @@ fn random_access_vortex(c: &mut Criterion) { }) }); - let r2_fs = AmazonS3Builder::from_env().build().unwrap(); + let r2_fs = Arc::new(AmazonS3Builder::from_env().build().unwrap()) as Arc; let r2_path = object_store::path::Path::from_url_path(taxi_vortex.file_name().unwrap().to_str().unwrap()) .unwrap(); diff --git a/bench-vortex/src/data_downloads.rs b/bench-vortex/src/data_downloads.rs index dc07b745f0..2cdeea796d 100644 --- a/bench-vortex/src/data_downloads.rs +++ b/bench-vortex/src/data_downloads.rs @@ -9,9 +9,9 @@ use bzip2::read::BzDecoder; use log::info; use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; use tokio::runtime::Runtime; -use vortex::array::chunked::ChunkedArray; +use vortex::array::ChunkedArray; use vortex::arrow::FromArrowType; -use vortex::{IntoArray, ToArrayData}; +use vortex::{Array, IntoArray}; use vortex_dtype::DType; use vortex_error::{VortexError, VortexResult}; use vortex_serde::io::TokioAdapter; @@ -46,7 +46,7 @@ pub fn data_vortex_uncompressed(fname_out: &str, downloaded_data: PathBuf) -> Pa let array = ChunkedArray::try_new( reader .into_iter() - .map(|batch_result| batch_result.unwrap().to_array_data().into_array()) + .map(|batch_result| Array::from(batch_result.unwrap())) .collect(), dtype, ) diff --git a/bench-vortex/src/lib.rs b/bench-vortex/src/lib.rs index fe2b91142e..216ef859b1 100644 --- a/bench-vortex/src/lib.rs +++ b/bench-vortex/src/lib.rs @@ -15,11 +15,11 @@ use log::{info, LevelFilter}; use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; use parquet::arrow::ProjectionMask; use simplelog::{ColorChoice, Config, TermLogger, TerminalMode}; -use vortex::array::chunked::ChunkedArray; +use vortex::array::ChunkedArray; use vortex::arrow::FromArrowType; use vortex::compress::CompressionStrategy; use vortex::encoding::EncodingRef; -use vortex::{Array, Context, IntoArray, ToArrayData}; +use vortex::{Array, Context, IntoArray}; use vortex_alp::ALPEncoding; use vortex_datetime_parts::DateTimePartsEncoding; use vortex_dict::DictEncoding; @@ -188,7 +188,7 @@ pub fn compress_taxi_data() -> Array { let chunks = reader .into_iter() .map(|batch_result| batch_result.unwrap()) - .map(|batch| batch.to_array_data().into_array()) + .map(Array::from) .map(|array| { uncompressed_size += array.nbytes(); compressor.compress(&array).unwrap() @@ -262,7 +262,7 @@ mod test { use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; use vortex::arrow::FromArrowArray; use vortex::compress::CompressionStrategy; - use vortex::{ArrayData, IntoArray, IntoCanonical}; + use vortex::{Array, IntoCanonical}; use vortex_sampling_compressor::SamplingCompressor; use crate::taxi_data::taxi_data_parquet; @@ -285,7 +285,7 @@ mod test { for record_batch in reader.map(|batch_result| batch_result.unwrap()) { let struct_arrow: ArrowStructArray = record_batch.into(); let arrow_array: ArrowArrayRef = Arc::new(struct_arrow); - let vortex_array = ArrayData::from_arrow(arrow_array.clone(), false).into_array(); + let vortex_array = Array::from_arrow(arrow_array.clone(), false); let vortex_as_arrow = vortex_array.into_canonical().unwrap().into_arrow(); assert_eq!(vortex_as_arrow.deref(), arrow_array.deref()); } @@ -304,7 +304,7 @@ mod test { for record_batch in reader.map(|batch_result| batch_result.unwrap()) { let struct_arrow: ArrowStructArray = record_batch.into(); let arrow_array: ArrowArrayRef = Arc::new(struct_arrow); - let vortex_array = ArrayData::from_arrow(arrow_array.clone(), false).into_array(); + let vortex_array = Array::from_arrow(arrow_array.clone(), false); let compressed = compressor.compress(&vortex_array).unwrap(); let compressed_as_arrow = compressed.into_canonical().unwrap().into_arrow(); diff --git a/bench-vortex/src/reader.rs b/bench-vortex/src/reader.rs index bd00a4e9ee..bef82d4e57 100644 --- a/bench-vortex/src/reader.rs +++ b/bench-vortex/src/reader.rs @@ -22,12 +22,11 @@ use parquet::arrow::async_reader::{AsyncFileReader, ParquetObjectReader}; use parquet::arrow::ParquetRecordBatchStreamBuilder; use serde::{Deserialize, Serialize}; use stream::StreamExt; -use vortex::array::chunked::ChunkedArray; -use vortex::array::primitive::PrimitiveArray; +use vortex::array::{ChunkedArray, PrimitiveArray}; use vortex::arrow::FromArrowType; use vortex::compress::CompressionStrategy; use vortex::stream::ArrayStreamExt; -use vortex::{Array, IntoArray, IntoCanonical, ToArrayData}; +use vortex::{Array, IntoArray, IntoCanonical}; use vortex_buffer::Buffer; use vortex_dtype::DType; use vortex_error::{vortex_err, VortexResult}; @@ -99,7 +98,7 @@ pub fn compress_parquet_to_vortex(parquet_path: &Path) -> VortexResult( ) } -pub async fn take_vortex_object_store( - fs: &O, +pub async fn take_vortex_object_store( + fs: &Arc, path: &object_store::path::Path, indices: &[u64], ) -> VortexResult { @@ -171,7 +170,7 @@ pub async fn take_vortex_object_store( .take_rows(&indices_array) .await?; // For equivalence.... we flatten to make sure we're not cheating too much. - Ok(taken.into_canonical()?.into_array()) + Ok(taken.into_canonical()?.into()) } pub async fn take_vortex_tokio(path: &Path, indices: &[u64]) -> VortexResult { @@ -182,7 +181,7 @@ pub async fn take_vortex_tokio(path: &Path, indices: &[u64]) -> VortexResult