diff --git a/bench-vortex/src/lib.rs b/bench-vortex/src/lib.rs index d8621fb82b..b90e91d4cd 100644 --- a/bench-vortex/src/lib.rs +++ b/bench-vortex/src/lib.rs @@ -184,19 +184,19 @@ pub struct CompressionRunStats { impl CompressionRunStats { pub fn to_results(&self, dataset_name: String) -> Vec { - let DType::Struct(ns, fs) = &self.schema else { + let DType::Struct(st, _) = &self.schema else { unreachable!() }; self.compressed_sizes .iter() - .zip_eq(ns.iter().zip_eq(fs)) + .zip_eq(st.names().iter().zip_eq(st.dtypes().iter())) .map( |(&size, (column_name, column_type))| CompressionRunResults { dataset_name: dataset_name.clone(), file_name: self.file_name.clone(), file_type: self.file_type.to_string(), - column_name: (**column_name).clone(), + column_name: (**column_name).to_string(), column_type: column_type.to_string(), compressed_size: size, total_compressed_size: self.total_compressed_size, diff --git a/bench-vortex/src/vortex_utils.rs b/bench-vortex/src/vortex_utils.rs index b266ac07af..b24f14a6f7 100644 --- a/bench-vortex/src/vortex_utils.rs +++ b/bench-vortex/src/vortex_utils.rs @@ -16,15 +16,15 @@ pub fn vortex_chunk_sizes(path: &Path) -> VortexResult { let file = File::open(path)?; let total_compressed_size = file.metadata()?.size(); let vortex = open_vortex(path)?; - let DType::Struct(ns, _) = vortex.dtype() else { + let DType::Struct(st, _) = vortex.dtype() else { unreachable!() }; - let mut compressed_sizes = vec![0; ns.len()]; + let mut compressed_sizes = vec![0; st.names().len()]; let chunked_array = ChunkedArray::try_from(vortex).unwrap(); for chunk in chunked_array.chunks() { let struct_arr = StructArray::try_from(chunk).unwrap(); - for (i, f) in (0..struct_arr.nfields()).map(|i| (i, struct_arr.child(i).unwrap())) { + for (i, f) in (0..struct_arr.nfields()).map(|i| (i, struct_arr.field(i).unwrap())) { compressed_sizes[i] += f.nbytes() as u64; } } diff --git a/vortex-array/src/array/struct/compute.rs b/vortex-array/src/array/struct/compute.rs index 94bb83ca63..3579f23649 100644 --- a/vortex-array/src/array/struct/compute.rs +++ b/vortex-array/src/array/struct/compute.rs @@ -15,6 +15,7 @@ use crate::compute::scalar_at::{scalar_at, ScalarAtFn}; use crate::compute::slice::{slice, SliceFn}; use crate::compute::take::{take, TakeFn}; use crate::compute::ArrayCompute; +use crate::validity::Validity; use crate::ArrayTrait; use crate::{Array, ArrayDType, IntoArray, OwnedArray}; @@ -49,10 +50,10 @@ impl AsArrowArray for StructArray<'_> { .names() .iter() .zip(field_arrays.iter()) - .zip(self.fields().iter()) + .zip(self.dtypes().iter()) .map(|((name, arrow_field), vortex_field)| { Field::new( - name.as_str(), + &**name, arrow_field.data_type().clone(), vortex_field.is_nullable(), ) @@ -74,13 +75,19 @@ impl AsContiguousFn for StructArray<'_> { .iter() .map(StructArray::try_from) .collect::>>()?; - let mut fields = vec![Vec::new(); self.fields().len()]; + let mut fields = vec![Vec::new(); self.dtypes().len()]; for array in struct_arrays.iter() { - for f in 0..self.fields().len() { - fields[f].push(array.child(f).unwrap()) + for (f, field) in fields.iter_mut().enumerate() { + field.push(array.field(f).unwrap()); } } + let validity = if self.dtype().is_nullable() { + Validity::from_iter(arrays.iter().map(|a| a.with_dyn(|a| a.logical_validity()))) + } else { + Validity::NonNullable + }; + StructArray::try_new( self.names().clone(), fields @@ -88,6 +95,7 @@ impl AsContiguousFn for StructArray<'_> { .map(|field_arrays| as_contiguous(field_arrays)) .try_collect()?, self.len(), + validity, ) .map(|a| a.into_array()) } @@ -113,6 +121,7 @@ impl TakeFn for StructArray<'_> { .map(|field| take(&field, indices)) .try_collect()?, indices.len(), + self.validity().take(indices)?, ) .map(|a| a.into_array()) } @@ -124,6 +133,12 @@ impl SliceFn for StructArray<'_> { .children() .map(|field| slice(&field, start, stop)) .try_collect()?; - StructArray::try_new(self.names().clone(), fields, stop - start).map(|a| a.into_array()) + StructArray::try_new( + self.names().clone(), + fields, + stop - start, + self.validity().slice(start, stop)?, + ) + .map(|a| a.into_array()) } } diff --git a/vortex-array/src/array/struct/mod.rs b/vortex-array/src/array/struct/mod.rs index 2fd5b87d59..85fde7c674 100644 --- a/vortex-array/src/array/struct/mod.rs +++ b/vortex-array/src/array/struct/mod.rs @@ -1,9 +1,9 @@ use serde::{Deserialize, Serialize}; -use vortex_dtype::FieldNames; +use vortex_dtype::{FieldNames, Nullability, StructDType}; use vortex_error::vortex_bail; use crate::stats::ArrayStatisticsCompute; -use crate::validity::{ArrayValidity, LogicalValidity}; +use crate::validity::{ArrayValidity, LogicalValidity, Validity, ValidityMetadata}; use crate::visitor::{AcceptArrayVisitor, ArrayVisitor}; use crate::{impl_encoding, ArrayDType}; use crate::{ArrayFlatten, IntoArrayData}; @@ -15,44 +15,56 @@ impl_encoding!("vortex.struct", Struct); #[derive(Clone, Debug, Serialize, Deserialize)] pub struct StructMetadata { length: usize, + validity: ValidityMetadata, } impl StructArray<'_> { - pub fn child(&self, idx: usize) -> Option { - let DType::Struct(_, fields) = self.dtype() else { + pub fn field(&self, idx: usize) -> Option { + let DType::Struct(st, _) = self.dtype() else { unreachable!() }; - let dtype = fields.get(idx)?; + let dtype = st.dtypes().get(idx)?; self.array().child(idx, dtype) } pub fn names(&self) -> &FieldNames { - let DType::Struct(names, _fields) = self.dtype() else { + let DType::Struct(st, _) = self.dtype() else { unreachable!() }; - names + st.names() } - pub fn fields(&self) -> &[DType] { - let DType::Struct(_names, fields) = self.dtype() else { + pub fn dtypes(&self) -> &[DType] { + let DType::Struct(st, _) = self.dtype() else { unreachable!() }; - fields.as_slice() + st.dtypes() } pub fn nfields(&self) -> usize { - self.fields().len() + self.dtypes().len() + } + + pub fn validity(&self) -> Validity { + self.metadata() + .validity + .to_validity(self.array().child(self.nfields(), &Validity::DTYPE)) } } impl<'a> StructArray<'a> { pub fn children(&'a self) -> impl Iterator> { - (0..self.nfields()).map(move |idx| self.child(idx).unwrap()) + (0..self.nfields()).map(move |idx| self.field(idx).unwrap()) } } impl StructArray<'_> { - pub fn try_new(names: FieldNames, fields: Vec, length: usize) -> VortexResult { + pub fn try_new( + names: FieldNames, + fields: Vec, + length: usize, + validity: Validity, + ) -> VortexResult { if names.len() != fields.len() { vortex_bail!("Got {} names and {} fields", names.len(), fields.len()); } @@ -62,10 +74,25 @@ impl StructArray<'_> { } let field_dtypes: Vec<_> = fields.iter().map(|d| d.dtype()).cloned().collect(); + + let validity_metadata = validity.to_metadata(length)?; + + let mut children = vec![]; + children.extend(fields.into_iter().map(|a| a.into_array_data())); + if let Some(v) = validity.into_array_data() { + children.push(v); + } + Self::try_from_parts( - DType::Struct(names, field_dtypes), - StructMetadata { length }, - fields.into_iter().map(|a| a.into_array_data()).collect(), + DType::Struct( + StructDType::new(names, field_dtypes), + Nullability::NonNullable, + ), + StructMetadata { + length, + validity: validity_metadata, + }, + children.into(), StatsSet::new(), ) } @@ -80,13 +107,14 @@ impl ArrayFlatten for StructArray<'_> { self.names().clone(), (0..self.nfields()) .map(|i| { - self.child(i) + self.field(i) .expect("Missing child") .flatten() .map(|f| f.into_array()) }) .collect::>>()?, self.len(), + self.validity(), )?)) } } @@ -110,7 +138,7 @@ impl ArrayValidity for StructArray<'_> { impl AcceptArrayVisitor for StructArray<'_> { fn accept(&self, visitor: &mut dyn ArrayVisitor) -> VortexResult<()> { for (idx, name) in self.names().iter().enumerate() { - let child = self.child(idx).unwrap(); + let child = self.field(idx).unwrap(); visitor.visit_child(&format!("\"{}\"", name), &child)?; } Ok(()) diff --git a/vortex-array/src/arrow/array.rs b/vortex-array/src/arrow/array.rs index 7d8b07377a..c93cd9dc34 100644 --- a/vortex-array/src/arrow/array.rs +++ b/vortex-array/src/arrow/array.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - use arrow_array::array::{ Array as ArrowArray, ArrayRef as ArrowArrayRef, BooleanArray as ArrowBooleanArray, GenericByteArray, NullArray as ArrowNullArray, PrimitiveArray as ArrowPrimitiveArray, @@ -21,6 +19,7 @@ use arrow_array::{BinaryViewArray, GenericByteViewArray, StringViewArray}; use arrow_buffer::buffer::{NullBuffer, OffsetBuffer}; use arrow_buffer::{ArrowNativeType, Buffer, ScalarBuffer}; use arrow_schema::{DataType, TimeUnit}; +use itertools::Itertools; use vortex_dtype::DType; use vortex_dtype::NativePType; use vortex_scalar::NullScalar; @@ -176,9 +175,9 @@ impl FromArrowArray<&ArrowStructArray> for ArrayData { value .column_names() .iter() - .map(|s| s.to_string()) - .map(Arc::new) - .collect(), + .map(|s| (*s).into()) + .collect_vec() + .into(), value .columns() .iter() @@ -188,6 +187,7 @@ impl FromArrowArray<&ArrowStructArray> for ArrayData { }) .collect(), value.len(), + nulls(value.nulls(), nullable), ) .unwrap() .into_array_data() diff --git a/vortex-array/src/arrow/dtype.rs b/vortex-array/src/arrow/dtype.rs index 4b14a1d6de..f8c60bc1b2 100644 --- a/vortex-array/src/arrow/dtype.rs +++ b/vortex-array/src/arrow/dtype.rs @@ -1,10 +1,8 @@ -use std::sync::Arc; - use arrow_schema::TimeUnit as ArrowTimeUnit; use arrow_schema::{DataType, Field, SchemaRef}; use itertools::Itertools; -use vortex_dtype::PType; use vortex_dtype::{DType, Nullability}; +use vortex_dtype::{PType, StructDType}; use vortex_error::{vortex_err, VortexResult}; use crate::array::datetime::{LocalDateTimeArray, TimeUnit}; @@ -41,16 +39,20 @@ impl TryFromArrowType<&DataType> for PType { impl FromArrowType for DType { fn from_arrow(value: SchemaRef) -> Self { DType::Struct( - value - .fields() - .iter() - .map(|f| Arc::new(f.name().clone())) - .collect(), - value - .fields() - .iter() - .map(|f| DType::from_arrow(f.as_ref())) - .collect_vec(), + StructDType::new( + value + .fields() + .iter() + .map(|f| f.name().as_str().into()) + .collect_vec() + .into(), + value + .fields() + .iter() + .map(|f| DType::from_arrow(f.as_ref())) + .collect_vec(), + ), + Nullability::NonNullable, ) } } @@ -82,10 +84,16 @@ impl FromArrowType<&Field> for DType { List(Box::new(DType::from_arrow(e.as_ref())), nullability) } DataType::Struct(f) => Struct( - f.iter().map(|f| Arc::new(f.name().clone())).collect(), - f.iter() - .map(|f| DType::from_arrow(f.as_ref())) - .collect_vec(), + StructDType::new( + f.iter() + .map(|f| f.name().as_str().into()) + .collect_vec() + .into(), + f.iter() + .map(|f| DType::from_arrow(f.as_ref())) + .collect_vec(), + ), + nullability, ), _ => unimplemented!("Arrow data type not yet supported: {:?}", field.data_type()), } diff --git a/vortex-array/src/arrow/recordbatch.rs b/vortex-array/src/arrow/recordbatch.rs index 2f9e61968a..b157b5a4d6 100644 --- a/vortex-array/src/arrow/recordbatch.rs +++ b/vortex-array/src/arrow/recordbatch.rs @@ -1,9 +1,9 @@ -use std::sync::Arc; - use arrow_array::RecordBatch; +use itertools::Itertools; use crate::array::r#struct::StructArray; use crate::arrow::FromArrowArray; +use crate::validity::Validity; use crate::{ArrayData, IntoArray, IntoArrayData, ToArrayData}; impl ToArrayData for RecordBatch { @@ -12,10 +12,9 @@ impl ToArrayData for RecordBatch { self.schema() .fields() .iter() - .map(|f| f.name()) - .map(|s| s.to_owned()) - .map(Arc::new) - .collect(), + .map(|f| f.name().as_str().into()) + .collect_vec() + .into(), self.columns() .iter() .zip(self.schema().fields()) @@ -24,6 +23,7 @@ impl ToArrayData for RecordBatch { }) .collect(), self.num_rows(), + Validity::AllValid, ) .unwrap() .into_array_data() diff --git a/vortex-array/src/compress.rs b/vortex-array/src/compress.rs index 3fb44ff686..25e2b37c57 100644 --- a/vortex-array/src/compress.rs +++ b/vortex-array/src/compress.rs @@ -242,10 +242,14 @@ impl CompressCtx { .children() .map(|field| self.compress_array(&field)) .collect::>>()?; - Ok( - StructArray::try_new(strct.names().clone(), compressed_fields, strct.len())? - .into_array(), - ) + let validity = self.compress_validity(strct.validity())?; + Ok(StructArray::try_new( + strct.names().clone(), + compressed_fields, + strct.len(), + validity, + )? + .into_array()) } _ => { // Otherwise, we run sampled compression over pluggable encodings diff --git a/vortex-dtype/flatbuffers/dtype.fbs b/vortex-dtype/flatbuffers/dtype.fbs index 61cc64a026..6ceaa4a459 100644 --- a/vortex-dtype/flatbuffers/dtype.fbs +++ b/vortex-dtype/flatbuffers/dtype.fbs @@ -50,6 +50,7 @@ table Binary { table Struct_ { names: [string]; fields: [DType]; + nullability: Nullability; } table List { diff --git a/vortex-dtype/src/deserialize.rs b/vortex-dtype/src/deserialize.rs index 03aa2fbdfa..2645ab1fcc 100644 --- a/vortex-dtype/src/deserialize.rs +++ b/vortex-dtype/src/deserialize.rs @@ -1,10 +1,9 @@ -use std::sync::Arc; - +use itertools::Itertools; use vortex_error::{vortex_err, VortexError, VortexResult}; use vortex_flatbuffers::ReadFlatBuffer; -use crate::DType; use crate::{flatbuffers as fb, ExtDType, ExtID, ExtMetadata, Nullability}; +use crate::{DType, StructDType}; impl ReadFlatBuffer for DType { type Source<'a> = fb::DType<'a>; @@ -43,15 +42,19 @@ impl ReadFlatBuffer for DType { .names() .unwrap() .iter() - .map(|n| Arc::new(n.to_string())) - .collect::>(); + .map(|n| n.into()) + .collect_vec() + .into(); let fields: Vec = fb_struct .fields() .unwrap() .iter() .map(|f| DType::read_flatbuffer(&f)) .collect::>>()?; - Ok(DType::Struct(names, fields)) + Ok(DType::Struct( + StructDType::new(names, fields), + fb_struct.nullability().try_into()?, + )) } fb::Type::Extension => { let fb_ext = fb.type__as_extension().unwrap(); diff --git a/vortex-dtype/src/dtype.rs b/vortex-dtype/src/dtype.rs index 4ee0e6eb6c..abdf2cce1a 100644 --- a/vortex-dtype/src/dtype.rs +++ b/vortex-dtype/src/dtype.rs @@ -43,7 +43,7 @@ impl Display for Nullability { } } -pub type FieldNames = Vec>; +pub type FieldNames = Arc<[Arc]>; pub type Metadata = Vec; @@ -55,7 +55,7 @@ pub enum DType { Primitive(PType, Nullability), Utf8(Nullability), Binary(Nullability), - Struct(FieldNames, Vec), + Struct(StructDType, Nullability), List(Box, Nullability), Extension(ExtDType, Nullability), } @@ -79,7 +79,7 @@ impl DType { Primitive(_, n) => matches!(n, Nullable), Utf8(n) => matches!(n, Nullable), Binary(n) => matches!(n, Nullable), - Struct(_, fs) => fs.iter().all(|f| f.is_nullable()), + Struct(st, _) => st.dtypes().iter().all(|f| f.is_nullable()), List(_, n) => matches!(n, Nullable), Extension(_, n) => matches!(n, Nullable), } @@ -100,10 +100,7 @@ impl DType { Primitive(p, _) => Primitive(*p, nullability), Utf8(_) => Utf8(nullability), Binary(_) => Binary(nullability), - Struct(n, fs) => Struct( - n.clone(), - fs.iter().map(|f| f.with_nullability(nullability)).collect(), - ), + Struct(st, _) => Struct(st.clone(), nullability), List(c, _) => List(c.clone(), nullability), Extension(ext, _) => Extension(ext.clone(), nullability), } @@ -122,13 +119,15 @@ impl Display for DType { Primitive(p, n) => write!(f, "{}{}", p, n), Utf8(n) => write!(f, "utf8{}", n), Binary(n) => write!(f, "binary{}", n), - Struct(n, dt) => write!( + Struct(st, n) => write!( f, - "{{{}}}", - n.iter() - .zip(dt.iter()) + "{{{}}}{}", + st.names() + .iter() + .zip(st.dtypes().iter()) .map(|(n, dt)| format!("{}={}", n, dt)) - .join(", ") + .join(", "), + n ), List(c, n) => write!(f, "list({}){}", c, n), Extension(ext, n) => write!( @@ -144,6 +143,30 @@ impl Display for DType { } } +#[derive(Debug, Clone, PartialOrd, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct StructDType { + names: FieldNames, + dtypes: Arc<[DType]>, +} + +impl StructDType { + pub fn new(names: FieldNames, dtypes: Vec) -> Self { + Self { + names, + dtypes: dtypes.into(), + } + } + + pub fn names(&self) -> &FieldNames { + &self.names + } + + pub fn dtypes(&self) -> &Arc<[DType]> { + &self.dtypes + } +} + #[cfg(test)] mod test { use std::mem; @@ -152,6 +175,6 @@ mod test { #[test] fn size_of() { - assert_eq!(mem::size_of::(), 56); + assert_eq!(mem::size_of::(), 40); } } diff --git a/vortex-dtype/src/serialize.rs b/vortex-dtype/src/serialize.rs index 42533fcbe1..f08064de90 100644 --- a/vortex-dtype/src/serialize.rs +++ b/vortex-dtype/src/serialize.rs @@ -45,20 +45,30 @@ impl WriteFlatBuffer for DType { }, ) .as_union_value(), - DType::Struct(names, dtypes) => { - let names = names + DType::Struct(st, n) => { + let names = st + .names() .iter() - .map(|n| fbb.create_string(n.as_str())) + .map(|n| fbb.create_string(n.as_ref())) .collect_vec(); let names = Some(fbb.create_vector(&names)); - let dtypes = dtypes + let dtypes = st + .dtypes() .iter() .map(|dtype| dtype.write_flatbuffer(fbb)) .collect_vec(); let fields = Some(fbb.create_vector(&dtypes)); - fb::Struct_::create(fbb, &fb::Struct_Args { names, fields }).as_union_value() + fb::Struct_::create( + fbb, + &fb::Struct_Args { + names, + fields, + nullability: n.into(), + }, + ) + .as_union_value() } DType::List(e, n) => { let element_type = Some(e.as_ref().write_flatbuffer(fbb)); @@ -166,12 +176,11 @@ impl TryFrom for PType { #[cfg(test)] mod test { - use std::sync::Arc; use flatbuffers::root; use vortex_flatbuffers::{FlatBufferToBytes, ReadFlatBuffer}; - use crate::{flatbuffers as fb, PType}; + use crate::{flatbuffers as fb, PType, StructDType}; use crate::{DType, Nullability}; fn roundtrip_dtype(dtype: DType) { @@ -192,11 +201,14 @@ mod test { Nullability::NonNullable, )); roundtrip_dtype(DType::Struct( - vec![Arc::new("strings".into()), Arc::new("ints".into())], - vec![ - DType::Utf8(Nullability::NonNullable), - DType::Primitive(PType::U16, Nullability::Nullable), - ], + StructDType::new( + ["strings".into(), "ints".into()].into(), + vec![ + DType::Utf8(Nullability::NonNullable), + DType::Primitive(PType::U16, Nullability::Nullable), + ], + ), + Nullability::NonNullable, )) } } diff --git a/vortex-ipc/src/lib.rs b/vortex-ipc/src/lib.rs index 19198f33be..d0c8ed0a02 100644 --- a/vortex-ipc/src/lib.rs +++ b/vortex-ipc/src/lib.rs @@ -37,10 +37,10 @@ pub(crate) const fn missing(field: &'static str) -> impl FnOnce() -> VortexError #[cfg(test)] mod tests { use std::io::{Cursor, Write}; - use std::sync::Arc; use vortex::array::primitive::PrimitiveArray; use vortex::array::r#struct::StructArray; + use vortex::validity::Validity; use vortex::SerdeContext; use vortex::{IntoArray, IntoArrayData}; @@ -52,16 +52,18 @@ mod tests { fn test_write_flatbuffer() { let col = PrimitiveArray::from(vec![0, 1, 2]).into_array(); let nested_struct = StructArray::try_new( - vec![Arc::new("x".into()), Arc::new("y".into())], + ["x".into(), "y".into()].into(), vec![col.clone(), col.clone()], 3, + Validity::AllValid, ) .unwrap(); let arr = StructArray::try_new( - vec![Arc::new("a".into()), Arc::new("b".into())], + ["a".into(), "b".into()].into(), vec![col.clone(), nested_struct.into_array()], 3, + Validity::AllValid, ) .unwrap() .into_array(); diff --git a/vortex-scalar/src/lib.rs b/vortex-scalar/src/lib.rs index 50de787034..e6a4279169 100644 --- a/vortex-scalar/src/lib.rs +++ b/vortex-scalar/src/lib.rs @@ -163,6 +163,6 @@ mod test { #[test] fn size_of() { - assert_eq!(mem::size_of::(), 88); + assert_eq!(mem::size_of::(), 72); } } diff --git a/vortex-scalar/src/struct_.rs b/vortex-scalar/src/struct_.rs index 50abe21673..d52e5f771f 100644 --- a/vortex-scalar/src/struct_.rs +++ b/vortex-scalar/src/struct_.rs @@ -1,9 +1,8 @@ use std::cmp::Ordering; use std::fmt::{Display, Formatter}; -use std::sync::Arc; use itertools::Itertools; -use vortex_dtype::DType; +use vortex_dtype::{DType, FieldNames, Nullability, StructDType}; use vortex_error::{vortex_bail, vortex_err, VortexResult}; use crate::Scalar; @@ -30,17 +29,20 @@ impl StructScalar { &self.dtype } - pub fn names(&self) -> &[Arc] { - let DType::Struct(ns, _) = self.dtype() else { + pub fn names(&self) -> &FieldNames { + let DType::Struct(st, _) = self.dtype() else { unreachable!("Not a scalar dtype"); }; - ns.as_slice() + st.names() } pub fn cast(&self, dtype: &DType) -> VortexResult { match dtype { - DType::Struct(names, field_dtypes) => { - if field_dtypes.len() != self.values.len() { + DType::Struct(st, n) => { + // TODO(ngates): check nullability. + assert_eq!(Nullability::NonNullable, *n); + + if st.dtypes().len() != self.values.len() { vortex_bail!( MismatchedTypes: format!("Struct with {} fields", self.values.len()), dtype @@ -50,13 +52,16 @@ impl StructScalar { let new_fields: Vec = self .values .iter() - .zip_eq(field_dtypes.iter()) + .zip_eq(st.dtypes().iter()) .map(|(field, field_dtype)| field.cast(field_dtype)) .try_collect()?; let new_type = DType::Struct( - names.clone(), - new_fields.iter().map(|x| x.dtype().clone()).collect(), + StructDType::new( + st.names().clone(), + new_fields.iter().map(|x| x.dtype().clone()).collect(), + ), + dtype.nullability(), ); Ok(StructScalar::new(new_type, new_fields).into()) } @@ -81,10 +86,10 @@ impl PartialOrd for StructScalar { impl Display for StructScalar { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let DType::Struct(names, _) = self.dtype() else { + let DType::Struct(st, _) = self.dtype() else { unreachable!() }; - for (n, v) in names.iter().zip(self.values.iter()) { + for (n, v) in st.names().iter().zip(self.values.iter()) { write!(f, "{} = {}", n, v)?; } Ok(())