From 38043046577fe7abf4b1e55f1041057bdcec76b4 Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs@live.co.uk>
Date: Wed, 18 Dec 2024 13:58:26 +0000
Subject: [PATCH 01/24] add a list builder

---
 vortex-array/src/builders/list.rs | 122 ++++++++++++++++++++++++++++++
 vortex-array/src/builders/mod.rs  |  15 ++--
 2 files changed, 132 insertions(+), 5 deletions(-)
 create mode 100644 vortex-array/src/builders/list.rs

diff --git a/vortex-array/src/builders/list.rs b/vortex-array/src/builders/list.rs
new file mode 100644
index 0000000000..e27ba01767
--- /dev/null
+++ b/vortex-array/src/builders/list.rs
@@ -0,0 +1,122 @@
+use std::any::Any;
+use std::sync::Arc;
+
+use vortex_dtype::{DType, Nullability, PType};
+use vortex_error::{VortexExpect, VortexResult};
+use vortex_scalar::{ListScalar, Scalar};
+
+use crate::array::ListArray;
+use crate::builders::{builder_with_capacity, ArrayBuilder, ArrayBuilderExt, BoolBuilder};
+use crate::validity::Validity;
+use crate::{ArrayData, IntoArrayData};
+
+pub struct ListBuilder {
+    value_builder: Box<dyn ArrayBuilder>,
+    index_builder: Box<dyn ArrayBuilder>,
+    validity: BoolBuilder,
+    nullability: Nullability,
+    dtype: DType,
+}
+
+impl ListBuilder {
+    pub fn with_capacity(
+        value_dtype: Arc<DType>,
+        nullability: Nullability,
+        capacity: usize,
+    ) -> Self {
+        let value_builder = builder_with_capacity(value_dtype.as_ref(), capacity);
+        let mut index_builder = if capacity < 2usize.pow(31) - 1 {
+            builder_with_capacity(
+                &DType::Primitive(PType::I32, Nullability::NonNullable),
+                capacity,
+            )
+        } else {
+            builder_with_capacity(
+                &DType::Primitive(PType::I64, Nullability::NonNullable),
+                capacity,
+            )
+        };
+
+        // The first index of the list, which is always 0 and represents an empty list.
+        index_builder.append_zero();
+
+        Self {
+            value_builder,
+            index_builder,
+            validity: BoolBuilder::with_capacity(Nullability::NonNullable, capacity),
+            nullability,
+            dtype: DType::List(value_dtype, nullability),
+        }
+    }
+
+    pub fn append_value(&mut self, value: ListScalar) -> VortexResult<()> {
+        let count = self.value_builder.len();
+        if value.is_null() {
+            self.append_null();
+            Ok(())
+        } else {
+            for scalar in value.elements() {
+                // TODO(joe): This is slow, we should be able to append multiple values at once,
+                // or the list scalar should hold an ArrayData
+                self.value_builder.append_scalar(&scalar)?;
+            }
+            self.index_builder
+                .append_scalar(&Scalar::from(count + self.value_builder.len()))
+        }
+    }
+}
+
+impl ArrayBuilder for ListBuilder {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn as_any_mut(&mut self) -> &mut dyn Any {
+        self
+    }
+
+    fn dtype(&self) -> &DType {
+        &self.dtype
+    }
+
+    fn len(&self) -> usize {
+        self.validity.len()
+    }
+
+    fn append_zeros(&mut self, n: usize) {
+        self.value_builder.append_zeros(n);
+        let count = self.value_builder.len();
+        for i in 0..n {
+            self.index_builder
+                .append_scalar(&Scalar::from(count + i + 1))
+                .vortex_expect("Failed to append index");
+        }
+        self.validity.append_values(true, n);
+    }
+
+    fn append_nulls(&mut self, n: usize) {
+        let count = self.value_builder.len();
+        for _ in 0..n {
+            // A list with a null element is can be a list with a zero-span offset and a validity
+            // bit set
+            self.index_builder
+                .append_scalar(&Scalar::from(count))
+                .vortex_expect("Failed to append null");
+            self.validity.append_null();
+        }
+    }
+
+    fn finish(&mut self) -> VortexResult<ArrayData> {
+        let validity = match self.nullability {
+            Nullability::NonNullable => Validity::NonNullable,
+            Nullability::Nullable => Validity::Array(self.validity.finish()?),
+        };
+
+        ListArray::try_new(
+            self.value_builder.finish()?,
+            self.index_builder.finish()?,
+            validity,
+        )
+        .map(ListArray::into_array)
+    }
+}
diff --git a/vortex-array/src/builders/mod.rs b/vortex-array/src/builders/mod.rs
index e27f85aa75..86f9f85424 100644
--- a/vortex-array/src/builders/mod.rs
+++ b/vortex-array/src/builders/mod.rs
@@ -1,6 +1,7 @@
 mod binary;
 mod bool;
 mod extension;
+mod list;
 mod null;
 mod primitive;
 mod struct_;
@@ -17,9 +18,11 @@ pub use utf8::*;
 use vortex_dtype::{match_each_native_ptype, DType};
 use vortex_error::{vortex_bail, vortex_err, VortexResult};
 use vortex_scalar::{
-    BinaryScalar, BoolScalar, ExtScalar, PrimitiveScalar, Scalar, StructScalar, Utf8Scalar,
+    BinaryScalar, BoolScalar, ExtScalar, ListScalar, PrimitiveScalar, Scalar, StructScalar,
+    Utf8Scalar,
 };
 
+use crate::builders::list::ListBuilder;
 use crate::builders::struct_::StructBuilder;
 use crate::ArrayData;
 
@@ -71,9 +74,7 @@ pub fn builder_with_capacity(dtype: &DType, capacity: usize) -> Box<dyn ArrayBui
             *n,
             capacity,
         )),
-        DType::List(..) => {
-            todo!()
-        }
+        DType::List(dtype, n) => Box::new(ListBuilder::with_capacity(dtype.clone(), *n, capacity)),
         DType::Extension(ext_dtype) => {
             Box::new(ExtensionBuilder::with_capacity(ext_dtype.clone(), capacity))
         }
@@ -127,7 +128,11 @@ pub trait ArrayBuilderExt: ArrayBuilder {
                 .downcast_mut::<StructBuilder>()
                 .ok_or_else(|| vortex_err!("Cannot append struct scalar to non-struct builder"))?
                 .append_value(StructScalar::try_from(scalar)?)?,
-            DType::List(..) => {}
+            DType::List(..) => self
+                .as_any_mut()
+                .downcast_mut::<ListBuilder>()
+                .ok_or_else(|| vortex_err!("Cannot append list scalar to non-list builder"))?
+                .append_value(ListScalar::try_from(scalar)?)?,
             DType::Extension(..) => self
                 .as_any_mut()
                 .downcast_mut::<ExtensionBuilder>()

From 4645e701708f9dc190f4036627b2310c0a50d644 Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs@live.co.uk>
Date: Wed, 18 Dec 2024 14:25:24 +0000
Subject: [PATCH 02/24] added tests

---
 vortex-array/src/builders/list.rs | 74 ++++++++++++++++++++++++++-----
 vortex-scalar/src/list.rs         |  9 +++-
 2 files changed, 72 insertions(+), 11 deletions(-)

diff --git a/vortex-array/src/builders/list.rs b/vortex-array/src/builders/list.rs
index e27ba01767..40053435f2 100644
--- a/vortex-array/src/builders/list.rs
+++ b/vortex-array/src/builders/list.rs
@@ -50,7 +50,6 @@ impl ListBuilder {
     }
 
     pub fn append_value(&mut self, value: ListScalar) -> VortexResult<()> {
-        let count = self.value_builder.len();
         if value.is_null() {
             self.append_null();
             Ok(())
@@ -60,10 +59,14 @@ impl ListBuilder {
                 // or the list scalar should hold an ArrayData
                 self.value_builder.append_scalar(&scalar)?;
             }
-            self.index_builder
-                .append_scalar(&Scalar::from(count + self.value_builder.len()))
+            self.append_index(self.value_builder.len() as u64)
         }
     }
+
+    fn append_index(&mut self, index: u64) -> VortexResult<()> {
+        self.index_builder
+            .append_scalar(&Scalar::from(index).cast(self.index_builder.dtype())?)
+    }
 }
 
 impl ArrayBuilder for ListBuilder {
@@ -84,11 +87,10 @@ impl ArrayBuilder for ListBuilder {
     }
 
     fn append_zeros(&mut self, n: usize) {
-        self.value_builder.append_zeros(n);
         let count = self.value_builder.len();
+        self.value_builder.append_zeros(n);
         for i in 0..n {
-            self.index_builder
-                .append_scalar(&Scalar::from(count + i + 1))
+            self.append_index((count + i + 1) as u64)
                 .vortex_expect("Failed to append index");
         }
         self.validity.append_values(true, n);
@@ -99,11 +101,10 @@ impl ArrayBuilder for ListBuilder {
         for _ in 0..n {
             // A list with a null element is can be a list with a zero-span offset and a validity
             // bit set
-            self.index_builder
-                .append_scalar(&Scalar::from(count))
-                .vortex_expect("Failed to append null");
-            self.validity.append_null();
+            self.append_index(count as u64)
+                .vortex_expect("Failed to append index");
         }
+        self.validity.append_values(false, n);
     }
 
     fn finish(&mut self) -> VortexResult<ArrayData> {
@@ -120,3 +121,56 @@ impl ArrayBuilder for ListBuilder {
         .map(ListArray::into_array)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use vortex_dtype::{DType, Nullability, PType};
+    use vortex_scalar::Scalar;
+
+    use crate::builders::list::ListBuilder;
+    use crate::builders::ArrayBuilder;
+    use crate::IntoArrayVariant;
+
+    #[test]
+    fn test_empty() {
+        let mut builder =
+            ListBuilder::with_capacity(Arc::new(PType::I32.into()), Nullability::NonNullable, 0);
+
+        let list = builder.finish().unwrap();
+        assert_eq!(list.len(), 0);
+    }
+
+    #[test]
+    fn test_values() {
+        let dtype: Arc<DType> = Arc::new(PType::I32.into());
+        let mut builder = ListBuilder::with_capacity(dtype.clone(), Nullability::NonNullable, 0);
+
+        builder
+            .append_value(
+                Scalar::list(dtype.clone(), vec![1i32.into(), 2i32.into(), 3i32.into()]).as_list(),
+            )
+            .unwrap();
+
+        builder
+            .append_value(Scalar::empty(dtype.clone()).as_list())
+            .unwrap();
+
+        builder
+            .append_value(
+                Scalar::list(dtype.clone(), vec![4i32.into(), 5i32.into(), 6i32.into()]).as_list(),
+            )
+            .unwrap();
+
+        let list = builder.finish().unwrap();
+        assert_eq!(list.len(), 3);
+
+        let list_array = list.into_list().unwrap();
+        println!("{:?}", list_array);
+
+        assert_eq!(list_array.elements_at(0).unwrap().len(), 3);
+        assert!(list_array.elements_at(1).unwrap().is_empty());
+        assert_eq!(list_array.elements_at(2).unwrap().len(), 3);
+    }
+}
diff --git a/vortex-scalar/src/list.rs b/vortex-scalar/src/list.rs
index 925cd0011b..bdc1fd0212 100644
--- a/vortex-scalar/src/list.rs
+++ b/vortex-scalar/src/list.rs
@@ -2,7 +2,7 @@ use std::ops::Deref;
 use std::sync::Arc;
 
 use vortex_dtype::DType;
-use vortex_dtype::Nullability::NonNullable;
+use vortex_dtype::Nullability::{NonNullable, Nullable};
 use vortex_error::{vortex_bail, vortex_panic, VortexError, VortexResult};
 
 use crate::value::{InnerScalarValue, ScalarValue};
@@ -89,6 +89,13 @@ impl Scalar {
             )),
         }
     }
+
+    pub fn empty(element_dtype: Arc<DType>) -> Self {
+        Self {
+            dtype: DType::List(element_dtype, Nullable),
+            value: ScalarValue(InnerScalarValue::Null),
+        }
+    }
 }
 
 impl<'a> TryFrom<&'a Scalar> for ListScalar<'a> {

From 30b757d2f98c0d04e9682d5ebe07e4731de0d902 Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs@live.co.uk>
Date: Wed, 18 Dec 2024 14:46:46 +0000
Subject: [PATCH 03/24] clippy

---
 vortex-array/src/builders/list.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/vortex-array/src/builders/list.rs b/vortex-array/src/builders/list.rs
index 40053435f2..6ac037fd7d 100644
--- a/vortex-array/src/builders/list.rs
+++ b/vortex-array/src/builders/list.rs
@@ -159,7 +159,7 @@ mod tests {
 
         builder
             .append_value(
-                Scalar::list(dtype.clone(), vec![4i32.into(), 5i32.into(), 6i32.into()]).as_list(),
+                Scalar::list(dtype, vec![4i32.into(), 5i32.into(), 6i32.into()]).as_list(),
             )
             .unwrap();
 
@@ -167,7 +167,6 @@ mod tests {
         assert_eq!(list.len(), 3);
 
         let list_array = list.into_list().unwrap();
-        println!("{:?}", list_array);
 
         assert_eq!(list_array.elements_at(0).unwrap().len(), 3);
         assert!(list_array.elements_at(1).unwrap().is_empty());

From e85127a5b7e067be074354022b41fc3aa399cc19 Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs@live.co.uk>
Date: Wed, 18 Dec 2024 14:48:26 +0000
Subject: [PATCH 04/24] pub list

---
 vortex-array/src/builders/mod.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/vortex-array/src/builders/mod.rs b/vortex-array/src/builders/mod.rs
index 86f9f85424..51f8a2e44b 100644
--- a/vortex-array/src/builders/mod.rs
+++ b/vortex-array/src/builders/mod.rs
@@ -12,6 +12,7 @@ use std::any::Any;
 pub use binary::*;
 pub use bool::*;
 pub use extension::*;
+pub use list::*;
 pub use null::*;
 pub use primitive::*;
 pub use utf8::*;

From e6919598f75648d16f4d7b15b32c34cf57bf6bdb Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs@live.co.uk>
Date: Wed, 18 Dec 2024 15:00:38 +0000
Subject: [PATCH 05/24] remove list use

---
 vortex-array/src/builders/mod.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/vortex-array/src/builders/mod.rs b/vortex-array/src/builders/mod.rs
index 51f8a2e44b..86f9f85424 100644
--- a/vortex-array/src/builders/mod.rs
+++ b/vortex-array/src/builders/mod.rs
@@ -12,7 +12,6 @@ use std::any::Any;
 pub use binary::*;
 pub use bool::*;
 pub use extension::*;
-pub use list::*;
 pub use null::*;
 pub use primitive::*;
 pub use utf8::*;

From 30296c724f2f005d099e7189c99ce472f3e58c38 Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs@live.co.uk>
Date: Wed, 18 Dec 2024 15:42:33 +0000
Subject: [PATCH 06/24] use u32

---
 vortex-array/src/builders/list.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/vortex-array/src/builders/list.rs b/vortex-array/src/builders/list.rs
index 6ac037fd7d..655db23887 100644
--- a/vortex-array/src/builders/list.rs
+++ b/vortex-array/src/builders/list.rs
@@ -25,14 +25,14 @@ impl ListBuilder {
         capacity: usize,
     ) -> Self {
         let value_builder = builder_with_capacity(value_dtype.as_ref(), capacity);
-        let mut index_builder = if capacity < 2usize.pow(31) - 1 {
+        let mut index_builder = if capacity < 2usize.pow(32) - 1 {
             builder_with_capacity(
-                &DType::Primitive(PType::I32, Nullability::NonNullable),
+                &DType::Primitive(PType::U32, Nullability::NonNullable),
                 capacity,
             )
         } else {
             builder_with_capacity(
-                &DType::Primitive(PType::I64, Nullability::NonNullable),
+                &DType::Primitive(PType::U64, Nullability::NonNullable),
                 capacity,
             )
         };

From 45c04341c6d0dfc0ad2272cbd070c43cd38c6195 Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs@live.co.uk>
Date: Wed, 18 Dec 2024 16:37:08 +0000
Subject: [PATCH 07/24] generic list

---
 vortex-array/src/builders/list.rs | 61 +++++++++++++++++--------------
 vortex-array/src/builders/mod.rs  |  8 +++-
 2 files changed, 40 insertions(+), 29 deletions(-)

diff --git a/vortex-array/src/builders/list.rs b/vortex-array/src/builders/list.rs
index 655db23887..4b2c06e4cf 100644
--- a/vortex-array/src/builders/list.rs
+++ b/vortex-array/src/builders/list.rs
@@ -1,41 +1,40 @@
 use std::any::Any;
 use std::sync::Arc;
 
-use vortex_dtype::{DType, Nullability, PType};
+use num_traits::{AsPrimitive, PrimInt, Unsigned};
+use vortex_dtype::{DType, NativePType, Nullability};
 use vortex_error::{VortexExpect, VortexResult};
 use vortex_scalar::{ListScalar, Scalar};
 
 use crate::array::ListArray;
-use crate::builders::{builder_with_capacity, ArrayBuilder, ArrayBuilderExt, BoolBuilder};
+use crate::builders::{
+    builder_with_capacity, ArrayBuilder, ArrayBuilderExt, BoolBuilder, PrimitiveBuilder,
+};
 use crate::validity::Validity;
 use crate::{ArrayData, IntoArrayData};
 
-pub struct ListBuilder {
+pub struct ListBuilder<O: PrimInt + Unsigned + NativePType> {
     value_builder: Box<dyn ArrayBuilder>,
-    index_builder: Box<dyn ArrayBuilder>,
+    index_builder: PrimitiveBuilder<O>,
     validity: BoolBuilder,
     nullability: Nullability,
     dtype: DType,
 }
 
-impl ListBuilder {
+impl<O> ListBuilder<O>
+where
+    O: PrimInt + Unsigned + NativePType + 'static,
+    Scalar: From<O>,
+    usize: AsPrimitive<O>,
+{
     pub fn with_capacity(
         value_dtype: Arc<DType>,
         nullability: Nullability,
         capacity: usize,
     ) -> Self {
-        let value_builder = builder_with_capacity(value_dtype.as_ref(), capacity);
-        let mut index_builder = if capacity < 2usize.pow(32) - 1 {
-            builder_with_capacity(
-                &DType::Primitive(PType::U32, Nullability::NonNullable),
-                capacity,
-            )
-        } else {
-            builder_with_capacity(
-                &DType::Primitive(PType::U64, Nullability::NonNullable),
-                capacity,
-            )
-        };
+        // I would expect the list to have more than one value per index
+        let value_builder = builder_with_capacity(value_dtype.as_ref(), 2 * capacity);
+        let mut index_builder = PrimitiveBuilder::with_capacity(nullability, capacity);
 
         // The first index of the list, which is always 0 and represents an empty list.
         index_builder.append_zero();
@@ -59,17 +58,21 @@ impl ListBuilder {
                 // or the list scalar should hold an ArrayData
                 self.value_builder.append_scalar(&scalar)?;
             }
-            self.append_index(self.value_builder.len() as u64)
+            self.append_index(self.value_builder.len().as_())
         }
     }
 
-    fn append_index(&mut self, index: u64) -> VortexResult<()> {
-        self.index_builder
-            .append_scalar(&Scalar::from(index).cast(self.index_builder.dtype())?)
+    fn append_index(&mut self, index: O) -> VortexResult<()> {
+        self.index_builder.append_scalar(&Scalar::from(index))
     }
 }
 
-impl ArrayBuilder for ListBuilder {
+impl<O> ArrayBuilder for ListBuilder<O>
+where
+    O: PrimInt + Unsigned + NativePType + 'static,
+    Scalar: From<O>,
+    usize: AsPrimitive<O>,
+{
     fn as_any(&self) -> &dyn Any {
         self
     }
@@ -90,7 +93,7 @@ impl ArrayBuilder for ListBuilder {
         let count = self.value_builder.len();
         self.value_builder.append_zeros(n);
         for i in 0..n {
-            self.append_index((count + i + 1) as u64)
+            self.append_index((count + i + 1).as_())
                 .vortex_expect("Failed to append index");
         }
         self.validity.append_values(true, n);
@@ -101,7 +104,7 @@ impl ArrayBuilder for ListBuilder {
         for _ in 0..n {
             // A list with a null element is can be a list with a zero-span offset and a validity
             // bit set
-            self.append_index(count as u64)
+            self.append_index(count.as_())
                 .vortex_expect("Failed to append index");
         }
         self.validity.append_values(false, n);
@@ -135,8 +138,11 @@ mod tests {
 
     #[test]
     fn test_empty() {
-        let mut builder =
-            ListBuilder::with_capacity(Arc::new(PType::I32.into()), Nullability::NonNullable, 0);
+        let mut builder = ListBuilder::<u32>::with_capacity(
+            Arc::new(PType::I32.into()),
+            Nullability::NonNullable,
+            0,
+        );
 
         let list = builder.finish().unwrap();
         assert_eq!(list.len(), 0);
@@ -145,7 +151,8 @@ mod tests {
     #[test]
     fn test_values() {
         let dtype: Arc<DType> = Arc::new(PType::I32.into());
-        let mut builder = ListBuilder::with_capacity(dtype.clone(), Nullability::NonNullable, 0);
+        let mut builder =
+            ListBuilder::<u32>::with_capacity(dtype.clone(), Nullability::NonNullable, 0);
 
         builder
             .append_value(
diff --git a/vortex-array/src/builders/mod.rs b/vortex-array/src/builders/mod.rs
index 86f9f85424..fe6172c4b7 100644
--- a/vortex-array/src/builders/mod.rs
+++ b/vortex-array/src/builders/mod.rs
@@ -74,7 +74,11 @@ pub fn builder_with_capacity(dtype: &DType, capacity: usize) -> Box<dyn ArrayBui
             *n,
             capacity,
         )),
-        DType::List(dtype, n) => Box::new(ListBuilder::with_capacity(dtype.clone(), *n, capacity)),
+        DType::List(dtype, n) => Box::new(ListBuilder::<u64>::with_capacity(
+            dtype.clone(),
+            *n,
+            capacity,
+        )),
         DType::Extension(ext_dtype) => {
             Box::new(ExtensionBuilder::with_capacity(ext_dtype.clone(), capacity))
         }
@@ -130,7 +134,7 @@ pub trait ArrayBuilderExt: ArrayBuilder {
                 .append_value(StructScalar::try_from(scalar)?)?,
             DType::List(..) => self
                 .as_any_mut()
-                .downcast_mut::<ListBuilder>()
+                .downcast_mut::<ListBuilder<u64>>()
                 .ok_or_else(|| vortex_err!("Cannot append list scalar to non-list builder"))?
                 .append_value(ListScalar::try_from(scalar)?)?,
             DType::Extension(..) => self

From 6a5482a71b8ef036c66fcb7901cfce6c33ea1615 Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs@live.co.uk>
Date: Wed, 18 Dec 2024 16:38:03 +0000
Subject: [PATCH 08/24] remove unsigned

---
 vortex-array/src/builders/list.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/vortex-array/src/builders/list.rs b/vortex-array/src/builders/list.rs
index 4b2c06e4cf..f5809b4bdb 100644
--- a/vortex-array/src/builders/list.rs
+++ b/vortex-array/src/builders/list.rs
@@ -1,7 +1,7 @@
 use std::any::Any;
 use std::sync::Arc;
 
-use num_traits::{AsPrimitive, PrimInt, Unsigned};
+use num_traits::{AsPrimitive, PrimInt};
 use vortex_dtype::{DType, NativePType, Nullability};
 use vortex_error::{VortexExpect, VortexResult};
 use vortex_scalar::{ListScalar, Scalar};
@@ -13,7 +13,7 @@ use crate::builders::{
 use crate::validity::Validity;
 use crate::{ArrayData, IntoArrayData};
 
-pub struct ListBuilder<O: PrimInt + Unsigned + NativePType> {
+pub struct ListBuilder<O: PrimInt + NativePType> {
     value_builder: Box<dyn ArrayBuilder>,
     index_builder: PrimitiveBuilder<O>,
     validity: BoolBuilder,
@@ -23,7 +23,7 @@ pub struct ListBuilder<O: PrimInt + Unsigned + NativePType> {
 
 impl<O> ListBuilder<O>
 where
-    O: PrimInt + Unsigned + NativePType + 'static,
+    O: PrimInt + NativePType + 'static,
     Scalar: From<O>,
     usize: AsPrimitive<O>,
 {
@@ -69,7 +69,7 @@ where
 
 impl<O> ArrayBuilder for ListBuilder<O>
 where
-    O: PrimInt + Unsigned + NativePType + 'static,
+    O: PrimInt + NativePType + 'static,
     Scalar: From<O>,
     usize: AsPrimitive<O>,
 {

From 5eca6e1d56f44c0bfd09996262773eaf83ef6047 Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs@live.co.uk>
Date: Wed, 18 Dec 2024 17:00:02 +0000
Subject: [PATCH 09/24] add static

---
 vortex-array/src/builders/list.rs | 4 ++--
 vortex-dtype/src/ptype.rs         | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/vortex-array/src/builders/list.rs b/vortex-array/src/builders/list.rs
index f5809b4bdb..91fcb8d0e6 100644
--- a/vortex-array/src/builders/list.rs
+++ b/vortex-array/src/builders/list.rs
@@ -23,7 +23,7 @@ pub struct ListBuilder<O: PrimInt + NativePType> {
 
 impl<O> ListBuilder<O>
 where
-    O: PrimInt + NativePType + 'static,
+    O: PrimInt + NativePType,
     Scalar: From<O>,
     usize: AsPrimitive<O>,
 {
@@ -69,7 +69,7 @@ where
 
 impl<O> ArrayBuilder for ListBuilder<O>
 where
-    O: PrimInt + NativePType + 'static,
+    O: PrimInt + NativePType,
     Scalar: From<O>,
     usize: AsPrimitive<O>,
 {
diff --git a/vortex-dtype/src/ptype.rs b/vortex-dtype/src/ptype.rs
index 0b406b6adf..d1ebf27896 100644
--- a/vortex-dtype/src/ptype.rs
+++ b/vortex-dtype/src/ptype.rs
@@ -60,6 +60,7 @@ pub trait NativePType:
     + FromPrimitive
     + ToBytes
     + TryFromBytes
+    + 'static
 {
     /// The PType that corresponds to this native type
     const PTYPE: PType;

From d72ba7a68a7692e704090cfa90f318aa142b0721 Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs@live.co.uk>
Date: Wed, 18 Dec 2024 14:46:14 +0000
Subject: [PATCH 10/24] wip

---
 vortex-array/src/array/arbitrary.rs | 39 ++++++++++++++++++++++++++---
 vortex-array/src/array/list/mod.rs  |  2 ++
 2 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/vortex-array/src/array/arbitrary.rs b/vortex-array/src/array/arbitrary.rs
index 9012c5ac72..a419ec892e 100644
--- a/vortex-array/src/array/arbitrary.rs
+++ b/vortex-array/src/array/arbitrary.rs
@@ -5,7 +5,7 @@ use arrow_buffer::BooleanBuffer;
 use vortex_dtype::{DType, NativePType, Nullability, PType};
 use vortex_error::{VortexExpect, VortexUnwrap};
 
-use super::{BoolArray, ChunkedArray, NullArray, PrimitiveArray, StructArray};
+use super::{BoolArray, ChunkedArray, ListArray, NullArray, PrimitiveArray, StructArray};
 use crate::array::{VarBinArray, VarBinViewArray};
 use crate::validity::Validity;
 use crate::{ArrayDType, ArrayData, IntoArrayData as _, IntoArrayVariant};
@@ -81,9 +81,15 @@ fn random_array(u: &mut Unstructured, dtype: &DType, len: Option<usize>) -> Resu
                     .vortex_unwrap()
                     .into_array())
                 }
-                // TOOD(joe): add arbitrary list
-                DType::List(..) => {
-                    todo!("List arrays are not implemented")
+                DType::List(ldt, n) => {
+                    let list_len = u.int_in_range(0..20)?;
+                    let builder = ListBuilder::new(ldt.clone(), list_len);
+                    for _ in 0..list_len {
+                        let null_value = u.arbitrary::<bool>()?;
+                        if null_value {}
+                        let values = random_array(u, ldt, chunk_len)?;
+                    }
+                    let values = random_array(u, ldt, chunk_len)?;
                 }
                 DType::Extension(..) => {
                     todo!("Extension arrays are not implemented")
@@ -136,6 +142,31 @@ fn random_string(
     }
 }
 
+fn random_list(
+    u: &mut Unstructured,
+    nullability: Nullability,
+    len: Option<usize>,
+) -> Result<ArrayData> {
+    match nullability {
+        Nullability::NonNullable => {
+            let v = arbitrary_vec_of_len::<Vec<u8>>(u, len)?;
+            Ok(match u.int_in_range(0..=1)? {
+                0 => ListArray::from_vec(v, DType::Binary(Nullability::NonNullable)).into_array(),
+                1 => VarBinViewArray::from_iter_bin(v).into_array(),
+                _ => unreachable!(),
+            })
+        }
+        Nullability::Nullable => {
+            let v = arbitrary_vec_of_len::<Option<Vec<u8>>>(u, len)?;
+            Ok(match u.int_in_range(0..=1)? {
+                0 => VarBinArray::from_iter(v, DType::Binary(Nullability::Nullable)).into_array(),
+                1 => VarBinViewArray::from_iter_nullable_bin(v).into_array(),
+                _ => unreachable!(),
+            })
+        }
+    }
+}
+
 fn random_bytes(
     u: &mut Unstructured,
     nullability: Nullability,
diff --git a/vortex-array/src/array/list/mod.rs b/vortex-array/src/array/list/mod.rs
index 45f25f76e8..f0dce1bf95 100644
--- a/vortex-array/src/array/list/mod.rs
+++ b/vortex-array/src/array/list/mod.rs
@@ -88,6 +88,8 @@ impl ListArray {
         )
     }
 
+    pub fn from_vec()
+
     pub fn validity(&self) -> Validity {
         self.metadata().validity.to_validity(|| {
             self.as_ref()

From 144fcab222eee62f90930e5d2b93839a2c10b82f Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs@live.co.uk>
Date: Wed, 18 Dec 2024 15:00:11 +0000
Subject: [PATCH 11/24] wip

---
 vortex-array/src/array/arbitrary.rs | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/vortex-array/src/array/arbitrary.rs b/vortex-array/src/array/arbitrary.rs
index a419ec892e..b646480284 100644
--- a/vortex-array/src/array/arbitrary.rs
+++ b/vortex-array/src/array/arbitrary.rs
@@ -2,13 +2,16 @@ use std::iter;
 
 use arbitrary::{Arbitrary, Result, Unstructured};
 use arrow_buffer::BooleanBuffer;
+use builders::ListBuilder;
 use vortex_dtype::{DType, NativePType, Nullability, PType};
 use vortex_error::{VortexExpect, VortexUnwrap};
+use vortex_scalar::ListScalar;
 
 use super::{BoolArray, ChunkedArray, ListArray, NullArray, PrimitiveArray, StructArray};
 use crate::array::{VarBinArray, VarBinViewArray};
+use crate::builders::ArrayBuilder;
 use crate::validity::Validity;
-use crate::{ArrayDType, ArrayData, IntoArrayData as _, IntoArrayVariant};
+use crate::{builders, ArrayDType, ArrayData, IntoArrayData as _, IntoArrayVariant};
 
 impl<'a> Arbitrary<'a> for ArrayData {
     fn arbitrary(u: &mut Unstructured<'a>) -> Result<Self> {
@@ -82,14 +85,14 @@ fn random_array(u: &mut Unstructured, dtype: &DType, len: Option<usize>) -> Resu
                     .into_array())
                 }
                 DType::List(ldt, n) => {
-                    let list_len = u.int_in_range(0..20)?;
-                    let builder = ListBuilder::new(ldt.clone(), list_len);
+                    let list_len = u.int_in_range(0..=20)?;
                     for _ in 0..list_len {
-                        let null_value = u.arbitrary::<bool>()?;
-                        if null_value {}
-                        let values = random_array(u, ldt, chunk_len)?;
+                        let value = random_array(u, ldt, None)?;
+                        let mut builder = ListBuilder::with_capacity(ldt.clone(), *n, 1);
+                        ListScalar::builder.append_elements(value)?;
+                        builder.finish()?;
                     }
-                    let values = random_array(u, ldt, chunk_len)?;
+                    Ok(builder.finish()?)
                 }
                 DType::Extension(..) => {
                     todo!("Extension arrays are not implemented")

From ca42a9bd5722df8e067c9824910946d4a4db2f8e Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs@live.co.uk>
Date: Wed, 18 Dec 2024 15:07:53 +0000
Subject: [PATCH 12/24] add list arb

---
 vortex-array/src/array/arbitrary.rs | 17 ++++++++++++-----
 vortex-array/src/array/list/mod.rs  |  2 --
 vortex-array/src/builders/mod.rs    |  1 +
 3 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/vortex-array/src/array/arbitrary.rs b/vortex-array/src/array/arbitrary.rs
index b646480284..4923f1c1b2 100644
--- a/vortex-array/src/array/arbitrary.rs
+++ b/vortex-array/src/array/arbitrary.rs
@@ -5,7 +5,8 @@ use arrow_buffer::BooleanBuffer;
 use builders::ListBuilder;
 use vortex_dtype::{DType, NativePType, Nullability, PType};
 use vortex_error::{VortexExpect, VortexUnwrap};
-use vortex_scalar::ListScalar;
+use vortex_scalar::arbitrary::random_scalar;
+use vortex_scalar::{ListScalar, Scalar};
 
 use super::{BoolArray, ChunkedArray, ListArray, NullArray, PrimitiveArray, StructArray};
 use crate::array::{VarBinArray, VarBinViewArray};
@@ -86,11 +87,17 @@ fn random_array(u: &mut Unstructured, dtype: &DType, len: Option<usize>) -> Resu
                 }
                 DType::List(ldt, n) => {
                     let list_len = u.int_in_range(0..=20)?;
+                    let mut builder = ListBuilder::with_capacity(ldt.clone(), *n, 1);
                     for _ in 0..list_len {
-                        let value = random_array(u, ldt, None)?;
-                        let mut builder = ListBuilder::with_capacity(ldt.clone(), *n, 1);
-                        ListScalar::builder.append_elements(value)?;
-                        builder.finish()?;
+                        if u.arbitrary::<bool>() {
+                            let elem_len = u.int_in_range(0..=20)?;
+                            let elem = (0..elem_len)
+                                .map(|_| random_scalar(u, ldt))
+                                .collect::<Result<Vec<_>>>()?;
+                            builder.append_value(Scalar::list(ldt.clone(), elem).as_list())?;
+                        } else {
+                            builder.append_null();
+                        }
                     }
                     Ok(builder.finish()?)
                 }
diff --git a/vortex-array/src/array/list/mod.rs b/vortex-array/src/array/list/mod.rs
index f0dce1bf95..45f25f76e8 100644
--- a/vortex-array/src/array/list/mod.rs
+++ b/vortex-array/src/array/list/mod.rs
@@ -88,8 +88,6 @@ impl ListArray {
         )
     }
 
-    pub fn from_vec()
-
     pub fn validity(&self) -> Validity {
         self.metadata().validity.to_validity(|| {
             self.as_ref()
diff --git a/vortex-array/src/builders/mod.rs b/vortex-array/src/builders/mod.rs
index fe6172c4b7..1bcc228017 100644
--- a/vortex-array/src/builders/mod.rs
+++ b/vortex-array/src/builders/mod.rs
@@ -12,6 +12,7 @@ use std::any::Any;
 pub use binary::*;
 pub use bool::*;
 pub use extension::*;
+pub use list::*;
 pub use null::*;
 pub use primitive::*;
 pub use utf8::*;

From 37932ef47888db2e3565c8aa350fc77bb6b39e5b Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs@live.co.uk>
Date: Wed, 18 Dec 2024 15:11:53 +0000
Subject: [PATCH 13/24] wip

---
 vortex-array/src/array/arbitrary.rs | 27 +--------------------------
 1 file changed, 1 insertion(+), 26 deletions(-)

diff --git a/vortex-array/src/array/arbitrary.rs b/vortex-array/src/array/arbitrary.rs
index 4923f1c1b2..a0c986c597 100644
--- a/vortex-array/src/array/arbitrary.rs
+++ b/vortex-array/src/array/arbitrary.rs
@@ -6,7 +6,7 @@ use builders::ListBuilder;
 use vortex_dtype::{DType, NativePType, Nullability, PType};
 use vortex_error::{VortexExpect, VortexUnwrap};
 use vortex_scalar::arbitrary::random_scalar;
-use vortex_scalar::{ListScalar, Scalar};
+use vortex_scalar::Scalar;
 
 use super::{BoolArray, ChunkedArray, ListArray, NullArray, PrimitiveArray, StructArray};
 use crate::array::{VarBinArray, VarBinViewArray};
@@ -152,31 +152,6 @@ fn random_string(
     }
 }
 
-fn random_list(
-    u: &mut Unstructured,
-    nullability: Nullability,
-    len: Option<usize>,
-) -> Result<ArrayData> {
-    match nullability {
-        Nullability::NonNullable => {
-            let v = arbitrary_vec_of_len::<Vec<u8>>(u, len)?;
-            Ok(match u.int_in_range(0..=1)? {
-                0 => ListArray::from_vec(v, DType::Binary(Nullability::NonNullable)).into_array(),
-                1 => VarBinViewArray::from_iter_bin(v).into_array(),
-                _ => unreachable!(),
-            })
-        }
-        Nullability::Nullable => {
-            let v = arbitrary_vec_of_len::<Option<Vec<u8>>>(u, len)?;
-            Ok(match u.int_in_range(0..=1)? {
-                0 => VarBinArray::from_iter(v, DType::Binary(Nullability::Nullable)).into_array(),
-                1 => VarBinViewArray::from_iter_nullable_bin(v).into_array(),
-                _ => unreachable!(),
-            })
-        }
-    }
-}
-
 fn random_bytes(
     u: &mut Unstructured,
     nullability: Nullability,

From 9264fa33337738f502f3e084fc25dd2470ddd998 Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs@live.co.uk>
Date: Wed, 18 Dec 2024 15:31:18 +0000
Subject: [PATCH 14/24] wip

---
 vortex-array/src/array/arbitrary.rs | 8 +++++---
 vortex-array/src/builders/mod.rs    | 1 -
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/vortex-array/src/array/arbitrary.rs b/vortex-array/src/array/arbitrary.rs
index a0c986c597..6c14472e12 100644
--- a/vortex-array/src/array/arbitrary.rs
+++ b/vortex-array/src/array/arbitrary.rs
@@ -8,7 +8,7 @@ use vortex_error::{VortexExpect, VortexUnwrap};
 use vortex_scalar::arbitrary::random_scalar;
 use vortex_scalar::Scalar;
 
-use super::{BoolArray, ChunkedArray, ListArray, NullArray, PrimitiveArray, StructArray};
+use super::{BoolArray, ChunkedArray, NullArray, PrimitiveArray, StructArray};
 use crate::array::{VarBinArray, VarBinViewArray};
 use crate::builders::ArrayBuilder;
 use crate::validity::Validity;
@@ -89,12 +89,14 @@ fn random_array(u: &mut Unstructured, dtype: &DType, len: Option<usize>) -> Resu
                     let list_len = u.int_in_range(0..=20)?;
                     let mut builder = ListBuilder::with_capacity(ldt.clone(), *n, 1);
                     for _ in 0..list_len {
-                        if u.arbitrary::<bool>() {
+                        if u.arbitrary::<bool>()? {
                             let elem_len = u.int_in_range(0..=20)?;
                             let elem = (0..elem_len)
                                 .map(|_| random_scalar(u, ldt))
                                 .collect::<Result<Vec<_>>>()?;
-                            builder.append_value(Scalar::list(ldt.clone(), elem).as_list())?;
+                            builder
+                                .append_value(Scalar::list(ldt.clone(), elem).as_list())
+                                .vortex_expect("can append value");
                         } else {
                             builder.append_null();
                         }
diff --git a/vortex-array/src/builders/mod.rs b/vortex-array/src/builders/mod.rs
index 1bcc228017..7b48fbd721 100644
--- a/vortex-array/src/builders/mod.rs
+++ b/vortex-array/src/builders/mod.rs
@@ -23,7 +23,6 @@ use vortex_scalar::{
     Utf8Scalar,
 };
 
-use crate::builders::list::ListBuilder;
 use crate::builders::struct_::StructBuilder;
 use crate::ArrayData;
 

From e3dbb6eff2abc2de7633dcf8377b905e9ab488e9 Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs@live.co.uk>
Date: Wed, 18 Dec 2024 15:54:10 +0000
Subject: [PATCH 15/24] clean up

---
 vortex-array/src/array/arbitrary.rs | 39 ++++++++++++++++-------------
 1 file changed, 21 insertions(+), 18 deletions(-)

diff --git a/vortex-array/src/array/arbitrary.rs b/vortex-array/src/array/arbitrary.rs
index 6c14472e12..10bb4e5898 100644
--- a/vortex-array/src/array/arbitrary.rs
+++ b/vortex-array/src/array/arbitrary.rs
@@ -1,4 +1,5 @@
 use std::iter;
+use std::sync::Arc;
 
 use arbitrary::{Arbitrary, Result, Unstructured};
 use arrow_buffer::BooleanBuffer;
@@ -85,24 +86,7 @@ fn random_array(u: &mut Unstructured, dtype: &DType, len: Option<usize>) -> Resu
                     .vortex_unwrap()
                     .into_array())
                 }
-                DType::List(ldt, n) => {
-                    let list_len = u.int_in_range(0..=20)?;
-                    let mut builder = ListBuilder::with_capacity(ldt.clone(), *n, 1);
-                    for _ in 0..list_len {
-                        if u.arbitrary::<bool>()? {
-                            let elem_len = u.int_in_range(0..=20)?;
-                            let elem = (0..elem_len)
-                                .map(|_| random_scalar(u, ldt))
-                                .collect::<Result<Vec<_>>>()?;
-                            builder
-                                .append_value(Scalar::list(ldt.clone(), elem).as_list())
-                                .vortex_expect("can append value");
-                        } else {
-                            builder.append_null();
-                        }
-                    }
-                    Ok(builder.finish()?)
-                }
+                DType::List(ldt, n) => random_list(u, ldt, n),
                 DType::Extension(..) => {
                     todo!("Extension arrays are not implemented")
                 }
@@ -120,6 +104,25 @@ fn random_array(u: &mut Unstructured, dtype: &DType, len: Option<usize>) -> Resu
     }
 }
 
+fn random_list(u: &mut Unstructured, ldt: &Arc<DType>, n: &Nullability) -> Result<ArrayData> {
+    let list_len = u.int_in_range(0..=20)?;
+    let mut builder = ListBuilder::with_capacity(ldt.clone(), *n, 1);
+    for _ in 0..list_len {
+        if u.arbitrary::<bool>()? {
+            let elem_len = u.int_in_range(0..=20)?;
+            let elem = (0..elem_len)
+                .map(|_| random_scalar(u, ldt))
+                .collect::<Result<Vec<_>>>()?;
+            builder
+                .append_value(Scalar::list(ldt.clone(), elem).as_list())
+                .vortex_expect("can append value");
+        } else {
+            builder.append_null();
+        }
+    }
+    Ok(builder.finish().vortex_expect("builder cannot error"))
+}
+
 fn split_number_into_parts(n: usize, parts: usize) -> Vec<usize> {
     let reminder = n % parts;
     let division = (n - reminder) / parts;

From 7e7ca792f417f6379d5198de3765526f764471c1 Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs@live.co.uk>
Date: Wed, 18 Dec 2024 16:06:50 +0000
Subject: [PATCH 16/24] fix cargo

---
 vortex-array/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vortex-array/Cargo.toml b/vortex-array/Cargo.toml
index 6a61606750..1e7bbc2281 100644
--- a/vortex-array/Cargo.toml
+++ b/vortex-array/Cargo.toml
@@ -53,7 +53,7 @@ vortex-datetime-dtype = { workspace = true }
 vortex-dtype = { workspace = true, features = ["flatbuffers", "serde"] }
 vortex-error = { workspace = true, features = ["flatbuffers", "flexbuffers"] }
 vortex-flatbuffers = { workspace = true, features = ["array"] }
-vortex-scalar = { workspace = true, features = ["flatbuffers", "serde"] }
+vortex-scalar = { workspace = true, features = ["flatbuffers", "serde", "arbitrary"] }
 
 [features]
 arbitrary = ["dep:arbitrary", "vortex-dtype/arbitrary"]

From 59365eb8c80bd6d894d2217f39572caf5f28cfe1 Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs@live.co.uk>
Date: Wed, 18 Dec 2024 16:45:55 +0000
Subject: [PATCH 17/24] arb offset

---
 vortex-array/src/array/arbitrary.rs | 32 ++++++++++++++++++++++-------
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/vortex-array/src/array/arbitrary.rs b/vortex-array/src/array/arbitrary.rs
index 10bb4e5898..0617939b8f 100644
--- a/vortex-array/src/array/arbitrary.rs
+++ b/vortex-array/src/array/arbitrary.rs
@@ -4,10 +4,12 @@ use std::sync::Arc;
 use arbitrary::{Arbitrary, Result, Unstructured};
 use arrow_buffer::BooleanBuffer;
 use builders::ListBuilder;
+use num_traits::PrimInt;
 use vortex_dtype::{DType, NativePType, Nullability, PType};
 use vortex_error::{VortexExpect, VortexUnwrap};
 use vortex_scalar::arbitrary::random_scalar;
 use vortex_scalar::Scalar;
+use Nullability::Nullable;
 
 use super::{BoolArray, ChunkedArray, NullArray, PrimitiveArray, StructArray};
 use crate::array::{VarBinArray, VarBinViewArray};
@@ -105,10 +107,26 @@ fn random_array(u: &mut Unstructured, dtype: &DType, len: Option<usize>) -> Resu
 }
 
 fn random_list(u: &mut Unstructured, ldt: &Arc<DType>, n: &Nullability) -> Result<ArrayData> {
+    match u.int_in_range(0..=5)? {
+        0 => random_list_offset::<i16>(u, ldt, n),
+        1 => random_list_offset::<i32>(u, ldt, n),
+        2 => random_list_offset::<i64>(u, ldt, n),
+        3 => random_list_offset::<u16>(u, ldt, n),
+        4 => random_list_offset::<u32>(u, ldt, n),
+        5 => random_list_offset::<u64>(u, ldt, n),
+        _ => unreachable!("int_in_range returns a value in the above range"),
+    }
+}
+
+fn random_list_offset<O: PrimInt + NativePType>(
+    u: &mut Unstructured,
+    ldt: &Arc<DType>,
+    n: &Nullability,
+) -> Result<ArrayData> {
     let list_len = u.int_in_range(0..=20)?;
-    let mut builder = ListBuilder::with_capacity(ldt.clone(), *n, 1);
+    let mut builder = ListBuilder::<u64>::with_capacity(ldt.clone(), *n, 1);
     for _ in 0..list_len {
-        if u.arbitrary::<bool>()? {
+        if matches!(n, Nullable) || u.arbitrary::<bool>()? {
             let elem_len = u.int_in_range(0..=20)?;
             let elem = (0..elem_len)
                 .map(|_| random_scalar(u, ldt))
@@ -146,10 +164,10 @@ fn random_string(
                 _ => unreachable!(),
             })
         }
-        Nullability::Nullable => {
+        Nullable => {
             let v = arbitrary_vec_of_len::<Option<String>>(u, len)?;
             Ok(match u.int_in_range(0..=1)? {
-                0 => VarBinArray::from_iter(v, DType::Utf8(Nullability::Nullable)).into_array(),
+                0 => VarBinArray::from_iter(v, DType::Utf8(Nullable)).into_array(),
                 1 => VarBinViewArray::from_iter_nullable_str(v).into_array(),
                 _ => unreachable!(),
             })
@@ -171,10 +189,10 @@ fn random_bytes(
                 _ => unreachable!(),
             })
         }
-        Nullability::Nullable => {
+        Nullable => {
             let v = arbitrary_vec_of_len::<Option<Vec<u8>>>(u, len)?;
             Ok(match u.int_in_range(0..=1)? {
-                0 => VarBinArray::from_iter(v, DType::Binary(Nullability::Nullable)).into_array(),
+                0 => VarBinArray::from_iter(v, DType::Binary(Nullable)).into_array(),
                 1 => VarBinViewArray::from_iter_nullable_bin(v).into_array(),
                 _ => unreachable!(),
             })
@@ -207,7 +225,7 @@ fn random_bool(
 fn random_validity(u: &mut Unstructured, nullability: Nullability, len: usize) -> Result<Validity> {
     match nullability {
         Nullability::NonNullable => Ok(Validity::NonNullable),
-        Nullability::Nullable => Ok(match u.int_in_range(0..=2)? {
+        Nullable => Ok(match u.int_in_range(0..=2)? {
             0 => Validity::AllValid,
             1 => Validity::AllInvalid,
             2 => Validity::from_iter(arbitrary_vec_of_len::<bool>(u, Some(len))?),

From 495a9d47b3ff3d9d9b9ed410060791651454db95 Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs@live.co.uk>
Date: Wed, 18 Dec 2024 16:46:40 +0000
Subject: [PATCH 18/24] arb offset

---
 vortex-array/src/array/arbitrary.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vortex-array/src/array/arbitrary.rs b/vortex-array/src/array/arbitrary.rs
index 0617939b8f..125fa3e1c2 100644
--- a/vortex-array/src/array/arbitrary.rs
+++ b/vortex-array/src/array/arbitrary.rs
@@ -124,7 +124,7 @@ fn random_list_offset<O: PrimInt + NativePType>(
     n: &Nullability,
 ) -> Result<ArrayData> {
     let list_len = u.int_in_range(0..=20)?;
-    let mut builder = ListBuilder::<u64>::with_capacity(ldt.clone(), *n, 1);
+    let mut builder = ListBuilder::<O>::with_capacity(ldt.clone(), *n, 1);
     for _ in 0..list_len {
         if matches!(n, Nullable) || u.arbitrary::<bool>()? {
             let elem_len = u.int_in_range(0..=20)?;

From d066657e7a0d12b4f4c8918dcaf83563394d5623 Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs@live.co.uk>
Date: Wed, 18 Dec 2024 16:48:01 +0000
Subject: [PATCH 19/24] arb offset

---
 vortex-array/src/array/arbitrary.rs | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/vortex-array/src/array/arbitrary.rs b/vortex-array/src/array/arbitrary.rs
index 125fa3e1c2..52c605298e 100644
--- a/vortex-array/src/array/arbitrary.rs
+++ b/vortex-array/src/array/arbitrary.rs
@@ -4,7 +4,7 @@ use std::sync::Arc;
 use arbitrary::{Arbitrary, Result, Unstructured};
 use arrow_buffer::BooleanBuffer;
 use builders::ListBuilder;
-use num_traits::PrimInt;
+use num_traits::{AsPrimitive, PrimInt};
 use vortex_dtype::{DType, NativePType, Nullability, PType};
 use vortex_error::{VortexExpect, VortexUnwrap};
 use vortex_scalar::arbitrary::random_scalar;
@@ -118,11 +118,16 @@ fn random_list(u: &mut Unstructured, ldt: &Arc<DType>, n: &Nullability) -> Resul
     }
 }
 
-fn random_list_offset<O: PrimInt + NativePType>(
+fn random_list_offset<O>(
     u: &mut Unstructured,
     ldt: &Arc<DType>,
     n: &Nullability,
-) -> Result<ArrayData> {
+) -> Result<ArrayData>
+where
+    O: PrimInt + NativePType + 'static,
+    Scalar: From<O>,
+    usize: AsPrimitive<O>,
+{
     let list_len = u.int_in_range(0..=20)?;
     let mut builder = ListBuilder::<O>::with_capacity(ldt.clone(), *n, 1);
     for _ in 0..list_len {

From 1871664c7059ca7989af492940c1e6719c0eede6 Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs@live.co.uk>
Date: Wed, 18 Dec 2024 17:17:42 +0000
Subject: [PATCH 20/24] remove static

---
 vortex-array/src/array/arbitrary.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vortex-array/src/array/arbitrary.rs b/vortex-array/src/array/arbitrary.rs
index 52c605298e..34cc249606 100644
--- a/vortex-array/src/array/arbitrary.rs
+++ b/vortex-array/src/array/arbitrary.rs
@@ -124,7 +124,7 @@ fn random_list_offset<O>(
     n: &Nullability,
 ) -> Result<ArrayData>
 where
-    O: PrimInt + NativePType + 'static,
+    O: PrimInt + NativePType,
     Scalar: From<O>,
     usize: AsPrimitive<O>,
 {

From df7a32d37fb75a86aaf6f7892c913b125d040215 Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs@live.co.uk>
Date: Wed, 18 Dec 2024 18:31:11 +0000
Subject: [PATCH 21/24] remove

---
 vortex-array/src/builders/mod.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/vortex-array/src/builders/mod.rs b/vortex-array/src/builders/mod.rs
index 1bcc228017..7b48fbd721 100644
--- a/vortex-array/src/builders/mod.rs
+++ b/vortex-array/src/builders/mod.rs
@@ -23,7 +23,6 @@ use vortex_scalar::{
     Utf8Scalar,
 };
 
-use crate::builders::list::ListBuilder;
 use crate::builders::struct_::StructBuilder;
 use crate::ArrayData;
 

From a22616fe939b085e2ae5f01f95918e574deb80dc Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs@live.co.uk>
Date: Thu, 19 Dec 2024 12:23:35 +0000
Subject: [PATCH 22/24] update

---
 vortex-array/src/array/arbitrary.rs | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/vortex-array/src/array/arbitrary.rs b/vortex-array/src/array/arbitrary.rs
index 34cc249606..23b38d33c9 100644
--- a/vortex-array/src/array/arbitrary.rs
+++ b/vortex-array/src/array/arbitrary.rs
@@ -9,7 +9,6 @@ use vortex_dtype::{DType, NativePType, Nullability, PType};
 use vortex_error::{VortexExpect, VortexUnwrap};
 use vortex_scalar::arbitrary::random_scalar;
 use vortex_scalar::Scalar;
-use Nullability::Nullable;
 
 use super::{BoolArray, ChunkedArray, NullArray, PrimitiveArray, StructArray};
 use crate::array::{VarBinArray, VarBinViewArray};
@@ -131,7 +130,7 @@ where
     let list_len = u.int_in_range(0..=20)?;
     let mut builder = ListBuilder::<O>::with_capacity(ldt.clone(), *n, 1);
     for _ in 0..list_len {
-        if matches!(n, Nullable) || u.arbitrary::<bool>()? {
+        if matches!(n, Nullability::Nullable) || u.arbitrary::<bool>()? {
             let elem_len = u.int_in_range(0..=20)?;
             let elem = (0..elem_len)
                 .map(|_| random_scalar(u, ldt))
@@ -169,10 +168,10 @@ fn random_string(
                 _ => unreachable!(),
             })
         }
-        Nullable => {
+        Nullability::Nullable => {
             let v = arbitrary_vec_of_len::<Option<String>>(u, len)?;
             Ok(match u.int_in_range(0..=1)? {
-                0 => VarBinArray::from_iter(v, DType::Utf8(Nullable)).into_array(),
+                0 => VarBinArray::from_iter(v, DType::Utf8(Nullability::Nullable)).into_array(),
                 1 => VarBinViewArray::from_iter_nullable_str(v).into_array(),
                 _ => unreachable!(),
             })
@@ -194,10 +193,10 @@ fn random_bytes(
                 _ => unreachable!(),
             })
         }
-        Nullable => {
+        Nullability::Nullable => {
             let v = arbitrary_vec_of_len::<Option<Vec<u8>>>(u, len)?;
             Ok(match u.int_in_range(0..=1)? {
-                0 => VarBinArray::from_iter(v, DType::Binary(Nullable)).into_array(),
+                0 => VarBinArray::from_iter(v, DType::Binary(Nullability::Nullable)).into_array(),
                 1 => VarBinViewArray::from_iter_nullable_bin(v).into_array(),
                 _ => unreachable!(),
             })
@@ -230,7 +229,7 @@ fn random_bool(
 fn random_validity(u: &mut Unstructured, nullability: Nullability, len: usize) -> Result<Validity> {
     match nullability {
         Nullability::NonNullable => Ok(Validity::NonNullable),
-        Nullable => Ok(match u.int_in_range(0..=2)? {
+        Nullability::Nullable => Ok(match u.int_in_range(0..=2)? {
             0 => Validity::AllValid,
             1 => Validity::AllInvalid,
             2 => Validity::from_iter(arbitrary_vec_of_len::<bool>(u, Some(len))?),

From 734abae8fd81b7d020f233e9119212a085b22a55 Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs@live.co.uk>
Date: Thu, 19 Dec 2024 12:51:01 +0000
Subject: [PATCH 23/24] fix

---
 vortex-array/src/array/arbitrary.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vortex-array/src/array/arbitrary.rs b/vortex-array/src/array/arbitrary.rs
index 23b38d33c9..e633971de3 100644
--- a/vortex-array/src/array/arbitrary.rs
+++ b/vortex-array/src/array/arbitrary.rs
@@ -136,7 +136,7 @@ where
                 .map(|_| random_scalar(u, ldt))
                 .collect::<Result<Vec<_>>>()?;
             builder
-                .append_value(Scalar::list(ldt.clone(), elem).as_list())
+                .append_value(Scalar::list(ldt.clone(), elem, n.clone()).as_list())
                 .vortex_expect("can append value");
         } else {
             builder.append_null();

From 2e94c19bba7eb667f364c564b2905b5d1d6ec16d Mon Sep 17 00:00:00 2001
From: Joe Isaacs <joe.isaacs@live.co.uk>
Date: Fri, 20 Dec 2024 11:14:50 +0000
Subject: [PATCH 24/24] fix

---
 vortex-array/src/array/arbitrary.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vortex-array/src/array/arbitrary.rs b/vortex-array/src/array/arbitrary.rs
index e633971de3..854ea88174 100644
--- a/vortex-array/src/array/arbitrary.rs
+++ b/vortex-array/src/array/arbitrary.rs
@@ -136,7 +136,7 @@ where
                 .map(|_| random_scalar(u, ldt))
                 .collect::<Result<Vec<_>>>()?;
             builder
-                .append_value(Scalar::list(ldt.clone(), elem, n.clone()).as_list())
+                .append_value(Scalar::list(ldt.clone(), elem, *n).as_list())
                 .vortex_expect("can append value");
         } else {
             builder.append_null();