From 704d3f31becbb63b50b47cfc7b7a093e0a65c117 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Tue, 26 Mar 2024 21:35:03 +0000 Subject: [PATCH] Remove unknown (#156) Fixes #155 --- pyvortex/src/lib.rs | 27 ++++++++++++++++++--------- pyvortex/test/test_dtype.py | 7 ++++--- vortex-array/src/formatter.rs | 7 ++----- vortex-array/src/ptype.rs | 11 ++++------- vortex-array/src/scalar/mod.rs | 21 ++++----------------- vortex-schema/flatbuffers/schema.fbs | 9 +++------ vortex-schema/src/dtype.rs | 15 ++++----------- vortex-schema/src/serde.rs | 6 ------ 8 files changed, 39 insertions(+), 64 deletions(-) diff --git a/pyvortex/src/lib.rs b/pyvortex/src/lib.rs index 6f93ba046d..c3cfeb4f00 100644 --- a/pyvortex/src/lib.rs +++ b/pyvortex/src/lib.rs @@ -3,6 +3,7 @@ use pyo3::prelude::*; use dtype::PyDType; use vortex_schema::DType; +use vortex_schema::Signedness::{Signed, Unsigned}; use crate::array::*; use crate::compress::PyCompressConfig; @@ -56,6 +57,7 @@ fn _lib(_py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_function(wrap_pyfunction!(dtype_int, m)?)?; + m.add_function(wrap_pyfunction!(dtype_uint, m)?)?; m.add_function(wrap_pyfunction!(dtype_float, m)?)?; m.add_function(wrap_pyfunction!(dtype_bool, m)?)?; m.add_function(wrap_pyfunction!(dtype_utf8, m)?)?; @@ -70,23 +72,30 @@ fn dtype_bool(py: Python<'_>, nullable: bool) -> PyResult> { } #[pyfunction(name = "int")] -#[pyo3(signature = (width = None, signed = true, nullable = false))] -fn dtype_int( - py: Python<'_>, - width: Option, - signed: bool, - nullable: bool, -) -> PyResult> { +#[pyo3(signature = (width = None, nullable = false))] +fn dtype_int(py: Python<'_>, width: Option, nullable: bool) -> PyResult> { + PyDType::wrap( + py, + DType::Int(width.unwrap_or(64).into(), Signed, nullable.into()), + ) +} + +#[pyfunction(name = "uint")] +#[pyo3(signature = (width = None, nullable = false))] +fn dtype_uint(py: Python<'_>, width: Option, nullable: bool) -> PyResult> { PyDType::wrap( py, - DType::Int(width.unwrap_or(0).into(), signed.into(), nullable.into()), + DType::Int(width.unwrap_or(64).into(), Unsigned, nullable.into()), ) } #[pyfunction(name = "float")] #[pyo3(signature = (width = None, nullable = false))] fn dtype_float(py: Python<'_>, width: Option, nullable: bool) -> PyResult> { - PyDType::wrap(py, DType::Float(width.unwrap_or(0).into(), nullable.into())) + PyDType::wrap( + py, + DType::Float(width.unwrap_or(64).into(), nullable.into()), + ) } #[pyfunction(name = "utf8")] diff --git a/pyvortex/test/test_dtype.py b/pyvortex/test/test_dtype.py index e8761f34b4..c7758983aa 100644 --- a/pyvortex/test/test_dtype.py +++ b/pyvortex/test/test_dtype.py @@ -2,8 +2,9 @@ def test_int(): - assert str(vortex.int()) == "signed_int(_)" - assert str(vortex.int(32)) == "signed_int(32)" - assert str(vortex.int(32, signed=False)) == "unsigned_int(32)" + assert str(vortex.int()) == "int(64)" + assert str(vortex.int(32)) == "int(32)" + assert str(vortex.int(32, nullable=True)) == "int(32)?" + assert str(vortex.uint(32)) == "uint(32)" assert str(vortex.float(16)) == "float(16)" assert str(vortex.bool(nullable=True)) == "bool?" diff --git a/vortex-array/src/formatter.rs b/vortex-array/src/formatter.rs index ecddd88d04..7f35b8eafd 100644 --- a/vortex-array/src/formatter.rs +++ b/vortex-array/src/formatter.rs @@ -120,15 +120,12 @@ mod test { #[test] fn display_primitive() { let arr: ArrayRef = (0..100).collect::>().into_array(); - assert_eq!( - format!("{}", arr), - "vortex.primitive(signed_int(32), len=100)" - ); + assert_eq!(format!("{}", arr), "vortex.primitive(int(32), len=100)"); } #[test] fn tree_display_primitive() { let arr: ArrayRef = (0..100).collect::>().into_array(); - assert_eq!(display_tree(&arr), "root: vortex.primitive(signed_int(32), len=100) nbytes=400 B (100.00%)\n values: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]...\n validity: None\n") + assert_eq!(display_tree(&arr), "root: vortex.primitive(int(32), len=100) nbytes=400 B (100.00%)\n values: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]...\n validity: None\n") } } diff --git a/vortex-array/src/ptype.rs b/vortex-array/src/ptype.rs index e7da99ef04..03e15c4242 100644 --- a/vortex-array/src/ptype.rs +++ b/vortex-array/src/ptype.rs @@ -182,19 +182,16 @@ impl TryFrom<&DType> for PType { use vortex_schema::Signedness::*; match value { Int(w, s, _) => match (w, s) { - (IntWidth::Unknown, Unknown | Signed) => Ok(PType::I64), - (IntWidth::_8, Unknown | Signed) => Ok(PType::I8), - (IntWidth::_16, Unknown | Signed) => Ok(PType::I16), - (IntWidth::_32, Unknown | Signed) => Ok(PType::I32), - (IntWidth::_64, Unknown | Signed) => Ok(PType::I64), - (IntWidth::Unknown, Unsigned) => Ok(PType::U64), + (IntWidth::_8, Signed) => Ok(PType::I8), + (IntWidth::_16, Signed) => Ok(PType::I16), + (IntWidth::_32, Signed) => Ok(PType::I32), + (IntWidth::_64, Signed) => Ok(PType::I64), (IntWidth::_8, Unsigned) => Ok(PType::U8), (IntWidth::_16, Unsigned) => Ok(PType::U16), (IntWidth::_32, Unsigned) => Ok(PType::U32), (IntWidth::_64, Unsigned) => Ok(PType::U64), }, Float(f, _) => match f { - FloatWidth::Unknown => Ok(PType::F64), FloatWidth::_16 => Ok(PType::F16), FloatWidth::_32 => Ok(PType::F32), FloatWidth::_64 => Ok(PType::F64), diff --git a/vortex-array/src/scalar/mod.rs b/vortex-array/src/scalar/mod.rs index 16f2fc0671..78e830b37a 100644 --- a/vortex-array/src/scalar/mod.rs +++ b/vortex-array/src/scalar/mod.rs @@ -110,22 +110,10 @@ impl Scalar { DType::Null => NullScalar::new().into(), DType::Bool(_) => BoolScalar::none().into(), DType::Int(w, s, _) => match (w, s) { - (IntWidth::Unknown, Signedness::Unknown | Signedness::Signed) => { - PrimitiveScalar::none::().into() - } - (IntWidth::_8, Signedness::Unknown | Signedness::Signed) => { - PrimitiveScalar::none::().into() - } - (IntWidth::_16, Signedness::Unknown | Signedness::Signed) => { - PrimitiveScalar::none::().into() - } - (IntWidth::_32, Signedness::Unknown | Signedness::Signed) => { - PrimitiveScalar::none::().into() - } - (IntWidth::_64, Signedness::Unknown | Signedness::Signed) => { - PrimitiveScalar::none::().into() - } - (IntWidth::Unknown, Signedness::Unsigned) => PrimitiveScalar::none::().into(), + (IntWidth::_8, Signedness::Signed) => PrimitiveScalar::none::().into(), + (IntWidth::_16, Signedness::Signed) => PrimitiveScalar::none::().into(), + (IntWidth::_32, Signedness::Signed) => PrimitiveScalar::none::().into(), + (IntWidth::_64, Signedness::Signed) => PrimitiveScalar::none::().into(), (IntWidth::_8, Signedness::Unsigned) => PrimitiveScalar::none::().into(), (IntWidth::_16, Signedness::Unsigned) => PrimitiveScalar::none::().into(), (IntWidth::_32, Signedness::Unsigned) => PrimitiveScalar::none::().into(), @@ -133,7 +121,6 @@ impl Scalar { }, DType::Decimal(_, _, _) => unimplemented!("DecimalScalar"), DType::Float(w, _) => match w { - FloatWidth::Unknown => PrimitiveScalar::none::().into(), FloatWidth::_16 => PrimitiveScalar::none::().into(), FloatWidth::_32 => PrimitiveScalar::none::().into(), FloatWidth::_64 => PrimitiveScalar::none::().into(), diff --git a/vortex-schema/flatbuffers/schema.fbs b/vortex-schema/flatbuffers/schema.fbs index a4fc42865c..0a885e1371 100644 --- a/vortex-schema/flatbuffers/schema.fbs +++ b/vortex-schema/flatbuffers/schema.fbs @@ -6,11 +6,9 @@ enum Nullability: byte { enum Signedness: byte { Signed, Unsigned, - Unknown, } enum IntWidth: byte { - Unknown, _8, _16, _32, @@ -18,7 +16,6 @@ enum IntWidth: byte { } enum FloatWidth: byte { - Unknown, _16, _32, _64, @@ -31,8 +28,8 @@ table Bool { } table Int { - width: IntWidth = Unknown; - signedness: Signedness = Unknown; + width: IntWidth; + signedness: Signedness; nullability: Nullability; } @@ -46,7 +43,7 @@ table Decimal { } table Float { - width: FloatWidth = Unknown; + width: FloatWidth; nullability: Nullability; } diff --git a/vortex-schema/src/dtype.rs b/vortex-schema/src/dtype.rs index 44e0ac0611..334e4687ad 100644 --- a/vortex-schema/src/dtype.rs +++ b/vortex-schema/src/dtype.rs @@ -36,7 +36,6 @@ impl Display for Nullability { #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)] pub enum Signedness { - Unknown, Unsigned, Signed, } @@ -54,7 +53,6 @@ impl From for Signedness { impl Display for Signedness { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { - Signedness::Unknown => write!(f, "unknown"), Signedness::Unsigned => write!(f, "unsigned"), Signedness::Signed => write!(f, "signed"), } @@ -63,7 +61,6 @@ impl Display for Signedness { #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)] pub enum IntWidth { - Unknown, _8, _16, _32, @@ -77,7 +74,7 @@ impl From for IntWidth { 16 => IntWidth::_16, 32 => IntWidth::_32, 64 => IntWidth::_64, - _ => IntWidth::Unknown, + _ => panic!("Invalid int width: {}", item), } } } @@ -85,7 +82,6 @@ impl From for IntWidth { impl Display for IntWidth { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { - IntWidth::Unknown => write!(f, "_"), IntWidth::_8 => write!(f, "8"), IntWidth::_16 => write!(f, "16"), IntWidth::_32 => write!(f, "32"), @@ -96,7 +92,6 @@ impl Display for IntWidth { #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)] pub enum FloatWidth { - Unknown, _16, _32, _64, @@ -108,7 +103,7 @@ impl From for FloatWidth { 16 => FloatWidth::_16, 32 => FloatWidth::_32, 64 => FloatWidth::_64, - _ => FloatWidth::Unknown, + _ => panic!("Invalid float width: {}", item), } } } @@ -116,7 +111,6 @@ impl From for FloatWidth { impl Display for FloatWidth { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { - FloatWidth::Unknown => write!(f, "_"), FloatWidth::_16 => write!(f, "16"), FloatWidth::_32 => write!(f, "32"), FloatWidth::_64 => write!(f, "64"), @@ -209,9 +203,8 @@ impl Display for DType { Null => write!(f, "null"), Bool(n) => write!(f, "bool{}", n), Int(w, s, n) => match s { - Unknown => write!(f, "int({}){}", w, n), - Unsigned => write!(f, "unsigned_int({}){}", w, n), - Signed => write!(f, "signed_int({}){}", w, n), + Unsigned => write!(f, "uint({}){}", w, n), + Signed => write!(f, "int({}){}", w, n), }, Decimal(p, s, n) => write!(f, "decimal({}, {}){}", p, s, n), Float(w, n) => write!(f, "float({}){}", w, n), diff --git a/vortex-schema/src/serde.rs b/vortex-schema/src/serde.rs index 89910bbfd1..42026ecb3a 100644 --- a/vortex-schema/src/serde.rs +++ b/vortex-schema/src/serde.rs @@ -301,7 +301,6 @@ impl TryFrom for Nullability { impl From<&IntWidth> for FbIntWidth { fn from(value: &IntWidth) -> Self { match value { - IntWidth::Unknown => FbIntWidth::Unknown, IntWidth::_8 => FbIntWidth::_8, IntWidth::_16 => FbIntWidth::_16, IntWidth::_32 => FbIntWidth::_32, @@ -315,7 +314,6 @@ impl TryFrom for IntWidth { fn try_from(value: FbIntWidth) -> SchemaResult { match value { - FbIntWidth::Unknown => Ok(IntWidth::Unknown), FbIntWidth::_8 => Ok(IntWidth::_8), FbIntWidth::_16 => Ok(IntWidth::_16), FbIntWidth::_32 => Ok(IntWidth::_32), @@ -330,7 +328,6 @@ impl TryFrom for IntWidth { impl From<&Signedness> for FbSignedness { fn from(value: &Signedness) -> Self { match value { - Signedness::Unknown => FbSignedness::Unknown, Signedness::Unsigned => FbSignedness::Unsigned, Signedness::Signed => FbSignedness::Signed, } @@ -342,7 +339,6 @@ impl TryFrom for Signedness { fn try_from(value: FbSignedness) -> SchemaResult { match value { - FbSignedness::Unknown => Ok(Signedness::Unknown), FbSignedness::Unsigned => Ok(Signedness::Unsigned), FbSignedness::Signed => Ok(Signedness::Signed), _ => Err(SchemaError::InvalidArgument( @@ -355,7 +351,6 @@ impl TryFrom for Signedness { impl From<&FloatWidth> for FbFloatWidth { fn from(value: &FloatWidth) -> Self { match value { - FloatWidth::Unknown => FbFloatWidth::Unknown, FloatWidth::_16 => FbFloatWidth::_16, FloatWidth::_32 => FbFloatWidth::_32, FloatWidth::_64 => FbFloatWidth::_64, @@ -368,7 +363,6 @@ impl TryFrom for FloatWidth { fn try_from(value: FbFloatWidth) -> SchemaResult { match value { - FbFloatWidth::Unknown => Ok(FloatWidth::Unknown), FbFloatWidth::_16 => Ok(FloatWidth::_16), FbFloatWidth::_32 => Ok(FloatWidth::_32), FbFloatWidth::_64 => Ok(FloatWidth::_64),