feat: sqlite decimal

connector_arrow/Cargo.toml

@@ -47,7 +47,7 @@ default-features = false
 optional = true
 
 [dependencies.duckdb]
-version = "0.10"
+version = "0.9.2"
 default-features = false
 optional = true
 

connector_arrow/src/sqlite/append.rs

@@ -126,6 +126,30 @@ impl ConsumeTy<NullType> for Vec<Value> {
     }
 }
 
+impl ConsumeTy<Decimal128Type> for Vec<Value> {
+    fn consume(&mut self, ty: &DataType, value: i128) {
+        self.push(Value::Text(crate::util::decimal::decimal128_to_string(
+            ty, value,
+        )));
+    }
+
+    fn consume_null(&mut self) {
+        self.push(Value::Null);
+    }
+}
+
+impl ConsumeTy<Decimal256Type> for Vec<Value> {
+    fn consume(&mut self, ty: &DataType, value: i256) {
+        self.push(Value::Text(crate::util::decimal::decimal256_to_string(
+            ty, value,
+        )));
+    }
+
+    fn consume_null(&mut self) {
+        self.push(Value::Null);
+    }
+}
+
 impl_consume_ty!(BooleanType, Value::Integer, i64::from);
 impl_consume_ty!(Int8Type, Value::Integer, i64::from);
 impl_consume_ty!(Int16Type, Value::Integer, i64::from);
@@ -164,8 +188,6 @@ impl_consume_unsupported!(
         DurationMillisecondType,
         DurationMicrosecondType,
         DurationNanosecondType,
-        Decimal128Type,
-        Decimal256Type,
     )
 );
 

connector_arrow/src/sqlite/mod.rs

@@ -59,6 +59,8 @@ impl Connection for SQLiteConnection {
             DataType::LargeBinary => Some(DataType::LargeBinary),
             DataType::Utf8 => Some(DataType::LargeUtf8),
             DataType::LargeUtf8 => Some(DataType::LargeUtf8),
+            DataType::Decimal128(_, _) => Some(DataType::LargeUtf8),
+            DataType::Decimal256(_, _) => Some(DataType::LargeUtf8),
             _ => None,
         }
     }

connector_arrow/src/util/decimal.rs

@@ -0,0 +1,33 @@
+use arrow::datatypes::{i256, DataType};
+
+pub fn decimal128_to_string(ty: &DataType, value: i128) -> String {
+    let DataType::Decimal128(_, scale) = ty else {
+        unreachable!();
+    };
+    insert_dot(scale, value.to_string())
+}
+
+pub fn decimal256_to_string(ty: &DataType, value: i256) -> String {
+    let DataType::Decimal256(_, scale) = ty else {
+        unreachable!();
+    };
+    insert_dot(scale, value.to_string())
+}
+
+fn insert_dot(scale: &i8, value: String) -> String {
+    if *scale > 0 {
+        let minus_len: usize = if value.starts_with('-') { 1 } else { 0 };
+
+        let fracs = *scale as usize;
+        if fracs >= value.len() - minus_len {
+            let (minus, value) = value.split_at(minus_len);
+            format!("{minus}0.{}{value}", "0".repeat(fracs - value.len()))
+        } else {
+            let mut value = value;
+            value.insert(value.len() - fracs, '.');
+            value
+        }
+    } else {
+        value
+    }
+}

connector_arrow/src/util/mod.rs

@@ -3,6 +3,7 @@
 
 mod arrow_reader;
 pub mod coerce;
+pub mod decimal;
 mod row_collect;
 mod row_reader;
 mod row_writer;

connector_arrow/tests/it/test_sqlite.rs

@@ -20,6 +20,7 @@ fn query_01() {
 #[case::int("roundtrip::int", spec::int())]
 #[case::uint("roundtrip::uint", spec::uint())]
 #[case::float("roundtrip::float", spec::float())]
+#[case::decimal("roundtrip::decimal", spec::decimal())]
 fn roundtrip(#[case] table_name: &str, #[case] spec: spec::ArrowGenSpec) {
     let mut conn = init();
     super::tests::roundtrip(&mut conn, table_name, spec);

flake.nix

@@ -19,12 +19,14 @@
         dontCheckPython = drv: drv.overridePythonAttrs (old: { doCheck = false; });
 
         essentials = with pkgs; [
-          fenix_pkgs.stable.cargo
-          fenix_pkgs.stable.clippy
-          fenix_pkgs.stable.rust-src
-          fenix_pkgs.stable.rustc
-          fenix_pkgs.stable.rustfmt
-          fenix_pkgs.stable.rust-analyzer
+          (fenix_pkgs.complete.withComponents [
+            "cargo"
+            "clippy"
+            "rust-src"
+            "rustc"
+            "rustfmt"
+            "rust-analyzer"
+          ])
           clang
 
           # tools
@@ -50,6 +52,9 @@
       {
         devShells.default = pkgs.mkShell {
           buildInputs = essentials ++ dbs;
+
+          # linking duckdb needs this
+          LD_LIBRARY_PATH = "${pkgs.stdenv.cc.cc.lib}/lib";
         };
       });
 }