Merge remote-tracking branch 'duckdb/main' into fix-more-null-issues-…

…in-vectors

.rustfmt.toml

@@ -1,2 +1,7 @@
 max_width = 120
-imports_granularity = "Crate"
+imports_granularity = "Crate"
+reorder_imports = true
+fn_call_width = 72
+# indent_style = "Block"
+# tab_spaces = 2
+# group_imports="StdExternalCrate"

Cargo.toml

@@ -34,10 +34,11 @@ doc-comment = "0.3"
 fallible-iterator = "0.3"
 fallible-streaming-iterator = "0.1"
 flate2 = "1.0"
-hashlink = "0.8"
+hashlink = "0.9"
 lazy_static = "1.4"
 memchr = "2.3"
 num = { version = "0.4", default-features = false }
+num-integer = "0.1.46"
 pkg-config = "0.3.24"
 polars = "0.35.4"
 polars-core = "0.35.4"

crates/duckdb/Cargo.toml

@@ -56,7 +56,7 @@ calamine = { workspace = true, optional = true }
 num = { workspace = true, features = ["std"], optional = true }
 duckdb-loadable-macros = { workspace = true, optional = true }
 polars = { workspace = true, features = ["dtype-full"], optional = true }
-num-integer = {version = "0.1.46"}
+num-integer = { workspace = true }
 
 [dev-dependencies]
 doc-comment = { workspace = true }

crates/duckdb/examples/hello-ext/main.rs

@@ -3,7 +3,8 @@ extern crate duckdb_loadable_macros;
 extern crate libduckdb_sys;
 
 use duckdb::{
-    vtab::{BindInfo, DataChunk, Free, FunctionInfo, InitInfo, Inserter, LogicalType, LogicalTypeId, VTab},
+    core::{DataChunkHandle, Inserter, LogicalTypeHandle, LogicalTypeId},
+    vtab::{BindInfo, Free, FunctionInfo, InitInfo, VTab},
     Connection, Result,
 };
 use duckdb_loadable_macros::duckdb_entrypoint;
@@ -43,7 +44,7 @@ impl VTab for HelloVTab {
     type BindData = HelloBindData;
 
     unsafe fn bind(bind: &BindInfo, data: *mut HelloBindData) -> Result<(), Box<dyn std::error::Error>> {
-        bind.add_result_column("column0", LogicalType::new(LogicalTypeId::Varchar));
+        bind.add_result_column("column0", LogicalTypeHandle::from(LogicalTypeId::Varchar));
         let param = bind.get_parameter(0).to_string();
         unsafe {
             (*data).name = CString::new(param).unwrap().into_raw();
@@ -58,7 +59,7 @@ impl VTab for HelloVTab {
         Ok(())
     }
 
-    unsafe fn func(func: &FunctionInfo, output: &mut DataChunk) -> Result<(), Box<dyn std::error::Error>> {
+    unsafe fn func(func: &FunctionInfo, output: &mut DataChunkHandle) -> Result<(), Box<dyn std::error::Error>> {
         let init_info = func.get_init_data::<HelloInitData>();
         let bind_info = func.get_bind_data::<HelloBindData>();
 
@@ -79,8 +80,8 @@ impl VTab for HelloVTab {
         Ok(())
     }
 
-    fn parameters() -> Option<Vec<LogicalType>> {
-        Some(vec![LogicalType::new(LogicalTypeId::Varchar)])
+    fn parameters() -> Option<Vec<LogicalTypeHandle>> {
+        Some(vec![LogicalTypeHandle::from(LogicalTypeId::Varchar)])
     }
 }
 

crates/duckdb/src/appender/arrow.rs

@@ -1,7 +1,8 @@
 use super::{ffi, Appender, Result};
 use crate::{
+    core::{DataChunkHandle, LogicalTypeHandle},
     error::result_from_duckdb_appender,
-    vtab::{record_batch_to_duckdb_data_chunk, to_duckdb_logical_type, DataChunk, LogicalType},
+    vtab::{record_batch_to_duckdb_data_chunk, to_duckdb_logical_type},
     Error,
 };
 use arrow::record_batch::RecordBatch;
@@ -28,14 +29,14 @@ impl Appender<'_> {
     #[inline]
     pub fn append_record_batch(&mut self, record_batch: RecordBatch) -> Result<()> {
         let schema = record_batch.schema();
-        let mut logical_type: Vec<LogicalType> = vec![];
+        let mut logical_type: Vec<LogicalTypeHandle> = vec![];
         for field in schema.fields() {
             let logical_t = to_duckdb_logical_type(field.data_type())
                 .map_err(|_op| Error::ArrowTypeToDuckdbType(field.to_string(), field.data_type().clone()))?;
             logical_type.push(logical_t);
         }
 
-        let mut data_chunk = DataChunk::new(&logical_type);
+        let mut data_chunk = DataChunkHandle::new(&logical_type);
         record_batch_to_duckdb_data_chunk(&record_batch, &mut data_chunk).map_err(|_op| Error::AppendError)?;
 
         let rc = unsafe { duckdb_append_data_chunk(self.app, data_chunk.get_ptr()) };

crates/duckdb/src/config.rs

@@ -65,6 +65,12 @@ impl Config {
         Ok(self)
     }
 
+    /// Metadata from DuckDB callers
+    pub fn custom_user_agent(mut self, custom_user_agent: &str) -> Result<Config> {
+        self.set("custom_user_agent", custom_user_agent)?;
+        Ok(self)
+    }
+
     /// The order type used when none is specified ([ASC] or DESC)
     pub fn default_order(mut self, order: DefaultOrder) -> Result<Config> {
         self.set("default_order", &order.to_string())?;
@@ -190,6 +196,7 @@ mod test {
             .enable_object_cache(false)?
             .enable_autoload_extension(true)?
             .allow_unsigned_extensions()?
+            .custom_user_agent("test_user_agent")?
             .max_memory("2GB")?
             .threads(4)?
             .with("preserve_insertion_order", "true")?;
@@ -215,6 +222,10 @@ mod test {
         assert!(iter.next().unwrap().is_none());
         assert_eq!(iter.next(), None);
 
+        let user_agent: Result<String> = db.query_row("PRAGMA USER_AGENT", [], |row| row.get(0));
+        let user_agent = user_agent.unwrap();
+        assert!(&user_agent.ends_with("rust test_user_agent"));
+
         Ok(())
     }
 

crates/duckdb/src/vtab/data_chunk.rs → crates/duckdb/src/core/data_chunk.rs

@@ -1,28 +1,41 @@
 use super::{
-    logical_type::LogicalType,
+    logical_type::LogicalTypeHandle,
     vector::{ArrayVector, FlatVector, ListVector, StructVector},
 };
 use crate::ffi::{
     duckdb_create_data_chunk, duckdb_data_chunk, duckdb_data_chunk_get_column_count, duckdb_data_chunk_get_size,
     duckdb_data_chunk_get_vector, duckdb_data_chunk_set_size, duckdb_destroy_data_chunk,
 };
 
-/// DataChunk in DuckDB.
-pub struct DataChunk {
+/// Handle to the DataChunk in DuckDB.
+pub struct DataChunkHandle {
     /// Pointer to the DataChunk in duckdb C API.
     ptr: duckdb_data_chunk,
 
-    /// Whether this [DataChunk] own the [DataChunk::ptr].
+    /// Whether this [DataChunkHandle] own the [DataChunk::ptr].
     owned: bool,
 }
 
-impl DataChunk {
-    /// Create a new [DataChunk] with the given [LogicalType]s.
-    pub fn new(logical_types: &[LogicalType]) -> Self {
+impl Drop for DataChunkHandle {
+    fn drop(&mut self) {
+        if self.owned && !self.ptr.is_null() {
+            unsafe { duckdb_destroy_data_chunk(&mut self.ptr) }
+            self.ptr = std::ptr::null_mut();
+        }
+    }
+}
+
+impl DataChunkHandle {
+    pub(crate) unsafe fn new_unowned(ptr: duckdb_data_chunk) -> Self {
+        Self { ptr, owned: false }
+    }
+
+    /// Create a new [DataChunkHandle] with the given [LogicalTypeHandle]s.
+    pub fn new(logical_types: &[LogicalTypeHandle]) -> Self {
         let num_columns = logical_types.len();
         let mut c_types = logical_types.iter().map(|t| t.ptr).collect::<Vec<_>>();
         let ptr = unsafe { duckdb_create_data_chunk(c_types.as_mut_ptr(), num_columns as u64) };
-        DataChunk { ptr, owned: true }
+        DataChunkHandle { ptr, owned: true }
     }
 
     /// Get the vector at the specific column index: `idx`.
@@ -50,49 +63,34 @@ impl DataChunk {
         unsafe { duckdb_data_chunk_set_size(self.ptr, new_len as u64) };
     }
 
-    /// Get the length / the number of rows in this [DataChunk].
+    /// Get the length / the number of rows in this [DataChunkHandle].
     pub fn len(&self) -> usize {
         unsafe { duckdb_data_chunk_get_size(self.ptr) as usize }
     }
 
-    /// Check whether this [DataChunk] is empty.
+    /// Check whether this [DataChunkHandle] is empty.
     pub fn is_empty(&self) -> bool {
         self.len() == 0
     }
 
-    /// Get the number of columns in this [DataChunk].
+    /// Get the number of columns in this [DataChunkHandle].
     pub fn num_columns(&self) -> usize {
         unsafe { duckdb_data_chunk_get_column_count(self.ptr) as usize }
     }
 
-    /// Get the ptr of duckdb_data_chunk in this [DataChunk].
+    /// Get the ptr of duckdb_data_chunk in this [DataChunkHandle].
     pub fn get_ptr(&self) -> duckdb_data_chunk {
         self.ptr
     }
 }
 
-impl From<duckdb_data_chunk> for DataChunk {
-    fn from(ptr: duckdb_data_chunk) -> Self {
-        Self { ptr, owned: false }
-    }
-}
-
-impl Drop for DataChunk {
-    fn drop(&mut self) {
-        if self.owned && !self.ptr.is_null() {
-            unsafe { duckdb_destroy_data_chunk(&mut self.ptr) }
-            self.ptr = std::ptr::null_mut();
-        }
-    }
-}
-
 #[cfg(test)]
 mod test {
     use super::{super::logical_type::LogicalTypeId, *};
 
     #[test]
     fn test_data_chunk_construction() {
-        let dc = DataChunk::new(&[LogicalType::new(LogicalTypeId::Integer)]);
+        let dc = DataChunkHandle::new(&[LogicalTypeHandle::from(LogicalTypeId::Integer)]);
 
         assert_eq!(dc.num_columns(), 1);
 
@@ -101,7 +99,7 @@ mod test {
 
     #[test]
     fn test_vector() {
-        let datachunk = DataChunk::new(&[LogicalType::new(LogicalTypeId::Bigint)]);
+        let datachunk = DataChunkHandle::new(&[LogicalTypeHandle::from(LogicalTypeId::Bigint)]);
         let mut vector = datachunk.flat_vector(0);
         let data = vector.as_mut_slice::<i64>();
 
@@ -110,11 +108,11 @@ mod test {
 
     #[test]
     fn test_logi() {
-        let key = LogicalType::new(LogicalTypeId::Varchar);
+        let key = LogicalTypeHandle::from(LogicalTypeId::Varchar);
 
-        let value = LogicalType::new(LogicalTypeId::UTinyint);
+        let value = LogicalTypeHandle::from(LogicalTypeId::UTinyint);
 
-        let map = LogicalType::map(&key, &value);
+        let map = LogicalTypeHandle::map(&key, &value);
 
         assert_eq!(map.id(), LogicalTypeId::Map);