C Extension API

.github/workflows/rust.yaml

@@ -38,7 +38,7 @@ jobs:
         name: Download duckdb
         with:
           repository: "duckdb/duckdb"
-          tag: "v1.0.0"
+          tag: "v1.1.0"
           fileName: ${{ matrix.duckdb }}
           out-file-path: .
 

.gitignore

@@ -27,6 +27,6 @@ Cargo.lock
 
 *.db
 
-crates/libduckdb-sys/duckdb-sources/
+crates/libduckdb-sys/duckdb-sources/*
 crates/libduckdb-sys/duckdb/
 crates/libduckdb-sys/._duckdb

.gitmodules

@@ -1,4 +1,3 @@
 [submodule "crates/libduckdb-sys/duckdb-sources"]
 	path = crates/libduckdb-sys/duckdb-sources
 	url = https://github.com/duckdb/duckdb
-	update = none

Cargo.toml

@@ -7,7 +7,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.0.0"
+version = "1.1.0"
 authors = ["wangfenjin <[email protected]>"]
 edition = "2021"
 repository = "https://github.com/duckdb/duckdb-rs"
@@ -19,8 +19,8 @@ license = "MIT"
 categories = ["database"]
 
 [workspace.dependencies]
-duckdb = { version = "1.0.0", path = "crates/duckdb" }
-libduckdb-sys = { version = "1.0.0", path = "crates/libduckdb-sys" }
+duckdb = { version = "1.1.0", path = "crates/duckdb" }
+libduckdb-sys = { version = "1.1.0", path = "crates/libduckdb-sys" }
 duckdb-loadable-macros = { version = "0.1.2", path = "crates/duckdb-loadable-macros" }
 autocfg = "1.0"
 bindgen = { version = "0.69", default-features = false }
@@ -43,6 +43,7 @@ pkg-config = "0.3.24"
 polars = "0.35.4"
 polars-core = "0.35.4"
 pretty_assertions = "1.4.0"
+prettyplease = "0.2.20"
 proc-macro2 = "1.0.56"
 quote = "1.0.21"
 r2d2 = "0.8.9"

Makefile

@@ -12,3 +12,9 @@ all:
 
 test:
 	cargo test --features bundled --features modern-full -- --nocapture
+
+# Build the rust-based CAPI extension demo, change the target arch below to
+EXAMPLE_TARGET_ARCH = osx_arm64
+examples-capi-demo:
+	cargo build --example hello-ext-capi --features="vtab-loadable,loadable_extension" --release
+	python3 scripts/append_extension_metadata.py -l target/release/examples/libhello_ext_capi.dylib -n rusty_quack -dv v0.0.1 -ev v0.0.1 -p $EXAMPLE_TARGET_ARCH

add_rustfmt_hook.sh

@@ -8,7 +8,7 @@ command -v rustfmt >/dev/null 2>&1 || { echo >&2 "Rustfmt is required but it's n
 # write a whole script to pre-commit hook
 # NOTE: it will overwrite pre-commit file!
 cat > .git/hooks/pre-commit <<'EOF'
-#!/bin/bash -e
+#!/bin/bash
 declare -a rust_files=()
 files=$(git diff-index --name-only --cached HEAD)
 echo 'Formatting source files'

crates/duckdb-loadable-macros/Cargo.toml

@@ -14,6 +14,7 @@ description = "Native bindings to the libduckdb library, C API; build loadable e
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+darling = "0.20.10"
 proc-macro2 = { workspace = true }
 quote = { workspace = true }
 syn = { workspace = true, features = ["extra-traits", "full", "fold", "parsing"] }

crates/duckdb-loadable-macros/src/lib.rs

@@ -1,11 +1,84 @@
 #![allow(clippy::redundant_clone)]
-use proc_macro2::{Ident, Span};
+use proc_macro2::{Ident, Literal, Punct, Span};
 
 use syn::{parse_macro_input, spanned::Spanned, Item};
 
 use proc_macro::TokenStream;
 use quote::quote_spanned;
 
+use darling::{ast::NestedMeta, Error, FromMeta};
+use syn::ItemFn;
+
+/// For parsing the arguments to the duckdb_entrypoint_c_api macro
+#[derive(Debug, FromMeta)]
+struct CEntryPointMacroArgs {
+    #[darling(default)]
+    /// The name to be given to this extension. This name is used in the entrypoint function called by duckdb
+    ext_name: String,
+    /// The minimum C API version this extension requires. It is recommended to set this to the lowest possible version
+    /// at which your extension still compiles
+    min_duckdb_version: Option<String>,
+}
+
+/// Wraps an entrypoint function to expose an unsafe extern "C" function of the same name.
+/// Warning: experimental!
+#[proc_macro_attribute]
+pub fn duckdb_entrypoint_c_api(attr: TokenStream, item: TokenStream) -> TokenStream {
+    let attr_args = match NestedMeta::parse_meta_list(attr.into()) {
+        Ok(v) => v,
+        Err(e) => {
+            return TokenStream::from(Error::from(e).write_errors());
+        }
+    };
+
+    let args = match CEntryPointMacroArgs::from_list(&attr_args) {
+        Ok(v) => v,
+        Err(e) => {
+            return TokenStream::from(e.write_errors());
+        }
+    };
+
+    // Set the minimum duckdb version (dev by default)
+    let minimum_duckdb_version = match args.min_duckdb_version {
+        Some(i) => i,
+        None => "dev".to_string(),
+    };
+
+    let ast = parse_macro_input!(item as syn::Item);
+
+    match ast {
+        Item::Fn(mut func) => {
+            let c_entrypoint = Ident::new(format!("{}_init_c_api", args.ext_name).as_str(), Span::call_site());
+            let original_funcname = func.sig.ident.to_string();
+            let prefixed_original_function = func.sig.ident.clone();
+
+            quote_spanned! {func.span()=>
+
+                /// # Safety
+                ///
+                /// Will be called by duckdb
+                #[no_mangle]
+                pub unsafe extern "C" fn #c_entrypoint(info: ffi::duckdb_extension_info, access: ffi::duckdb_extension_access) {
+                    let res = ffi::duckdb_rs_extension_api_init(info, access, #minimum_duckdb_version);
+
+                    if let Err(x) = res {
+                        // Error will be handled by DuckDB
+                        return;
+                    }
+
+                    let db : ffi::duckdb_database = *access.get_database.unwrap()(info);
+                    let connection = Connection::open_from_raw(db.cast()).expect("can't open db connection");
+                    #prefixed_original_function(connection).expect("init failed");
+                }
+
+                #func
+            }
+            .into()
+        }
+        _ => panic!("Only function items are allowed on duckdb_entrypoint"),
+    }
+}
+
 /// Wraps an entrypoint function to expose an unsafe extern "C" function of the same name.
 #[proc_macro_attribute]
 pub fn duckdb_entrypoint(_attr: TokenStream, item: TokenStream) -> TokenStream {

crates/duckdb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "duckdb"
-version = "1.0.0"
+version = "1.1.0"
 authors.workspace = true
 edition.workspace = true
 repository.workspace = true
@@ -93,3 +93,8 @@ all-features = false
 name = "hello-ext"
 crate-type = ["cdylib"]
 required-features = ["vtab-loadable"]
+
+[[example]]
+name = "hello-ext-capi"
+crate-type = ["cdylib"]
+required-features = ["vtab-loadable"]

crates/duckdb/examples/hello-ext-capi/main.rs

@@ -0,0 +1,93 @@
+extern crate duckdb;
+extern crate duckdb_loadable_macros;
+extern crate libduckdb_sys;
+
+use duckdb::{
+    core::{DataChunkHandle, Inserter, LogicalTypeHandle, LogicalTypeId},
+    vtab::{BindInfo, Free, FunctionInfo, InitInfo, VTab},
+    Connection, Result,
+};
+use duckdb_loadable_macros::duckdb_entrypoint_c_api;
+use libduckdb_sys as ffi;
+use std::{
+    error::Error,
+    ffi::{c_char, c_void, CString},
+};
+
+#[repr(C)]
+struct HelloBindData {
+    name: *mut c_char,
+}
+
+impl Free for HelloBindData {
+    fn free(&mut self) {
+        unsafe {
+            if self.name.is_null() {
+                return;
+            }
+            drop(CString::from_raw(self.name));
+        }
+    }
+}
+
+#[repr(C)]
+struct HelloInitData {
+    done: bool,
+}
+
+struct HelloVTab;
+
+impl Free for HelloInitData {}
+
+impl VTab for HelloVTab {
+    type InitData = HelloInitData;
+    type BindData = HelloBindData;
+
+    unsafe fn bind(bind: &BindInfo, data: *mut HelloBindData) -> Result<(), Box<dyn std::error::Error>> {
+        bind.add_result_column("column0", LogicalTypeHandle::from(LogicalTypeId::Varchar));
+        let param = bind.get_parameter(0).to_string();
+        unsafe {
+            (*data).name = CString::new(param).unwrap().into_raw();
+        }
+        Ok(())
+    }
+
+    unsafe fn init(_: &InitInfo, data: *mut HelloInitData) -> Result<(), Box<dyn std::error::Error>> {
+        unsafe {
+            (*data).done = false;
+        }
+        Ok(())
+    }
+
+    unsafe fn func(func: &FunctionInfo, output: &mut DataChunkHandle) -> Result<(), Box<dyn std::error::Error>> {
+        let init_info = func.get_init_data::<HelloInitData>();
+        let bind_info = func.get_bind_data::<HelloBindData>();
+
+        unsafe {
+            if (*init_info).done {
+                output.set_len(0);
+            } else {
+                (*init_info).done = true;
+                let vector = output.flat_vector(0);
+                let name = CString::from_raw((*bind_info).name);
+                let result = CString::new(format!("Hello {}", name.to_str()?))?;
+                // Can't consume the CString
+                (*bind_info).name = CString::into_raw(name);
+                vector.insert(0, result);
+                output.set_len(1);
+            }
+        }
+        Ok(())
+    }
+
+    fn parameters() -> Option<Vec<LogicalTypeHandle>> {
+        Some(vec![LogicalTypeHandle::from(LogicalTypeId::Varchar)])
+    }
+}
+
+#[duckdb_entrypoint_c_api(ext_name = "rusty_quack", min_duckdb_version = "v0.0.1")]
+pub unsafe fn ExtensionEntrypoint(con: Connection) -> Result<(), Box<dyn Error>> {
+    con.register_table_function::<HelloVTab>("hello")
+        .expect("Failed to register hello table function");
+    Ok(())
+}

crates/duckdb/src/config.rs

@@ -235,7 +235,7 @@ mod test {
         let res = Connection::open_in_memory_with_flags(config);
         assert_eq!(
             res.unwrap_err().to_string(),
-            "Invalid Input Error: Unrecognized configuration property \"some-invalid-setting\""
+            "Invalid Input Error: The following options were not recognized: some-invalid-setting"
         );
         Ok(())
     }

crates/duckdb/src/test_all_types.rs

@@ -22,7 +22,7 @@ fn test_large_arrow_types() -> crate::Result<()> {
 fn test_with_database(database: &Connection) -> crate::Result<()> {
     // uhugeint, time_tz, and dec38_10 aren't supported in the duckdb arrow layer
     // union is currently blocked by https://github.com/duckdb/duckdb/pull/11326
-    let excluded = ["uhugeint", "time_tz", "dec38_10", "union"];
+    let excluded = ["uhugeint", "time_tz", "dec38_10", "union", "varint"];
 
     let mut binding = database.prepare(&format!(
         "SELECT * EXCLUDE ({}) FROM test_all_types()",
@@ -147,7 +147,7 @@ fn test_single(idx: &mut i32, column: String, value: ValueRef) {
             _ => assert_eq!(value, ValueRef::Null),
         },
         "timestamp_ns" => match idx {
-            0 => assert_eq!(value, ValueRef::Timestamp(TimeUnit::Nanosecond, -9223372036854775808)),
+            0 => assert_eq!(value, ValueRef::Timestamp(TimeUnit::Nanosecond, -9223286400000000000)),
             1 => assert_eq!(value, ValueRef::Timestamp(TimeUnit::Nanosecond, 9223372036854775806)),
             _ => assert_eq!(value, ValueRef::Null),
         },

crates/duckdb/src/types/from_sql.rs

@@ -279,7 +279,7 @@ mod test {
         let sql = "BEGIN;
                    CREATE TABLE ts (sec TIMESTAMP_S, milli TIMESTAMP_MS, micro TIMESTAMP_US, nano TIMESTAMP_NS );
                    INSERT INTO ts VALUES (NULL,NULL,NULL,NULL );
-                   INSERT INTO ts VALUES ('2008-01-01 00:00:01','2008-01-01 00:00:01.594','2008-01-01 00:00:01.88926','2008-01-01 00:00:01.889268321' );
+                   INSERT INTO ts VALUES ('2008-01-01 00:00:01','2008-01-01 00:00:01.594','2008-01-01 00:00:01.88926','2008-01-01 00:00:01.889268000' );
                    -- INSERT INTO ts VALUES (NULL,NULL,NULL,1199145601889268321 );
                    END;";
         db.execute_batch(sql)?;
@@ -394,7 +394,7 @@ mod test {
         })?;
         assert_eq!(v, ("47183823-2574-4bfd-b411-99ed177d3e43".to_string(),));
         let v = db.query_row(
-            "SELECT u FROM uuid where u>?",
+            "SELECT u FROM uuid where u>?::UUID",
             ["10203040-5060-7080-0102-030405060708"],
             |row| <(String,)>::try_from(row),
         )?;

crates/libduckdb-sys/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "libduckdb-sys"
-version = "1.0.0"
+version = "1.1.0"
 authors.workspace = true
 edition.workspace = true
 license.workspace = true
@@ -23,6 +23,7 @@ json = ["bundled"]
 parquet = ["bundled"]
 extensions-full = ["json", "parquet"]
 winduckdb = []
+loadable_extension = ["prettyplease", "quote", "syn"]
 
 [dependencies]
 
@@ -36,6 +37,9 @@ vcpkg = { workspace = true, optional = true }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
 tar = { workspace = true }
+syn = { workspace = true, optional = true }
+quote = { workspace = true, optional = true }
+prettyplease = { workspace = true, optional = true }
 
 [dev-dependencies]
 arrow = { workspace = true, features = ["ffi"] }