From b310e9b7d19e32e6516d2b4390ed2cb1bc8d863e Mon Sep 17 00:00:00 2001 From: Wang Fenjin Date: Tue, 25 Apr 2023 22:37:13 +0800 Subject: [PATCH] build loadable ext (#149) * build loadable ext * fix warning * add loadable macros * fix comment * fix clippy error * fix clippy --- .github/workflows/rust.yaml | 7 ++- Cargo.toml | 9 ++- duckdb-loadable-macros/Cargo.toml | 22 ++++++++ duckdb-loadable-macros/LICENSE | 1 + duckdb-loadable-macros/README.md | 1 + duckdb-loadable-macros/src/lib.rs | 47 ++++++++++++++++ examples/hello-ext/main.rs | 93 +++++++++++++++++++++++++++++++ src/lib.rs | 19 ++++++- 8 files changed, 196 insertions(+), 3 deletions(-) create mode 100644 duckdb-loadable-macros/Cargo.toml create mode 120000 duckdb-loadable-macros/LICENSE create mode 120000 duckdb-loadable-macros/README.md create mode 100644 duckdb-loadable-macros/src/lib.rs create mode 100644 examples/hello-ext/main.rs diff --git a/.github/workflows/rust.yaml b/.github/workflows/rust.yaml index b19ee09a..fd239581 100644 --- a/.github/workflows/rust.yaml +++ b/.github/workflows/rust.yaml @@ -94,6 +94,8 @@ jobs: env: DUCKDB_LIB_DIR: ${{ github.workspace }}/libduckdb DUCKDB_INCLUDE_DIR: ${{ github.workspace }}/libduckdb + - name: Build loadable extension + run: cargo build --example hello-ext --features="vtab-loadable bundled" Windows: name: Windows build from source @@ -116,6 +118,8 @@ jobs: rust-version: stable targets: x86_64-pc-windows-msvc - run: cargo test --features "modern-full extensions-full" + - name: Build loadable extension + run: cargo build --example hello-ext --features="vtab-loadable bundled" Sanitizer: name: Address Sanitizer @@ -139,7 +143,8 @@ jobs: # as the other tests have them. RUST_BACKTRACE: "0" run: cargo -Z build-std test --features "modern-full extensions-full" --target x86_64-unknown-linux-gnu - + - name: Build loadable extension + run: cargo build --example hello-ext --features="vtab-loadable bundled" - uses: wangfenjin/publish-crates@main name: cargo publish --dry-run with: diff --git a/Cargo.toml b/Cargo.toml index 72c1b6a1..5b8d69f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ categories = ["database"] name = "duckdb" [workspace] -members = ["libduckdb-sys"] +members = ["libduckdb-sys", "duckdb-loadable-macros"] [features] default = [] @@ -27,6 +27,7 @@ httpfs = ["libduckdb-sys/httpfs", "bundled"] json = ["libduckdb-sys/json", "bundled"] parquet = ["libduckdb-sys/parquet", "bundled"] vtab = [] +vtab-loadable = ["vtab", "duckdb-loadable-macros"] vtab-excel = ["vtab", "calamine"] vtab-arrow = ["vtab", "num"] vtab-full = ["vtab-excel", "vtab-arrow"] @@ -61,6 +62,7 @@ strum = { version = "0.24", features = ["derive"] } r2d2 = { version = "0.8.9", optional = true } calamine = { version = "0.19.1", optional = true } num = { version = "0.4", optional = true, default-features = false, features = ["std"] } +duckdb-loadable-macros = { version = "0.1.0", path="./duckdb-loadable-macros", optional = true } [dev-dependencies] doc-comment = "0.3" @@ -90,3 +92,8 @@ default-target = "x86_64-unknown-linux-gnu" [package.metadata.playground] features = [] all-features = false + +[[example]] +name = "hello-ext" +crate-type = ["cdylib"] +required-features = ["vtab-loadable", "bundled"] \ No newline at end of file diff --git a/duckdb-loadable-macros/Cargo.toml b/duckdb-loadable-macros/Cargo.toml new file mode 100644 index 00000000..86587f64 --- /dev/null +++ b/duckdb-loadable-macros/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "duckdb-loadable-macros" +version = "0.1.0" +authors = ["wangfenjin "] +edition = "2021" +license = "MIT" +repository = "https://github.com/wangfenjin/duckdb-rs" +homepage = "https://github.com/wangfenjin/duckdb-rs" +keywords = ["duckdb", "ffi", "database"] +readme = "README.md" +categories = ["external-ffi-bindings", "database"] +description = "Native bindings to the libduckdb library, C API; build loadable extensions" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +proc-macro2 = { version = "1.0.56" } +quote = { version = "1.0.21" } +syn = { version = "1.0.95", features = [ "extra-traits", "full", "fold", "parsing" ] } + +[lib] +proc-macro = true diff --git a/duckdb-loadable-macros/LICENSE b/duckdb-loadable-macros/LICENSE new file mode 120000 index 00000000..ea5b6064 --- /dev/null +++ b/duckdb-loadable-macros/LICENSE @@ -0,0 +1 @@ +../LICENSE \ No newline at end of file diff --git a/duckdb-loadable-macros/README.md b/duckdb-loadable-macros/README.md new file mode 120000 index 00000000..32d46ee8 --- /dev/null +++ b/duckdb-loadable-macros/README.md @@ -0,0 +1 @@ +../README.md \ No newline at end of file diff --git a/duckdb-loadable-macros/src/lib.rs b/duckdb-loadable-macros/src/lib.rs new file mode 100644 index 00000000..92b8f2df --- /dev/null +++ b/duckdb-loadable-macros/src/lib.rs @@ -0,0 +1,47 @@ +use proc_macro2::Ident; + +use syn::{parse_macro_input, spanned::Spanned, Item}; + +use proc_macro::TokenStream; +use quote::quote_spanned; + +/// Wraps an entrypoint function to expose an unsafe extern "C" function of the same name. +#[proc_macro_attribute] +pub fn duckdb_entrypoint(_attr: TokenStream, item: TokenStream) -> TokenStream { + let ast = parse_macro_input!(item as syn::Item); + match ast { + Item::Fn(mut func) => { + let c_entrypoint = func.sig.ident.clone(); + + let original_funcname = func.sig.ident.to_string(); + func.sig.ident = Ident::new(format!("_{}", original_funcname).as_str(), func.sig.ident.span()); + + let prefixed_original_function = func.sig.ident.clone(); + + quote_spanned! {func.span()=> + #func + + /// # Safety + /// + /// Will be called by duckdb + #[no_mangle] + pub unsafe extern "C" fn #c_entrypoint(db: *mut c_void) { + let connection = Connection::open_from_raw(db.cast()).expect("can't open db connection"); + #prefixed_original_function(connection).expect("init failed"); + } + + /// # Safety + /// + /// Predefined function, don't need to change unless you are sure + #[no_mangle] + pub unsafe extern "C" fn libhello_ext_version() -> *const c_char { + ffi::duckdb_library_version() + } + + + } + .into() + } + _ => panic!("Only function items are allowed on duckdb_entrypoint"), + } +} diff --git a/examples/hello-ext/main.rs b/examples/hello-ext/main.rs new file mode 100644 index 00000000..298b844e --- /dev/null +++ b/examples/hello-ext/main.rs @@ -0,0 +1,93 @@ +extern crate duckdb; +extern crate duckdb_loadable_macros; +extern crate libduckdb_sys; + +use duckdb::{ + vtab::{BindInfo, DataChunk, Free, FunctionInfo, InitInfo, Inserter, LogicalType, LogicalTypeId, VTab}, + Connection, Result, +}; +use duckdb_loadable_macros::duckdb_entrypoint; +use libduckdb_sys as ffi; +use std::{ + error::Error, + ffi::{c_char, c_void, CString}, +}; + +#[repr(C)] +struct HelloBindData { + name: *mut c_char, +} + +impl Free for HelloBindData { + fn free(&mut self) { + unsafe { + if self.name.is_null() { + return; + } + drop(CString::from_raw(self.name)); + } + } +} + +#[repr(C)] +struct HelloInitData { + done: bool, +} + +struct HelloVTab; + +impl Free for HelloInitData {} + +impl VTab for HelloVTab { + type InitData = HelloInitData; + type BindData = HelloBindData; + + fn bind(bind: &BindInfo, data: *mut HelloBindData) -> Result<(), Box> { + bind.add_result_column("column0", LogicalType::new(LogicalTypeId::Varchar)); + let param = bind.get_parameter(0).to_string(); + unsafe { + (*data).name = CString::new(param).unwrap().into_raw(); + } + Ok(()) + } + + fn init(_: &InitInfo, data: *mut HelloInitData) -> Result<(), Box> { + unsafe { + (*data).done = false; + } + Ok(()) + } + + fn func(func: &FunctionInfo, output: &mut DataChunk) -> Result<(), Box> { + let init_info = func.get_init_data::(); + let bind_info = func.get_bind_data::(); + + unsafe { + if (*init_info).done { + output.set_len(0); + } else { + (*init_info).done = true; + let vector = output.flat_vector(0); + let name = CString::from_raw((*bind_info).name); + let result = CString::new(format!("Hello {}", name.to_str()?))?; + // Can't consume the CString + (*bind_info).name = CString::into_raw(name); + vector.insert(0, result); + output.set_len(1); + } + } + Ok(()) + } + + fn parameters() -> Option> { + Some(vec![LogicalType::new(LogicalTypeId::Varchar)]) + } +} + +// Exposes a extern C function named "libhello_ext_init" in the compiled dynamic library, +// the "entrypoint" that duckdb will use to load the extension. +#[duckdb_entrypoint] +pub fn libhello_ext_init(conn: Connection) -> Result<(), Box> { + conn.register_table_function::("hello")?; + Ok(()) +} diff --git a/src/lib.rs b/src/lib.rs index 593fab5f..894c503c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -249,6 +249,23 @@ impl Connection { Connection::open_in_memory_with_flags(Config::default()) } + /// Open a new connection to an ffi database. + /// + /// # Failure + /// + /// Will return `Err` if the underlying DuckDB open call fails. + /// # Safety + /// + /// Need to pass in a valid db instance + #[inline] + pub unsafe fn open_from_raw(raw: ffi::duckdb_database) -> Result { + InnerConnection::new(raw, false).map(|db| Connection { + db: RefCell::new(db), + cache: StatementCache::with_capacity(STATEMENT_CACHE_DEFAULT_CAPACITY), + path: None, // Can we know the path from connection? + }) + } + /// Open a new connection to a DuckDB database. /// /// # Failure @@ -554,7 +571,7 @@ mod test { // this function is never called, but is still type checked; in // particular, calls with specific instantiations will require // that those types are `Send`. - #[allow(dead_code, unconditional_recursion)] + #[allow(dead_code, unconditional_recursion, clippy::extra_unused_type_parameters)] fn ensure_send() { ensure_send::(); }