use arrow to query data (#8)

* use arrow to query data

.github/workflows/rust.yaml

@@ -12,10 +12,8 @@ env:
 jobs:
   test:
     name: Test ${{ matrix.target }}
-
     strategy:
       fail-fast: true
-
       matrix:
         include:
           #- { target: x86_64-pc-windows-msvc, os: windows-latest }
@@ -28,7 +26,6 @@ jobs:
             #}
 
     runs-on: ${{ matrix.os }}
-
     steps:
       - uses: actions/checkout@v2
       # This has a matcher for test panics, so we use it even though elsewhere
@@ -37,13 +34,21 @@ jobs:
         with:
           rust-version: stable${{ matrix.host }}
           targets: ${{ matrix.target }}
-
-      # - run: cargo build --features bundled --workspace --all-targets --verbose
-      - run: cargo test --features bundled --workspace --all-targets --verbose
-      # - run: cargo test --features bundled --workspace --doc --verbose
+          components: 'rustfmt, clippy'
+      - run: cargo fmt --all -- --check
+      - run: cargo clippy --all-targets --workspace --features bundled
+      - name: Run cargo-tarpaulin
+        uses: actions-rs/[email protected]
+        with:
+          # Intentionally omit time feature until we're on time 0.3, at which
+          # point it should be added to `bundled-full`.
+          args: '--features "bundled"'
+      - name: Upload to codecov.io
+        uses: codecov/codecov-action@v1
 
   sanitizer:
     name: Address Sanitizer
+    needs: test
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -56,62 +61,10 @@ jobs:
         env:
           RUSTFLAGS: -Zsanitizer=address
           RUSTDOCFLAGS: -Zsanitizer=address
-          ASAN_OPTIONS: "detect_stack_use_after_return=1:detect_leaks=0"
+          ASAN_OPTIONS: "detect_stack_use_after_return=1:detect_leaks=1"
           # Work around https://github.com/rust-lang/rust/issues/59125 by
           # disabling backtraces. In an ideal world we'd probably suppress the
           # leak sanitization, but we don't care about backtraces here, so long
           # as the other tests have them.
           RUST_BACKTRACE: "0"
         run: cargo -Z build-std test --features 'bundled' --target x86_64-unknown-linux-gnu
-
-  # Ensure clippy doesn't complain.
-  clippy:
-    name: Clippy
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: hecrj/setup-rust-action@v1
-        with:
-          components: clippy
-      - run: cargo clippy --all-targets --workspace --features bundled
-      # - run: cargo clippy --all-targets --workspace --features bundled -- -D warnings
-
-  # Ensure patch is formatted.
-  fmt:
-    name: Format
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: hecrj/setup-rust-action@v1
-        with:
-          components: rustfmt
-      - run: cargo fmt --all -- --check
-
-  # Detect cases where documentation links don't resolve and such.
-  # doc:
-    # name: Docs
-    # runs-on: ubuntu-latest
-    # steps:
-      # - uses: actions/checkout@v2
-      # - uses: hecrj/setup-rust-action@v1
-        # with:
-          # rust-version: nightly
-      # # Need to use `cargo rustdoc` to actually get it to respect -D
-      # # warnings... Note: this also requires nightly.
-      # - run: cargo rustdoc --features 'bundled' -- -D warnings
-
-  codecov:
-    name: Generate code coverage
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: hecrj/setup-rust-action@v1
-      - name: Run cargo-tarpaulin
-        uses: actions-rs/[email protected]
-        with:
-          # Intentionally omit time feature until we're on time 0.3, at which
-          # point it should be added to `bundled-full`.
-          args: '--features "bundled"'
-
-      - name: Upload to codecov.io
-        uses: codecov/codecov-action@v1

CONTRIBUTING.md

@@ -77,4 +77,10 @@ Use the bundled header file:
 ```shell
 cd ~/github/duckdb-rs
 cargo test --features bundled -- --nocapture
-```
+```
+
+Detect memory leaks:
+```shell
+cd ~/github/duckdb-rs
+ASAN_OPTIONS=detect_leaks=1 ASAN_SYMBOLIZER_PATH=/usr/local/opt/llvm/bin/llvm-symbolizer cargo test --features bundled -- --nocapture
+```

Cargo.toml

@@ -23,7 +23,7 @@ members = ["libduckdb-sys"]
 [features]
 bundled = ["libduckdb-sys/bundled"]
 buildtime_bindgen = ["libduckdb-sys/buildtime_bindgen"]
-default = ["bundled"]
+default = []
 
 [dependencies]
 time = { version = "0.2.23", optional = true }
@@ -41,6 +41,9 @@ memchr = "2.3"
 uuid = { version = "0.8", optional = true }
 smallvec = "1.6.1"
 cast = { version = "0.2", features = ["std", "x128"] }
+arrow = { version = "5.0", default-features = false }
+# arrow = { path = "../arrow-rs/arrow", default-features = false }
+rust_decimal = "1.14"
 
 [dev-dependencies]
 doc-comment = "0.3"

libduckdb-sys/Cargo.toml

@@ -16,7 +16,7 @@ description = "Native bindings to the libduckdb library, C API"
 
 [features]
 default = ["vcpkg", "pkg-config"]
-bundled = ["cc"]
+bundled = ["cc", "buildtime_bindgen"]
 buildtime_bindgen = ["bindgen", "pkg-config", "vcpkg"]
 
 [dependencies]
@@ -25,4 +25,7 @@ buildtime_bindgen = ["bindgen", "pkg-config", "vcpkg"]
 bindgen = { version = "0.58", optional = true, default-features = false, features = ["runtime"] }
 pkg-config = { version = "0.3.19", optional = true }
 cc = { version = "1.0", features = ["parallel"], optional = true }
-vcpkg = { version = "0.2", optional = true }
+vcpkg = { version = "0.2", optional = true }
+
+[dev-dependencies]
+arrow = { version = "5.0", default-features = false }

libduckdb-sys/build.rs

@@ -67,6 +67,8 @@ mod build_bundled {
             .shared_flag(true)
             .pic(true)
             .flag_if_supported("-std=c++11")
+            .flag_if_supported("-stdlib=libc++")
+            .flag_if_supported("-stdlib=libstdc++")
             .warnings(false);
         cfg.compile(lib_name);
 

libduckdb-sys/duckdb/bindgen_bundled_version.rs

@@ -10344,18 +10344,22 @@ pub const DUCKDB_TYPE_DUCKDB_TYPE_TINYINT: DUCKDB_TYPE = 2;
 pub const DUCKDB_TYPE_DUCKDB_TYPE_SMALLINT: DUCKDB_TYPE = 3;
 pub const DUCKDB_TYPE_DUCKDB_TYPE_INTEGER: DUCKDB_TYPE = 4;
 pub const DUCKDB_TYPE_DUCKDB_TYPE_BIGINT: DUCKDB_TYPE = 5;
-pub const DUCKDB_TYPE_DUCKDB_TYPE_FLOAT: DUCKDB_TYPE = 6;
-pub const DUCKDB_TYPE_DUCKDB_TYPE_DOUBLE: DUCKDB_TYPE = 7;
-pub const DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP: DUCKDB_TYPE = 8;
-pub const DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_S: DUCKDB_TYPE = 9;
-pub const DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_NS: DUCKDB_TYPE = 10;
-pub const DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_MS: DUCKDB_TYPE = 11;
-pub const DUCKDB_TYPE_DUCKDB_TYPE_DATE: DUCKDB_TYPE = 12;
-pub const DUCKDB_TYPE_DUCKDB_TYPE_TIME: DUCKDB_TYPE = 13;
-pub const DUCKDB_TYPE_DUCKDB_TYPE_INTERVAL: DUCKDB_TYPE = 14;
-pub const DUCKDB_TYPE_DUCKDB_TYPE_HUGEINT: DUCKDB_TYPE = 15;
-pub const DUCKDB_TYPE_DUCKDB_TYPE_VARCHAR: DUCKDB_TYPE = 16;
-pub const DUCKDB_TYPE_DUCKDB_TYPE_BLOB: DUCKDB_TYPE = 17;
+pub const DUCKDB_TYPE_DUCKDB_TYPE_UTINYINT: DUCKDB_TYPE = 6;
+pub const DUCKDB_TYPE_DUCKDB_TYPE_USMALLINT: DUCKDB_TYPE = 7;
+pub const DUCKDB_TYPE_DUCKDB_TYPE_UINTEGER: DUCKDB_TYPE = 8;
+pub const DUCKDB_TYPE_DUCKDB_TYPE_UBIGINT: DUCKDB_TYPE = 9;
+pub const DUCKDB_TYPE_DUCKDB_TYPE_FLOAT: DUCKDB_TYPE = 10;
+pub const DUCKDB_TYPE_DUCKDB_TYPE_DOUBLE: DUCKDB_TYPE = 11;
+pub const DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP: DUCKDB_TYPE = 12;
+pub const DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_S: DUCKDB_TYPE = 13;
+pub const DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_NS: DUCKDB_TYPE = 14;
+pub const DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_MS: DUCKDB_TYPE = 15;
+pub const DUCKDB_TYPE_DUCKDB_TYPE_DATE: DUCKDB_TYPE = 16;
+pub const DUCKDB_TYPE_DUCKDB_TYPE_TIME: DUCKDB_TYPE = 17;
+pub const DUCKDB_TYPE_DUCKDB_TYPE_INTERVAL: DUCKDB_TYPE = 18;
+pub const DUCKDB_TYPE_DUCKDB_TYPE_HUGEINT: DUCKDB_TYPE = 19;
+pub const DUCKDB_TYPE_DUCKDB_TYPE_VARCHAR: DUCKDB_TYPE = 20;
+pub const DUCKDB_TYPE_DUCKDB_TYPE_BLOB: DUCKDB_TYPE = 21;
 pub type DUCKDB_TYPE = ::std::os::raw::c_uint;
 pub use self::DUCKDB_TYPE as duckdb_type;
 #[repr(C)]
@@ -10399,18 +10403,18 @@ pub struct duckdb_time {
     pub hour: i8,
     pub min: i8,
     pub sec: i8,
-    pub micros: i16,
+    pub micros: i32,
 }
 #[test]
 fn bindgen_test_layout_duckdb_time() {
     assert_eq!(
         ::std::mem::size_of::<duckdb_time>(),
-        6usize,
+        8usize,
         concat!("Size of: ", stringify!(duckdb_time))
     );
     assert_eq!(
         ::std::mem::align_of::<duckdb_time>(),
-        2usize,
+        4usize,
         concat!("Alignment of ", stringify!(duckdb_time))
     );
     assert_eq!(
@@ -10698,14 +10702,97 @@ pub type duckdb_database = *mut ::std::os::raw::c_void;
 pub type duckdb_connection = *mut ::std::os::raw::c_void;
 pub type duckdb_prepared_statement = *mut ::std::os::raw::c_void;
 pub type duckdb_appender = *mut ::std::os::raw::c_void;
+pub type duckdb_arrow = *mut ::std::os::raw::c_void;
+pub type duckdb_config = *mut ::std::os::raw::c_void;
+pub type duckdb_arrow_schema = *mut ::std::os::raw::c_void;
+pub type duckdb_arrow_array = *mut ::std::os::raw::c_void;
 pub const duckdb_state_DuckDBSuccess: duckdb_state = 0;
 pub const duckdb_state_DuckDBError: duckdb_state = 1;
 pub type duckdb_state = ::std::os::raw::c_uint;
+extern "C" {
+    #[doc = "! query duckdb result as arrow data structure"]
+    pub fn duckdb_query_arrow(
+        connection: duckdb_connection,
+        query: *const ::std::os::raw::c_char,
+        out_result: *mut duckdb_arrow,
+    ) -> duckdb_state;
+}
+extern "C" {
+    #[doc = "! get arrow schema"]
+    pub fn duckdb_query_arrow_schema(result: duckdb_arrow, out_schema: *mut duckdb_arrow_schema) -> duckdb_state;
+}
+extern "C" {
+    #[doc = "! get arrow data array"]
+    #[doc = "! This function can be called multiple time to get next chunks, which will free the previous out_array."]
+    #[doc = "! So consume the out_array before call this function again"]
+    pub fn duckdb_query_arrow_array(result: duckdb_arrow, out_array: *mut duckdb_arrow_array) -> duckdb_state;
+}
+extern "C" {
+    #[doc = "! get arrow row count"]
+    pub fn duckdb_arrow_row_count(result: duckdb_arrow) -> idx_t;
+}
+extern "C" {
+    #[doc = "! get arrow column count"]
+    pub fn duckdb_arrow_column_count(result: duckdb_arrow) -> idx_t;
+}
+extern "C" {
+    #[doc = "! get arrow rows changed"]
+    pub fn duckdb_arrow_rows_changed(result: duckdb_arrow) -> idx_t;
+}
+extern "C" {
+    #[doc = "! get arrow error message"]
+    pub fn duckdb_query_arrow_error(result: duckdb_arrow) -> *const ::std::os::raw::c_char;
+}
+extern "C" {
+    #[doc = "! Destroys the arrow result"]
+    pub fn duckdb_destroy_arrow(result: *mut duckdb_arrow);
+}
+extern "C" {
+    #[doc = "! Creates a DuckDB configuration object. The created object must be destroyed with duckdb_destroy_config."]
+    pub fn duckdb_create_config(out_config: *mut duckdb_config) -> duckdb_state;
+}
+extern "C" {
+    #[doc = "! Returns the amount of config options available."]
+    #[doc = "! Should not be called in a loop as it internally loops over all the options."]
+    pub fn duckdb_config_count() -> size_t;
+}
+extern "C" {
+    #[doc = "! Returns the config name and description for the config at the specified index"]
+    #[doc = "! The result MUST NOT be freed"]
+    #[doc = "! Returns failure if the index is out of range (i.e. >= duckdb_config_count)"]
+    pub fn duckdb_get_config_flag(
+        index: size_t,
+        out_name: *mut *const ::std::os::raw::c_char,
+        out_description: *mut *const ::std::os::raw::c_char,
+    ) -> duckdb_state;
+}
+extern "C" {
+    #[doc = "! Sets the specified config option for the configuration"]
+    pub fn duckdb_set_config(
+        config: duckdb_config,
+        name: *const ::std::os::raw::c_char,
+        option: *const ::std::os::raw::c_char,
+    ) -> duckdb_state;
+}
+extern "C" {
+    #[doc = "! Destroys a config object created with duckdb_create_config"]
+    pub fn duckdb_destroy_config(config: *mut duckdb_config);
+}
 extern "C" {
     #[doc = "! Opens a database file at the given path (nullptr for in-memory). Returns DuckDBSuccess on success, or DuckDBError on"]
     #[doc = "! failure. [OUT: database]"]
     pub fn duckdb_open(path: *const ::std::os::raw::c_char, out_database: *mut duckdb_database) -> duckdb_state;
 }
+extern "C" {
+    #[doc = "! Opens a database file at the given path using the specified configuration"]
+    #[doc = "! If error is set the error will be reported"]
+    pub fn duckdb_open_ext(
+        path: *const ::std::os::raw::c_char,
+        out_database: *mut duckdb_database,
+        config: duckdb_config,
+        error: *mut *mut ::std::os::raw::c_char,
+    ) -> duckdb_state;
+}
 extern "C" {
     #[doc = "! Closes the database."]
     pub fn duckdb_close(database: *mut duckdb_database);
@@ -10806,6 +10893,9 @@ extern "C" {
         out_prepared_statement: *mut duckdb_prepared_statement,
     ) -> duckdb_state;
 }
+extern "C" {
+    pub fn duckdb_prepare_error(prepared_statement: duckdb_prepared_statement) -> *const ::std::os::raw::c_char;
+}
 extern "C" {
     pub fn duckdb_nparams(prepared_statement: duckdb_prepared_statement, nparams_out: *mut idx_t) -> duckdb_state;
 }
@@ -10900,6 +10990,13 @@ extern "C" {
         out_result: *mut duckdb_result,
     ) -> duckdb_state;
 }
+extern "C" {
+    #[doc = "! Executes the prepared statements with currently bound parameters and return arrow result"]
+    pub fn duckdb_execute_prepared_arrow(
+        prepared_statement: duckdb_prepared_statement,
+        out_result: *mut duckdb_arrow,
+    ) -> duckdb_state;
+}
 extern "C" {
     #[doc = "! Destroys the specified prepared statement descriptor"]
     pub fn duckdb_destroy_prepare(prepared_statement: *mut duckdb_prepared_statement);