Rework bundled compilation to support included extensions (#127)

* Get to openssl failure

* It compiles!?

* Add back in the other flags

* Add openssl code

* Add duckdb as a submodule

* Clean up the rest of the code

* Cleanups

* Exclude duckdb-sources

* Feature flags

* Rework to use a tar.gz

* Add to gitignore

* Keep the bindings around

* Fix Linux (exit 0's in the openssl code)

* Remove unused import

* Comments

* Fix clippy errors

* Only include all extensions when bundling and install OpenSSL on windows

* Add line

* Rework slightly

* Try the other solution

* fix openssl issue

Change-Id: I67dd345659e9957cac8590aad1d7dd8ac190cb53

* Revert "fix openssl issue"

This reverts commit 96bdc43.

* fix openssl error

Change-Id: Ifa042c7f4be521a210636c7870e1e225946910e8

---------

Co-authored-by: wangfenjin <[email protected]>

.github/workflows/rust.yaml

@@ -51,7 +51,7 @@ jobs:
           extract_dir: libduckdb
       - run: cargo fmt --all -- --check
         if: matrix.os == 'ubuntu-latest'
-      - run: cargo clippy --all-targets --workspace --features buildtime_bindgen --features modern-full -- -D warnings -A clippy::redundant-closure
+      - run: cargo clippy --all-targets --workspace --features buildtime_bindgen --features "modern-full" -- -D warnings -A clippy::redundant-closure
         if: matrix.os == 'ubuntu-latest'
         name: run cargo clippy
         env:
@@ -101,11 +101,19 @@ jobs:
     runs-on: windows-latest
     steps:
       - uses: actions/checkout@v2
+      - run: echo "VCPKG_ROOT=$env:VCPKG_INSTALLATION_ROOT" | Out-File -FilePath $env:GITHUB_ENV -Append
+      - run: vcpkg install openssl:x64-windows-static-md
+      - uses: actions/cache@v3
+        with:
+          path: ~/.cargo/registry/index
+          key: index-${{ runner.os }}-${{ github.run_number }}
+          restore-keys: |
+            index-${{ runner.os }}-
       - uses: hecrj/setup-rust-action@v1
         with:
           rust-version: stable
           targets: x86_64-pc-windows-msvc
-      - run: cargo test --features "bundled" --features "modern-full"
+      - run: cargo test --features "bundled modern-full extensions-full"
 
   Sanitizer:
     name: Address Sanitizer
@@ -128,7 +136,7 @@ jobs:
           # leak sanitization, but we don't care about backtraces here, so long
           # as the other tests have them.
           RUST_BACKTRACE: "0"
-        run: cargo -Z build-std test --features 'bundled' --features 'modern-full' --target x86_64-unknown-linux-gnu
+        run: cargo -Z build-std test --features "bundled modern-full extensions-full" --target x86_64-unknown-linux-gnu
 
       - uses: wangfenjin/publish-crates@main
         name: cargo publish --dry-run

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "libduckdb-sys/duckdb-sources"]
+	path = libduckdb-sys/duckdb-sources
+	url = [email protected]:duckdb/duckdb.git

Cargo.toml

@@ -23,6 +23,11 @@ members = ["libduckdb-sys"]
 [features]
 default = []
 bundled = ["libduckdb-sys/bundled"]
+httpfs = ["libduckdb-sys/httpfs"]
+json = ["libduckdb-sys/json"]
+parquet = ["libduckdb-sys/parquet"]
+extensions-full = ["httpfs", "json", "parquet"]
+
 buildtime_bindgen = ["libduckdb-sys/buildtime_bindgen"]
 modern-full = [
     "chrono",
@@ -79,4 +84,4 @@ default-target = "x86_64-unknown-linux-gnu"
 
 [package.metadata.playground]
 features = []
-all-features = false
+all-features = false

libduckdb-sys/.gitignore

@@ -0,0 +1,2 @@
+duckdb/*
+._duckdb

libduckdb-sys/Cargo.toml

@@ -11,6 +11,7 @@ keywords = ["duckdb", "ffi", "database"]
 readme = "README.md"
 categories = ["external-ffi-bindings", "database"]
 description = "Native bindings to the libduckdb library, C API"
+exclude = ["duckdb-sources"]
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -19,13 +20,23 @@ default = ["vcpkg", "pkg-config"]
 bundled = ["cc"]
 buildtime_bindgen = ["bindgen", "pkg-config", "vcpkg"]
 
+httpfs = ["bundled"]
+json = ["bundled"]
+parquet = ["bundled"]
+extensions-full = ["httpfs", "json", "parquet"]
+
 [dependencies]
 
 [build-dependencies]
+autocfg = "1.0"
 bindgen = { version = "0.64", optional = true, default-features = false, features = ["runtime"] }
+flate2 = "1.0"
 pkg-config = { version = "0.3.24", optional = true }
 cc = { version = "1.0", features = ["parallel"], optional = true }
 vcpkg = { version = "0.2", optional = true }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = { version = "1.0" }
+tar = "0.4.38"
 
 [dev-dependencies]
-arrow = { version = "34", default-features = false, features = ["ffi"] }
+arrow = { version = "34", default-features = false, features = ["ffi"] }

libduckdb-sys/build.rs

@@ -1,6 +1,9 @@
 use std::env;
 use std::path::Path;
 
+#[cfg(feature = "httpfs")]
+mod openssl;
+
 /// Tells whether we're building for Windows. This is more suitable than a plain
 /// `cfg!(windows)`, since the latter does not properly handle cross-compilation
 ///
@@ -36,13 +39,52 @@ fn main() {
 
 #[cfg(feature = "bundled")]
 mod build_bundled {
+    use std::collections::{HashMap, HashSet};
     use std::path::Path;
 
     use crate::win_target;
 
+    #[derive(serde::Deserialize)]
+    struct Sources {
+        cpp_files: HashSet<String>,
+        include_dirs: HashSet<String>,
+    }
+
+    #[derive(serde::Deserialize)]
+    struct Manifest {
+        base: Sources,
+
+        #[allow(unused)]
+        extensions: HashMap<String, Sources>,
+    }
+
+    #[allow(unused)]
+    fn add_extension(
+        cfg: &mut cc::Build,
+        manifest: &Manifest,
+        extension: &str,
+        cpp_files: &mut HashSet<String>,
+        include_dirs: &mut HashSet<String>,
+    ) {
+        cpp_files.extend(manifest.extensions.get(extension).unwrap().cpp_files.clone());
+        include_dirs.extend(manifest.extensions.get(extension).unwrap().include_dirs.clone());
+        cfg.define(&format!("BUILD_{}_EXTENSION", extension.to_uppercase()), Some("1"));
+    }
+
+    fn untar_archive() {
+        let path = "duckdb.tar.gz";
+
+        let tar_gz = std::fs::File::open(path).expect("archive file");
+        let tar = flate2::read::GzDecoder::new(tar_gz);
+        let mut archive = tar::Archive::new(tar);
+        archive.unpack(".").expect("archive");
+    }
+
     pub fn main(out_dir: &str, out_path: &Path) {
         let lib_name = super::lib_name();
 
+        untar_archive();
+
         if !cfg!(feature = "bundled") {
             // This is just a sanity check, the top level `main` should ensure this.
             panic!("This module should not be used: bundled feature has not been enabled");
@@ -51,7 +93,7 @@ mod build_bundled {
         #[cfg(feature = "buildtime_bindgen")]
         {
             use super::{bindings, HeaderLocation};
-            let header = HeaderLocation::FromPath(format!("{lib_name}/duckdb.h"));
+            let header = HeaderLocation::FromPath(format!("{}/src/include/duckdb.h", lib_name));
             bindings::write_to_out_dir(header, out_path);
         }
         #[cfg(not(feature = "buildtime_bindgen"))]
@@ -60,11 +102,48 @@ mod build_bundled {
             fs::copy(format!("{}/bindgen_bundled_version.rs", lib_name), out_path)
                 .expect("Could not copy bindings to output directory");
         }
-        println!("cargo:rerun-if-changed={lib_name}/duckdb.hpp");
-        println!("cargo:rerun-if-changed={lib_name}/duckdb.cpp");
+
+        let manifest_file = std::fs::File::open(format!("{}/manifest.json", lib_name)).expect("manifest file");
+        let manifest: Manifest = serde_json::from_reader(manifest_file).expect("reading manifest file");
+
+        let mut cpp_files = HashSet::new();
+        let mut include_dirs = HashSet::new();
+
+        cpp_files.extend(manifest.base.cpp_files.clone());
+        // otherwise clippy will remove the clone here...
+        // https://github.com/rust-lang/rust-clippy/issues/9011
+        #[allow(clippy::all)]
+        include_dirs.extend(manifest.base.include_dirs.clone());
+
         let mut cfg = cc::Build::new();
-        cfg.file(format!("{lib_name}/duckdb.cpp"))
-            .cpp(true)
+
+        #[cfg(feature = "httpfs")]
+        {
+            if let Ok((_, openssl_include_dir)) = super::openssl::get_openssl_v2() {
+                cfg.include(openssl_include_dir);
+            }
+            add_extension(&mut cfg, &manifest, "httpfs", &mut cpp_files, &mut include_dirs);
+        }
+
+        #[cfg(feature = "parquet")]
+        add_extension(&mut cfg, &manifest, "parquet", &mut cpp_files, &mut include_dirs);
+
+        #[cfg(feature = "json")]
+        add_extension(&mut cfg, &manifest, "json", &mut cpp_files, &mut include_dirs);
+
+        // Since the manifest controls the set of files, we require it to be changed to know whether
+        // to rebuild the project
+        println!("cargo:rerun-if-changed={}/manifest.json", lib_name);
+
+        cfg.include(lib_name);
+
+        cfg.includes(include_dirs.iter().map(|x| format!("{}/{}", lib_name, x)));
+
+        for f in cpp_files {
+            cfg.file(f);
+        }
+
+        cfg.cpp(true)
             .flag_if_supported("-std=c++11")
             .flag_if_supported("-stdlib=libc++")
             .flag_if_supported("-stdlib=libstdc++")
@@ -76,7 +155,6 @@ mod build_bundled {
         }
 
         cfg.compile(lib_name);
-
         println!("cargo:lib_dir={out_dir}");
     }
 }