From 04543bfdc4655b8bf25f578916190001fb4f6d8f Mon Sep 17 00:00:00 2001 From: Swoorup Joshi Date: Tue, 11 Jun 2024 20:04:55 +1000 Subject: [PATCH] Duckdb updates --- .rustfmt.toml | 9 +- Cargo.toml | 15 +- crates/duckdb/Cargo.toml | 20 +- crates/duckdb/benches/issue-282/.gitignore | 1 + .../benches/issue-282/generate-database.sh | 19 + crates/duckdb/benches/issue-282/main.rs | 125 ++ crates/duckdb/benches/issue-282/output.csv | 1063 +++++++++++++++++ crates/duckdb/examples/appender.rs | 50 +- crates/duckdb/examples/hello-ext/main.rs | 14 +- crates/duckdb/src/appender/arrow.rs | 82 -- crates/duckdb/src/appender/mod.rs | 213 +++- crates/duckdb/src/core/column_info.rs | 133 +++ crates/duckdb/src/core/data_chunk.rs | 144 +++ crates/duckdb/src/core/logical_type.rs | 524 ++++++++ crates/duckdb/src/core/mod.rs | 8 + crates/duckdb/src/core/vector.rs | 459 +++++++ crates/duckdb/src/error.rs | 28 +- crates/duckdb/src/extension.rs | 6 +- crates/duckdb/src/lib.rs | 1 + crates/duckdb/src/statement.rs | 58 +- crates/duckdb/src/vtab/arrow.rs | 536 +++++---- crates/duckdb/src/vtab/data_chunk.rs | 133 --- crates/duckdb/src/vtab/excel.rs | 27 +- crates/duckdb/src/vtab/function.rs | 8 +- crates/duckdb/src/vtab/logical_type.rs | 341 ------ crates/duckdb/src/vtab/mod.rs | 79 +- crates/duckdb/src/vtab/vector.rs | 282 ----- crates/libduckdb-sys/src/lib.rs | 3 + crates/libduckdb-sys/src/types.rs | 43 + 29 files changed, 3193 insertions(+), 1231 deletions(-) create mode 100644 crates/duckdb/benches/issue-282/.gitignore create mode 100755 crates/duckdb/benches/issue-282/generate-database.sh create mode 100644 crates/duckdb/benches/issue-282/main.rs create mode 100644 crates/duckdb/benches/issue-282/output.csv delete mode 100644 crates/duckdb/src/appender/arrow.rs create mode 100644 crates/duckdb/src/core/column_info.rs create mode 100644 crates/duckdb/src/core/data_chunk.rs create mode 100644 crates/duckdb/src/core/logical_type.rs create mode 100644 crates/duckdb/src/core/mod.rs create mode 100644 crates/duckdb/src/core/vector.rs delete mode 100644 crates/duckdb/src/vtab/data_chunk.rs delete mode 100644 crates/duckdb/src/vtab/logical_type.rs delete mode 100644 crates/duckdb/src/vtab/vector.rs create mode 100644 crates/libduckdb-sys/src/types.rs diff --git a/.rustfmt.toml b/.rustfmt.toml index 0b7ea326..31fbd8d1 100644 --- a/.rustfmt.toml +++ b/.rustfmt.toml @@ -1,2 +1,7 @@ -max_width = 120 -imports_granularity = "Crate" \ No newline at end of file +# indent_style = "Block" +reorder_imports = true +max_width=120 +fn_call_width=72 +# tab_spaces = 2 +# group_imports="StdExternalCrate" +# imports_granularity = "Module" \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 691d06e8..4bb824de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,7 @@ duckdb-loadable-macros = { version = "0.1.1", path = "crates/duckdb-loadable-mac autocfg = "1.0" bindgen = { version = "0.69", default-features = false } byteorder = "1.3" -calamine = "0.22.0" +calamine = "0.24" cast = "0.3" cc = "1.0" chrono = "0.4.22" @@ -34,13 +34,14 @@ doc-comment = "0.3" fallible-iterator = "0.3" fallible-streaming-iterator = "0.1" flate2 = "1.0" -hashlink = "0.8" +hashlink = "0.9" lazy_static = "1.4" memchr = "2.3" num = { version = "0.4", default-features = false } pkg-config = "0.3.24" -polars = "0.35.4" -polars-core = "0.35.4" +polars = "0.38" +polars-core = "0.38" +derive_more = "0.99" pretty_assertions = "1.4.0" proc-macro2 = "1.0.56" quote = "1.0.21" @@ -55,9 +56,15 @@ strum = "0.25" syn = "2.0.15" tar = "0.4.38" tempdir = "0.3.7" +thiserror = "1.0" tempfile = "3.1.0" unicase = "2.6.0" url = "2.1" uuid = "1.0" vcpkg = "0.2" arrow = { version = "52", default-features = false } +rusqlite = "0.31" +arrow_convert = "0.6" +itertools = "0.13" +criterion = { version = "0.5", features = [ "html_reports"] } +include_absolute_path = "0.1" \ No newline at end of file diff --git a/crates/duckdb/Cargo.toml b/crates/duckdb/Cargo.toml index 6a26438c..5a29e523 100644 --- a/crates/duckdb/Cargo.toml +++ b/crates/duckdb/Cargo.toml @@ -16,6 +16,7 @@ description = "Ergonomic wrapper for DuckDB" [lib] name = "duckdb" +bench = false [features] default = [] @@ -58,9 +59,12 @@ strum = { workspace = true, features = ["derive"] } r2d2 = { workspace = true, optional = true } calamine = { workspace = true, optional = true } num = { workspace = true, features = ["std"], optional = true } +derive_more = { workspace = true } duckdb-loadable-macros = { workspace = true, optional = true } polars = { workspace = true, features = ["dtype-full"], optional = true } -num-integer = {version = "0.1.46"} +thiserror = { workspace = true } +arrow_convert = { workspace = true } +itertools = { workspace = true } [dev-dependencies] doc-comment = { workspace = true } @@ -73,13 +77,14 @@ rand = { workspace = true } tempdir = { workspace = true } polars-core = { workspace = true } pretty_assertions = { workspace = true } -# criterion = "0.3" +rusqlite = { workspace = true } +criterion = { workspace = true } +include_absolute_path = { workspace = true } # [[bench]] # name = "data_types" # harness = false - [package.metadata.docs.rs] features = [] all-features = false @@ -94,3 +99,12 @@ all-features = false name = "hello-ext" crate-type = ["cdylib"] required-features = ["vtab-loadable"] + +[[example]] +name = "appender" +crate-type = ["cdylib"] +required-features = ["appender-arrow"] + +[[bench]] +name = "issue-282" +harness = false diff --git a/crates/duckdb/benches/issue-282/.gitignore b/crates/duckdb/benches/issue-282/.gitignore new file mode 100644 index 00000000..6afd071c --- /dev/null +++ b/crates/duckdb/benches/issue-282/.gitignore @@ -0,0 +1 @@ +db.* \ No newline at end of file diff --git a/crates/duckdb/benches/issue-282/generate-database.sh b/crates/duckdb/benches/issue-282/generate-database.sh new file mode 100755 index 00000000..8e62ae2e --- /dev/null +++ b/crates/duckdb/benches/issue-282/generate-database.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# SQLite +sqlite3 db.sqlite < rusqlite::Connection { + rusqlite::Connection::open(include_absolute_path!("./db.sqlite")).unwrap() +} + +pub fn duck_db() -> duckdb::Connection { + duckdb::Connection::open(include_absolute_path!("./db.duckdb")).unwrap() +} + +#[derive(Debug, ArrowField, ArrowSerialize, ArrowDeserialize)] +struct Income { + created_at: Option, + amount: Option, + category_id: i32, + wallet_id: Option, + meta: Option, +} + +impl Income { + fn select_duckdb_arrow( + conn: &duckdb::Connection, + start: u32, + end: u32, + ) -> Result, Box> { + let sql = format!( + "SELECT created_at, amount, category_id, wallet_id, meta \ + FROM 'income' \ + WHERE created_at >= {} AND created_at <= {}", + start, end + ); + let mut stmt = conn.prepare_cached(&sql)?; + let result = stmt.query_arrow_deserialized::([])?; + Ok(result) + } + + fn select_duckdb(conn: &duckdb::Connection, start: u32, end: u32) -> Result, Box> { + let mut arr = Vec::new(); + let sql = format!( + "SELECT created_at, amount, category_id, wallet_id, meta \ + FROM 'income' \ + WHERE created_at >= {} AND created_at <= {}", + start, end + ); + let mut stmt = conn.prepare_cached(&sql)?; + let result_iter = stmt.query_map([], |row| { + Ok(Self { + created_at: row.get(0)?, + amount: row.get(1)?, + category_id: row.get(2)?, + wallet_id: row.get(3)?, + meta: row.get(4)?, + }) + })?; + for result in result_iter { + arr.push(result?); + } + Ok(arr) + } + + fn select_sqlite( + conn: &rusqlite::Connection, + start: u32, + end: u32, + ) -> Result, Box> { + let mut arr = Vec::new(); + let sql = format!( + "SELECT created_at, amount, category_id, wallet_id, meta \ + FROM 'income' \ + WHERE created_at >= {} AND created_at <= {}", + start, end + ); + let mut stmt = conn.prepare(&sql)?; + let result_iter = stmt.query_map([], |row| { + Ok(Self { + created_at: row.get(0)?, + amount: row.get(1)?, + category_id: row.get(2)?, + wallet_id: row.get(3)?, + meta: row.get(4)?, + }) + })?; + for result in result_iter { + arr.push(result?); + } + Ok(arr) + } +} + +fn bench_sqlite(c: &mut Criterion) { + let sqlite_conn = sqlite_db(); + c.bench_function("sqlite_test", |b| { + b.iter(|| { + let out = Income::select_sqlite(&sqlite_conn, 1709292049, 1711375239).unwrap(); + out.len() + }) + }); +} + +fn bench_duckdb(c: &mut Criterion) { + let duckdb_conn = duck_db(); + c.bench_function("duckdb_test", |b| { + b.iter(|| { + let out = Income::select_duckdb(&duckdb_conn, 1709292049, 1711375239).unwrap(); + out.len() + }) + }); +} + +fn bench_duckdb_arrow(c: &mut Criterion) { + let duckdb_conn = duck_db(); + c.bench_function("duckdb_test_arrow", |b| { + b.iter(|| { + let out = Income::select_duckdb_arrow(&duckdb_conn, 1709292049, 1711375239).unwrap(); + out.len() + }) + }); +} + +criterion_group!(benches, bench_duckdb_arrow); +// criterion_group!(benches, bench_sqlite, bench_duckdb, bench_duckdb_arrow); +criterion_main!(benches); diff --git a/crates/duckdb/benches/issue-282/output.csv b/crates/duckdb/benches/issue-282/output.csv new file mode 100644 index 00000000..33996243 --- /dev/null +++ b/crates/duckdb/benches/issue-282/output.csv @@ -0,0 +1,1063 @@ +id,created_at,amount,category_id,wallet_id,meta +1,1665991212,800.0,19,2,"{""description"":""some"",""duration"":40}" +2,1666011332,3000.0,19,2,"{""description"":""some"",""duration"":40}" +3,1666092255,800.0,19,2,"{""description"":""some"",""duration"":40}" +4,1666092280,50.0,19,2,"{""description"":""some"",""duration"":40}" +5,1666092522,400.0,19,2,"{""description"":""some"",""duration"":40}" +6,1666092603,500.0,19,2,"{""description"":""some"",""duration"":40}" +7,1666099128,1800.0,19,2,"{""description"":""some"",""duration"":40}" +8,1666180623,1000.0,19,2,"{""description"":""some"",""duration"":40}" +9,1666259498,2500.0,19,2,"{""description"":""some"",""duration"":40}" +10,1666342098,1800.0,19,2,"{""description"":""some"",""duration"":40}" +11,1666343577,700.0,19,2,"{""description"":""some"",""duration"":40}" +12,1666436034,3000.0,19,2,"{""description"":""some"",""duration"":40}" +13,1666436046,1500.0,19,2,"{""description"":""some"",""duration"":40}" +14,1666618507,3000.0,19,2,"{""description"":""some"",""duration"":40}" +15,1666618523,800.0,19,2,"{""description"":""some"",""duration"":40}" +16,1666618538,200.0,19,2,"{""description"":""some"",""duration"":40}" +17,1666781099,800.0,19,2,"{""description"":""some"",""duration"":40}" +18,1666781118,100.0,19,2,"{""description"":""some"",""duration"":40}" +19,1666880169,3000.0,19,2,"{""description"":""some"",""duration"":40}" +20,1666880206,200.0,19,2,"{""description"":""some"",""duration"":40}" +21,1666880218,800.0,19,2,"{""description"":""some"",""duration"":40}" +22,1666880234,400.0,19,2,"{""description"":""some"",""duration"":40}" +23,1666880301,500.0,19,2,"{""description"":""some"",""duration"":40}" +24,1666967736,800.0,19,2,"{""description"":""some"",""duration"":40}" +25,1666967754,1800.0,19,2,"{""description"":""some"",""duration"":40}" +26,1666967769,3000.0,19,2,"{""description"":""some"",""duration"":40}" +27,1667039365,500.0,19,2,"{""description"":""some"",""duration"":40}" +28,1667039381,200.0,19,2,"{""description"":""some"",""duration"":40}" +29,1667042456,1800.0,19,2,"{""description"":""some"",""duration"":40}" +30,1667045415,800.0,19,2,"{""description"":""some"",""duration"":40}" +31,1667045434,400.0,19,2,"{""description"":""some"",""duration"":40}" +32,1667220004,500.0,19,2,"{""description"":""some"",""duration"":40}" +33,1667220014,200.0,19,2,"{""description"":""some"",""duration"":40}" +34,1667298931,800.0,19,2,"{""description"":""some"",""duration"":40}" +35,1667298941,800.0,19,2,"{""description"":""some"",""duration"":40}" +36,1667380456,800.0,19,2,"{""description"":""some"",""duration"":40}" +37,1667380467,400.0,19,2,"{""description"":""some"",""duration"":40}" +38,1667395164,3000.0,19,2,"{""description"":""some"",""duration"":40}" +39,1667473802,3000.0,19,2,"{""description"":""some"",""duration"":40}" +40,1667479293,200.0,19,2,"{""description"":""some"",""duration"":40}" +41,1667557832,500.0,19,2,"{""description"":""some"",""duration"":40}" +42,1667557842,500.0,19,2,"{""description"":""some"",""duration"":40}" +43,1667650725,3000.0,19,2,"{""description"":""some"",""duration"":40}" +44,1667930428,800.0,19,2,"{""description"":""some"",""duration"":40}" +45,1667987940,800.0,19,2,"{""description"":""some"",""duration"":40}" +46,1667987957,800.0,19,2,"{""description"":""some"",""duration"":40}" +47,1668002224,800.0,19,2,"{""description"":""some"",""duration"":40}" +48,1668002234,800.0,19,2,"{""description"":""some"",""duration"":40}" +49,1668002245,800.0,19,2,"{""description"":""some"",""duration"":40}" +50,1668064898,4000.0,19,2,"{""description"":""some"",""duration"":40}" +51,1668081047,1800.0,19,2,"{""description"":""some"",""duration"":40}" +52,1668090135,800.0,19,2,"{""description"":""some"",""duration"":40}" +53,1668090156,800.0,19,2,"{""description"":""some"",""duration"":40}" +54,1668090174,800.0,19,2,"{""description"":""some"",""duration"":40}" +55,1668090191,400.0,19,2,"{""description"":""some"",""duration"":40}" +56,1668176260,3000.0,19,2,"{""description"":""some"",""duration"":40}" +57,1668249763,1800.0,19,2,"{""description"":""some"",""duration"":40}" +58,1668249774,200.0,19,2,"{""description"":""some"",""duration"":40}" +59,1668422461,1800.0,19,2,"{""description"":""some"",""duration"":40}" +60,1668424057,800.0,19,2,"{""description"":""some"",""duration"":40}" +61,1668429753,300.0,19,2,"{""description"":""some"",""duration"":40}" +62,1668429903,800.0,19,2,"{""description"":""some"",""duration"":40}" +63,1668429920,400.0,19,2,"{""description"":""some"",""duration"":40}" +64,1668432205,500.0,19,2,"{""description"":""some"",""duration"":40}" +65,1668432218,200.0,19,2,"{""description"":""some"",""duration"":40}" +66,1668501917,800.0,19,2,"{""description"":""some"",""duration"":40}" +67,1668501955,200.0,19,2,"{""description"":""some"",""duration"":40}" +68,1668670998,800.0,19,2,"{""description"":""some"",""duration"":40}" +69,1668683440,1800.0,19,2,"{""description"":""some"",""duration"":40}" +70,1668772715,800.0,19,2,"{""description"":""some"",""duration"":40}" +71,1668889934,800.0,19,2,"{""description"":""some"",""duration"":40}" +72,1668889945,800.0,19,2,"{""description"":""some"",""duration"":40}" +73,1668889959,800.0,19,2,"{""description"":""some"",""duration"":40}" +74,1668889967,200.0,19,2,"{""description"":""some"",""duration"":40}" +75,1668889981,400.0,19,2,"{""description"":""some"",""duration"":40}" +76,1668889999,1800.0,19,2,"{""description"":""some"",""duration"":40}" +77,1669032406,3000.0,19,2,"{""description"":""some"",""duration"":40}" +78,1669032416,800.0,19,2,"{""description"":""some"",""duration"":40}" +79,1669039590,1800.0,19,2,"{""description"":""some"",""duration"":40}" +80,1669039602,500.0,19,2,"{""description"":""some"",""duration"":40}" +81,1669043306,800.0,19,2,"{""description"":""some"",""duration"":40}" +82,1669189532,800.0,19,2,"{""description"":""some"",""duration"":40}" +83,1669189541,800.0,19,2,"{""description"":""some"",""duration"":40}" +84,1669199163,800.0,19,2,"{""description"":""some"",""duration"":40}" +85,1669292359,500.0,19,2,"{""description"":""some"",""duration"":40}" +86,1669292368,200.0,19,2,"{""description"":""some"",""duration"":40}" +87,1669292378,800.0,19,2,"{""description"":""some"",""duration"":40}" +88,1669292398,1800.0,19,2,"{""description"":""some"",""duration"":40}" +89,1669371839,500.0,19,2,"{""description"":""some"",""duration"":40}" +90,1669371848,300.0,19,2,"{""description"":""some"",""duration"":40}" +91,1669371860,800.0,19,2,"{""description"":""some"",""duration"":40}" +92,1669410919,1800.0,19,2,"{""description"":""some"",""duration"":40}" +93,1669410967,400.0,19,2,"{""description"":""some"",""duration"":40}" +94,1669411110,500.0,19,2,"{""description"":""some"",""duration"":40}" +95,1669486955,2000.0,19,2,"{""description"":""some"",""duration"":40}" +96,1669637109,1800.0,19,2,"{""description"":""some"",""duration"":40}" +97,1669637124,200.0,19,2,"{""description"":""some"",""duration"":40}" +98,1669637135,150.0,19,2,"{""description"":""some"",""duration"":40}" +99,1669712508,3000.0,19,2,"{""description"":""some"",""duration"":40}" +100,1669720639,500.0,19,2,"{""description"":""some"",""duration"":40}" +101,1669747477,800.0,19,2,"{""description"":""some"",""duration"":40}" +102,1669813127,800.0,19,2,"{""description"":""some"",""duration"":40}" +103,1669813136,800.0,19,2,"{""description"":""some"",""duration"":40}" +104,1669813151,400.0,19,2,"{""description"":""some"",""duration"":40}" +105,1669813187,2000.0,19,2,"{""description"":""some"",""duration"":40}" +106,1669889997,800.0,19,2,"{""description"":""some"",""duration"":40}" +107,1669890009,5000.0,19,2,"{""description"":""some"",""duration"":40}" +108,1669907717,500.0,19,2,"{""description"":""some"",""duration"":40}" +109,1669907726,800.0,19,2,"{""description"":""some"",""duration"":40}" +110,1669976745,6500.0,19,2,"{""description"":""some"",""duration"":40}" +111,1669976949,1000.0,19,2,"{""description"":""some"",""duration"":40}" +112,1669996768,500.0,19,2,"{""description"":""some"",""duration"":40}" +113,1670067222,3000.0,19,2,"{""description"":""some"",""duration"":40}" +114,1670067231,800.0,19,2,"{""description"":""some"",""duration"":40}" +115,1670067239,800.0,19,2,"{""description"":""some"",""duration"":40}" +116,1670067251,500.0,19,2,"{""description"":""some"",""duration"":40}" +117,1670151315,4500.0,19,2,"{""description"":""some"",""duration"":40}" +118,1670220965,1500.0,19,2,"{""description"":""some"",""duration"":40}" +119,1670228907,700.0,19,2,"{""description"":""some"",""duration"":40}" +120,1670240285,500.0,19,2,"{""description"":""some"",""duration"":40}" +121,1670250803,1000.0,19,2,"{""description"":""some"",""duration"":40}" +122,1670250811,1000.0,19,2,"{""description"":""some"",""duration"":40}" +123,1670250822,1800.0,19,2,"{""description"":""some"",""duration"":40}" +124,1670251121,500.0,19,2,"{""description"":""some"",""duration"":40}" +125,1670259887,3500.0,19,2,"{""description"":""some"",""duration"":40}" +126,1670326583,1800.0,19,2,"{""description"":""some"",""duration"":40}" +127,1670403142,1000.0,19,2,"{""description"":""some"",""duration"":40}" +128,1670506048,1000.0,19,2,"{""description"":""some"",""duration"":40}" +129,1670586024,1000.0,19,2,"{""description"":""some"",""duration"":40}" +130,1670586034,1000.0,19,2,"{""description"":""some"",""duration"":40}" +131,1670586043,500.0,19,2,"{""description"":""some"",""duration"":40}" +132,1670864755,1000.0,19,2,"{""description"":""some"",""duration"":40}" +133,1670864825,400.0,19,2,"{""description"":""some"",""duration"":40}" +134,1670864836,500.0,19,2,"{""description"":""some"",""duration"":40}" +135,1670864848,1000.0,19,2,"{""description"":""some"",""duration"":40}" +136,1670864867,1000.0,19,2,"{""description"":""some"",""duration"":40}" +137,1671450861,1000.0,19,2,"{""description"":""some"",""duration"":40}" +138,1671450877,400.0,19,2,"{""description"":""some"",""duration"":40}" +139,1671450892,2000.0,19,2,"{""description"":""some"",""duration"":40}" +140,1671521885,1800.0,19,2,"{""description"":""some"",""duration"":40}" +141,1671525043,1000.0,19,2,"{""description"":""some"",""duration"":40}" +142,1671528490,1000.0,19,2,"{""description"":""some"",""duration"":40}" +143,1671528845,1000.0,19,2,"{""description"":""some"",""duration"":40}" +144,1671605508,11000.0,19,2,"{""description"":""some"",""duration"":40}" +145,1671702072,1000.0,19,2,"{""description"":""some"",""duration"":40}" +146,1671704433,1000.0,19,2,"{""description"":""some"",""duration"":40}" +147,1671710054,1000.0,19,2,"{""description"":""some"",""duration"":40}" +148,1671710064,500.0,19,2,"{""description"":""some"",""duration"":40}" +149,1671720933,1000.0,19,2,"{""description"":""some"",""duration"":40}" +150,1671798277,1000.0,19,2,"{""description"":""some"",""duration"":40}" +151,1671895489,1500.0,19,2,"{""description"":""some"",""duration"":40}" +152,1671895508,1000.0,19,2,"{""description"":""some"",""duration"":40}" +153,1671895524,3500.0,19,2,"{""description"":""some"",""duration"":40}" +154,1671895540,1000.0,19,2,"{""description"":""some"",""duration"":40}" +155,1671895551,3500.0,19,2,"{""description"":""some"",""duration"":40}" +156,1672043148,500.0,19,2,"{""description"":""some"",""duration"":40}" +157,1672043159,1000.0,19,2,"{""description"":""some"",""duration"":40}" +158,1672043170,700.0,19,2,"{""description"":""some"",""duration"":40}" +159,1672047482,700.0,19,2,"{""description"":""some"",""duration"":40}" +160,1672047501,300.0,19,2,"{""description"":""some"",""duration"":40}" +161,1672061644,1000.0,19,2,"{""description"":""some"",""duration"":40}" +162,1672061653,1800.0,19,2,"{""description"":""some"",""duration"":40}" +163,1672061669,800.0,19,2,"{""description"":""some"",""duration"":40}" +164,1672061683,200.0,19,2,"{""description"":""some"",""duration"":40}" +165,1672061696,1000.0,19,2,"{""description"":""some"",""duration"":40}" +166,1672125296,800.0,19,2,"{""description"":""some"",""duration"":40}" +167,1672125307,1000.0,19,2,"{""description"":""some"",""duration"":40}" +168,1672129676,1000.0,19,2,"{""description"":""some"",""duration"":40}" +169,1672129684,400.0,19,2,"{""description"":""some"",""duration"":40}" +170,1672135200,1000.0,19,2,"{""description"":""some"",""duration"":40}" +171,1672135217,800.0,19,2,"{""description"":""some"",""duration"":40}" +172,1672135227,200.0,19,2,"{""description"":""some"",""duration"":40}" +173,1672141191,2000.0,19,2,"{""description"":""some"",""duration"":40}" +174,1672145495,1000.0,19,2,"{""description"":""some"",""duration"":40}" +175,1672145512,400.0,19,2,"{""description"":""some"",""duration"":40}" +176,1672148170,1000.0,19,2,"{""description"":""some"",""duration"":40}" +177,1672148326,1000.0,19,2,"{""description"":""some"",""duration"":40}" +178,1672218345,1000.0,19,2,"{""description"":""some"",""duration"":40}" +179,1672218353,1000.0,19,2,"{""description"":""some"",""duration"":40}" +180,1672222138,1000.0,19,2,"{""description"":""some"",""duration"":40}" +181,1672227880,1000.0,19,2,"{""description"":""some"",""duration"":40}" +182,1672227938,400.0,19,2,"{""description"":""some"",""duration"":40}" +183,1672234010,700.0,19,2,"{""description"":""some"",""duration"":40}" +184,1672303709,1000.0,19,2,"{""description"":""some"",""duration"":40}" +185,1672308144,800.0,19,2,"{""description"":""some"",""duration"":40}" +186,1672313359,700.0,19,2,"{""description"":""some"",""duration"":40}" +187,1672313377,200.0,19,2,"{""description"":""some"",""duration"":40}" +188,1672318111,1000.0,19,2,"{""description"":""some"",""duration"":40}" +189,1672318132,300.0,19,2,"{""description"":""some"",""duration"":40}" +190,1672320161,1000.0,19,2,"{""description"":""some"",""duration"":40}" +191,1672320172,200.0,19,2,"{""description"":""some"",""duration"":40}" +192,1672326596,700.0,19,2,"{""description"":""some"",""duration"":40}" +193,1672326607,1000.0,19,2,"{""description"":""some"",""duration"":40}" +194,1672326618,200.0,19,2,"{""description"":""some"",""duration"":40}" +195,1672389579,1000.0,19,2,"{""description"":""some"",""duration"":40}" +196,1672393666,1000.0,19,2,"{""description"":""some"",""duration"":40}" +197,1672400309,2000.0,19,2,"{""description"":""some"",""duration"":40}" +198,1672400322,200.0,19,2,"{""description"":""some"",""duration"":40}" +199,1672400406,500.0,19,2,"{""description"":""some"",""duration"":40}" +200,1672405512,1000.0,19,2,"{""description"":""some"",""duration"":40}" +201,1672426125,4500.0,19,2,"{""description"":""some"",""duration"":40}" +202,1673258846,400.0,19,2,"{""description"":""some"",""duration"":40}" +203,1673275425,1000.0,19,2,"{""description"":""some"",""duration"":40}" +204,1673344593,1000.0,19,2,"{""description"":""some"",""duration"":40}" +205,1673344606,1000.0,19,2,"{""description"":""some"",""duration"":40}" +206,1673347846,1000.0,19,2,"{""description"":""some"",""duration"":40}" +207,1673354520,2000.0,19,2,"{""description"":""some"",""duration"":40}" +208,1673602888,20000.0,19,2,"{""description"":""some"",""duration"":40}" +209,1673609998,10.0,19,2,"{""description"":""some"",""duration"":40}" +210,1673610036,1000.0,19,2,"{""description"":""some"",""duration"":40}" +211,1673619701,3500.0,19,2,"{""description"":""some"",""duration"":40}" +212,1673619714,1000.0,19,2,"{""description"":""some"",""duration"":40}" +213,1674038890,1500.0,19,2,"{""description"":""some"",""duration"":40}" +214,1674038898,1000.0,19,2,"{""description"":""some"",""duration"":40}" +215,1674038906,1000.0,19,2,"{""description"":""some"",""duration"":40}" +216,1674124770,1800.0,19,2,"{""description"":""some"",""duration"":40}" +217,1674155991,1000.0,19,2,"{""description"":""some"",""duration"":40}" +218,1674156006,600.0,19,2,"{""description"":""some"",""duration"":40}" +219,1674214709,1000.0,19,2,"{""description"":""some"",""duration"":40}" +220,1674214719,1000.0,19,2,"{""description"":""some"",""duration"":40}" +221,1674214736,500.0,19,2,"{""description"":""some"",""duration"":40}" +222,1674214750,2000.0,19,2,"{""description"":""some"",""duration"":40}" +223,1674379144,1000.0,19,2,"{""description"":""some"",""duration"":40}" +224,1674379158,300.0,19,2,"{""description"":""some"",""duration"":40}" +225,1674471516,2000.0,19,2,"{""description"":""some"",""duration"":40}" +226,1674478214,1000.0,19,2,"{""description"":""some"",""duration"":40}" +227,1674478231,300.0,19,2,"{""description"":""some"",""duration"":40}" +228,1674540208,3000.0,19,2,"{""description"":""some"",""duration"":40}" +229,1674724411,1000.0,19,2,"{""description"":""some"",""duration"":40}" +230,1674997317,700.0,19,2,"{""description"":""some"",""duration"":40}" +231,1674997326,500.0,19,2,"{""description"":""some"",""duration"":40}" +232,1674997343,350.0,19,2,"{""description"":""some"",""duration"":40}" +233,1675022124,1000.0,19,2,"{""description"":""some"",""duration"":40}" +234,1675083832,25000.0,19,2,"{""description"":""some"",""duration"":40}" +235,1675083844,3500.0,19,2,"{""description"":""some"",""duration"":40}" +236,1675083857,2000.0,19,2,"{""description"":""some"",""duration"":40}" +237,1675174989,700.0,19,2,"{""description"":""some"",""duration"":40}" +238,1675175000,1000.0,19,2,"{""description"":""some"",""duration"":40}" +239,1675175017,1800.0,19,2,"{""description"":""some"",""duration"":40}" +240,1675191645,1000.0,19,2,"{""description"":""some"",""duration"":40}" +241,1675345422,1800.0,19,2,"{""description"":""some"",""duration"":40}" +242,1675345429,700.0,19,2,"{""description"":""some"",""duration"":40}" +243,1675509724,900.0,19,2,"{""description"":""some"",""duration"":40}" +244,1675509732,1000.0,19,2,"{""description"":""some"",""duration"":40}" +245,1675510973,10.0,19,2,"{""description"":""some"",""duration"":40}" +246,1675670571,1000.0,19,2,"{""description"":""some"",""duration"":40}" +247,1675670582,1000.0,19,2,"{""description"":""some"",""duration"":40}" +248,1675681184,1000.0,19,2,"{""description"":""some"",""duration"":40}" +249,1675682854,700.0,19,2,"{""description"":""some"",""duration"":40}" +250,1675837827,800.0,19,2,"{""description"":""some"",""duration"":40}" +251,1675843115,1000.0,19,2,"{""description"":""some"",""duration"":40}" +252,1675876971,3000.0,19,2,"{""description"":""some"",""duration"":40}" +253,1675932830,300.0,19,2,"{""description"":""some"",""duration"":40}" +254,1675932839,200.0,19,2,"{""description"":""some"",""duration"":40}" +255,1675944721,1000.0,19,2,"{""description"":""some"",""duration"":40}" +256,1676008329,2000.0,19,2,"{""description"":""some"",""duration"":40}" +257,1676022215,3500.0,19,2,"{""description"":""some"",""duration"":40}" +258,1676034759,3500.0,19,2,"{""description"":""some"",""duration"":40}" +259,1676040413,1700.0,19,2,"{""description"":""some"",""duration"":40}" +260,1676302586,1000.0,19,2,"{""description"":""some"",""duration"":40}" +261,1676304103,3500.0,19,2,"{""description"":""some"",""duration"":40}" +262,1676315047,20000.0,19,2,"{""description"":""some"",""duration"":40}" +263,1676357146,1000.0,19,2,"{""description"":""some"",""duration"":40}" +264,1676372490,1000.0,19,2,"{""description"":""some"",""duration"":40}" +265,1676372490,1000.0,19,2,"{""description"":""some"",""duration"":40}" +266,1676372512,200.0,19,2,"{""description"":""some"",""duration"":40}" +267,1676392182,2200.0,19,2,"{""description"":""some"",""duration"":40}" +268,1676394067,500.0,19,2,"{""description"":""some"",""duration"":40}" +269,1676394084,700.0,19,2,"{""description"":""some"",""duration"":40}" +270,1676394099,300.0,19,2,"{""description"":""some"",""duration"":40}" +271,1676481893,5000.0,19,2,"{""description"":""some"",""duration"":40}" +272,1676714787,1000.0,19,2,"{""description"":""some"",""duration"":40}" +273,1676714802,1000.0,19,2,"{""description"":""some"",""duration"":40}" +274,1676714814,3000.0,19,2,"{""description"":""some"",""duration"":40}" +275,1676714822,1000.0,19,2,"{""description"":""some"",""duration"":40}" +276,1676714831,300.0,19,2,"{""description"":""some"",""duration"":40}" +277,1676809111,500.0,19,2,"{""description"":""some"",""duration"":40}" +278,1676809188,1000.0,19,2,"{""description"":""some"",""duration"":40}" +279,1676809204,300.0,19,2,"{""description"":""some"",""duration"":40}" +280,1676809216,1000.0,19,2,"{""description"":""some"",""duration"":40}" +281,1676809225,300.0,19,2,"{""description"":""some"",""duration"":40}" +282,1676906359,1000.0,19,2,"{""description"":""some"",""duration"":40}" +283,1676906377,500.0,19,2,"{""description"":""some"",""duration"":40}" +284,1676906385,1000.0,19,2,"{""description"":""some"",""duration"":40}" +285,1676906393,300.0,19,2,"{""description"":""some"",""duration"":40}" +286,1676906402,400.0,19,2,"{""description"":""some"",""duration"":40}" +287,1676906415,1800.0,19,2,"{""description"":""some"",""duration"":40}" +288,1676906440,3500.0,19,2,"{""description"":""some"",""duration"":40}" +289,1677261147,1000.0,19,2,"{""description"":""some"",""duration"":40}" +290,1677261159,1800.0,19,2,"{""description"":""some"",""duration"":40}" +291,1677493836,1000.0,19,2,"{""description"":""some"",""duration"":40}" +292,1677524903,3500.0,19,2,"{""description"":""some"",""duration"":40}" +293,1677603379,1000.0,19,2,"{""description"":""some"",""duration"":40}" +294,1677603390,1000.0,19,2,"{""description"":""some"",""duration"":40}" +295,1677909684,1000.0,19,2,"{""description"":""some"",""duration"":40}" +296,1678015228,700.0,19,2,"{""description"":""some"",""duration"":40}" +297,1678119822,3500.0,19,2,"{""description"":""some"",""duration"":40}" +298,1678120298,2000.0,19,2,"{""description"":""some"",""duration"":40}" +299,1678198568,1000.0,19,2,"{""description"":""some"",""duration"":40}" +300,1678198581,500.0,19,2,"{""description"":""some"",""duration"":40}" +301,1678478679,1000.0,19,2,"{""description"":""some"",""duration"":40}" +302,1678527822,1000.0,19,2,"{""description"":""some"",""duration"":40}" +303,1678527834,500.0,19,2,"{""description"":""some"",""duration"":40}" +304,1678532181,2500.0,19,2,"{""description"":""some"",""duration"":40}" +305,1678541402,1000.0,19,2,"{""description"":""some"",""duration"":40}" +306,1678543730,1000.0,19,2,"{""description"":""some"",""duration"":40}" +307,1678717604,3000.0,19,2,"{""description"":""some"",""duration"":40}" +308,1678776984,1000.0,19,2,"{""description"":""some"",""duration"":40}" +309,1678777040,20000.0,19,2,"{""description"":""some"",""duration"":40}" +310,1678787613,700.0,19,2,"{""description"":""some"",""duration"":40}" +311,1678787625,500.0,19,2,"{""description"":""some"",""duration"":40}" +312,1678805199,1000.0,19,2,"{""description"":""some"",""duration"":40}" +313,1678805212,1800.0,19,2,"{""description"":""some"",""duration"":40}" +314,1678805611,700.0,19,2,"{""description"":""some"",""duration"":40}" +315,1678805621,500.0,19,2,"{""description"":""some"",""duration"":40}" +316,1678963626,1000.0,19,2,"{""description"":""some"",""duration"":40}" +317,1678963633,300.0,19,2,"{""description"":""some"",""duration"":40}" +318,1678966411,1000.0,19,2,"{""description"":""some"",""duration"":40}" +319,1679046241,4000.0,19,2,"{""description"":""some"",""duration"":40}" +320,1679056941,1000.0,19,2,"{""description"":""some"",""duration"":40}" +321,1679061998,1000.0,19,2,"{""description"":""some"",""duration"":40}" +322,1679068223,1000.0,19,2,"{""description"":""some"",""duration"":40}" +323,1679153697,14000.0,19,2,"{""description"":""some"",""duration"":40}" +324,1679224295,3000.0,19,2,"{""description"":""some"",""duration"":40}" +325,1679224306,2000.0,19,2,"{""description"":""some"",""duration"":40}" +326,1679224369,1000.0,19,2,"{""description"":""some"",""duration"":40}" +327,1679224422,300.0,19,2,"{""description"":""some"",""duration"":40}" +328,1679224486,700.0,19,2,"{""description"":""some"",""duration"":40}" +329,1679229904,1000.0,19,2,"{""description"":""some"",""duration"":40}" +330,1679229927,300.0,19,2,"{""description"":""some"",""duration"":40}" +331,1679332859,1000.0,19,2,"{""description"":""some"",""duration"":40}" +332,1679332870,3500.0,19,2,"{""description"":""some"",""duration"":40}" +333,1679332903,3500.0,19,2,"{""description"":""some"",""duration"":40}" +334,1679470210,3000.0,19,2,"{""description"":""some"",""duration"":40}" +335,1679470218,1000.0,19,2,"{""description"":""some"",""duration"":40}" +336,1679482021,1000.0,19,2,"{""description"":""some"",""duration"":40}" +337,1679563931,1000.0,19,2,"{""description"":""some"",""duration"":40}" +338,1679571532,1000.0,19,2,"{""description"":""some"",""duration"":40}" +339,1679571545,500.0,19,2,"{""description"":""some"",""duration"":40}" +340,1679571552,1000.0,19,2,"{""description"":""some"",""duration"":40}" +341,1679584622,4000.0,19,2,"{""description"":""some"",""duration"":40}" +342,1679684724,1000.0,19,2,"{""description"":""some"",""duration"":40}" +343,1679941789,3500.0,19,2,"{""description"":""some"",""duration"":40}" +344,1680003956,1000.0,19,2,"{""description"":""some"",""duration"":40}" +345,1680272450,1000.0,19,2,"{""description"":""some"",""duration"":40}" +346,1680272461,1000.0,19,2,"{""description"":""some"",""duration"":40}" +347,1680272470,1000.0,19,2,"{""description"":""some"",""duration"":40}" +348,1680510632,1000.0,19,2,"{""description"":""some"",""duration"":40}" +349,1680510641,300.0,19,2,"{""description"":""some"",""duration"":40}" +350,1680510652,300.0,19,2,"{""description"":""some"",""duration"":40}" +351,1680526841,1000.0,19,2,"{""description"":""some"",""duration"":40}" +352,1680526853,700.0,19,2,"{""description"":""some"",""duration"":40}" +353,1680526865,300.0,19,2,"{""description"":""some"",""duration"":40}" +354,1680526882,3000.0,19,2,"{""description"":""some"",""duration"":40}" +355,1680547813,3500.0,19,2,"{""description"":""some"",""duration"":40}" +356,1680599086,500.0,19,2,"{""description"":""some"",""duration"":40}" +357,1680617464,1000.0,19,2,"{""description"":""some"",""duration"":40}" +358,1680617474,500.0,19,2,"{""description"":""some"",""duration"":40}" +359,1680617495,1000.0,19,2,"{""description"":""some"",""duration"":40}" +360,1680617508,700.0,19,2,"{""description"":""some"",""duration"":40}" +361,1680617519,2500.0,19,2,"{""description"":""some"",""duration"":40}" +362,1680780471,500.0,19,2,"{""description"":""some"",""duration"":40}" +363,1680780485,700.0,19,2,"{""description"":""some"",""duration"":40}" +364,1680780497,200.0,19,2,"{""description"":""some"",""duration"":40}" +365,1680780505,1000.0,19,2,"{""description"":""some"",""duration"":40}" +366,1680801021,800.0,19,2,"{""description"":""some"",""duration"":40}" +367,1680853871,1000.0,19,2,"{""description"":""some"",""duration"":40}" +368,1680869380,15000.0,19,2,"{""description"":""some"",""duration"":40}" +369,1680879812,500.0,19,2,"{""description"":""some"",""duration"":40}" +370,1680879822,1000.0,19,2,"{""description"":""some"",""duration"":40}" +371,1680948931,1000.0,19,2,"{""description"":""some"",""duration"":40}" +372,1680956966,1000.0,19,2,"{""description"":""some"",""duration"":40}" +373,1680957007,3500.0,19,2,"{""description"":""some"",""duration"":40}" +374,1681222689,3500.0,19,2,"{""description"":""some"",""duration"":40}" +375,1681301475,1000.0,19,2,"{""description"":""some"",""duration"":40}" +376,1681301483,1000.0,19,2,"{""description"":""some"",""duration"":40}" +377,1681301495,500.0,19,2,"{""description"":""some"",""duration"":40}" +378,1681314721,1000.0,19,2,"{""description"":""some"",""duration"":40}" +379,1681375795,900.0,19,2,"{""description"":""some"",""duration"":40}" +380,1681381055,1000.0,19,2,"{""description"":""some"",""duration"":40}" +381,1681381063,200.0,19,2,"{""description"":""some"",""duration"":40}" +382,1681381081,700.0,19,2,"{""description"":""some"",""duration"":40}" +383,1681473375,950.0,19,2,"{""description"":""some"",""duration"":40}" +384,1681540834,20000.0,19,2,"{""description"":""some"",""duration"":40}" +385,1681556018,1000.0,19,2,"{""description"":""some"",""duration"":40}" +386,1681556025,300.0,19,2,"{""description"":""some"",""duration"":40}" +387,1681556033,1000.0,19,2,"{""description"":""some"",""duration"":40}" +388,1681559058,300.0,19,2,"{""description"":""some"",""duration"":40}" +389,1681731983,1200.0,19,2,"{""description"":""some"",""duration"":40}" +390,1681731997,3500.0,19,2,"{""description"":""some"",""duration"":40}" +391,1681819763,1000.0,19,2,"{""description"":""some"",""duration"":40}" +392,1681819813,300.0,19,2,"{""description"":""some"",""duration"":40}" +393,1681826833,1800.0,19,2,"{""description"":""some"",""duration"":40}" +394,1681826847,500.0,19,2,"{""description"":""some"",""duration"":40}" +395,1681983885,700.0,19,2,"{""description"":""some"",""duration"":40}" +396,1681983895,100.0,19,2,"{""description"":""some"",""duration"":40}" +397,1681995794,1000.0,19,2,"{""description"":""some"",""duration"":40}" +398,1682099095,500.0,19,2,"{""description"":""some"",""duration"":40}" +399,1682156794,1000.0,19,2,"{""description"":""some"",""duration"":40}" +400,1682572414,700.0,19,2,"{""description"":""some"",""duration"":40}" +401,1682572431,4000.0,19,2,"{""description"":""some"",""duration"":40}" +402,1682590229,1000.0,19,2,"{""description"":""some"",""duration"":40}" +403,1682590238,1000.0,19,2,"{""description"":""some"",""duration"":40}" +404,1682741765,500.0,19,2,"{""description"":""some"",""duration"":40}" +405,1682748977,3000.0,19,2,"{""description"":""some"",""duration"":40}" +406,1682748989,1000.0,19,2,"{""description"":""some"",""duration"":40}" +407,1682764243,1000.0,19,2,"{""description"":""some"",""duration"":40}" +408,1682764252,1000.0,19,2,"{""description"":""some"",""duration"":40}" +409,1682764265,300.0,19,2,"{""description"":""some"",""duration"":40}" +410,1682771152,1800.0,19,2,"{""description"":""some"",""duration"":40}" +411,1682771163,700.0,19,2,"{""description"":""some"",""duration"":40}" +412,1682773862,1000.0,19,2,"{""description"":""some"",""duration"":40}" +413,1683059696,10.0,19,2,"{""description"":""some"",""duration"":40}" +414,1683107172,1000.0,19,2,"{""description"":""some"",""duration"":40}" +415,1683113552,3000.0,19,2,"{""description"":""some"",""duration"":40}" +416,1683113563,1000.0,19,2,"{""description"":""some"",""duration"":40}" +417,1683124134,1000.0,19,2,"{""description"":""some"",""duration"":40}" +418,1683269852,800.0,19,2,"{""description"":""some"",""duration"":40}" +419,1683271255,700.0,19,2,"{""description"":""some"",""duration"":40}" +420,1683543424,1000.0,19,2,"{""description"":""some"",""duration"":40}" +421,1683554211,1000.0,19,2,"{""description"":""some"",""duration"":40}" +422,1683554224,300.0,19,2,"{""description"":""some"",""duration"":40}" +423,1683554236,1000.0,19,2,"{""description"":""some"",""duration"":40}" +424,1683554246,300.0,19,2,"{""description"":""some"",""duration"":40}" +425,1683557465,1000.0,19,2,"{""description"":""some"",""duration"":40}" +426,1683557477,350.0,19,2,"{""description"":""some"",""duration"":40}" +427,1683798328,1000.0,19,2,"{""description"":""some"",""duration"":40}" +428,1683798341,300.0,19,2,"{""description"":""some"",""duration"":40}" +429,1683798351,1000.0,19,2,"{""description"":""some"",""duration"":40}" +430,1683809307,1000.0,19,2,"{""description"":""some"",""duration"":40}" +431,1683809323,500.0,19,2,"{""description"":""some"",""duration"":40}" +432,1683809336,1000.0,19,2,"{""description"":""some"",""duration"":40}" +433,1684063474,700.0,19,2,"{""description"":""some"",""duration"":40}" +434,1684063508,300.0,19,2,"{""description"":""some"",""duration"":40}" +435,1684063533,2500.0,19,2,"{""description"":""some"",""duration"":40}" +436,1684068697,1000.0,19,2,"{""description"":""some"",""duration"":40}" +437,1684068708,300.0,19,2,"{""description"":""some"",""duration"":40}" +438,1684068723,2000.0,19,2,"{""description"":""some"",""duration"":40}" +439,1684068929,300.0,19,2,"{""description"":""some"",""duration"":40}" +440,1684142246,20000.0,19,2,"{""description"":""some"",""duration"":40}" +441,1684159623,10000.0,19,2,"{""description"":""some"",""duration"":40}" +442,1684159658,700.0,19,2,"{""description"":""some"",""duration"":40}" +443,1684234591,1000.0,19,2,"{""description"":""some"",""duration"":40}" +444,1684255447,1000.0,19,2,"{""description"":""some"",""duration"":40}" +445,1684255459,1000.0,19,2,"{""description"":""some"",""duration"":40}" +446,1684416481,1000.0,19,2,"{""description"":""some"",""duration"":40}" +447,1684423281,3500.0,19,2,"{""description"":""some"",""duration"":40}" +448,1684427403,1000.0,19,2,"{""description"":""some"",""duration"":40}" +449,1684507089,1000.0,19,2,"{""description"":""some"",""duration"":40}" +450,1684510171,2250.0,19,2,"{""description"":""some"",""duration"":40}" +451,1684515734,1000.0,19,2,"{""description"":""some"",""duration"":40}" +452,1684515742,1000.0,19,2,"{""description"":""some"",""duration"":40}" +453,1684515753,300.0,19,2,"{""description"":""some"",""duration"":40}" +454,1684581869,1000.0,19,2,"{""description"":""some"",""duration"":40}" +455,1684581878,300.0,19,2,"{""description"":""some"",""duration"":40}" +456,1684659380,1000.0,19,2,"{""description"":""some"",""duration"":40}" +457,1684659387,1000.0,19,2,"{""description"":""some"",""duration"":40}" +458,1684659400,300.0,19,2,"{""description"":""some"",""duration"":40}" +459,1684659410,1500.0,19,2,"{""description"":""some"",""duration"":40}" +460,1684659418,3000.0,19,2,"{""description"":""some"",""duration"":40}" +461,1684662528,500.0,19,2,"{""description"":""some"",""duration"":40}" +462,1684745403,1000.0,19,2,"{""description"":""some"",""duration"":40}" +463,1684844953,1000.0,19,2,"{""description"":""some"",""duration"":40}" +464,1684919223,1000.0,19,2,"{""description"":""some"",""duration"":40}" +465,1684919232,700.0,19,2,"{""description"":""some"",""duration"":40}" +466,1685251726,1000.0,19,2,"{""description"":""some"",""duration"":40}" +467,1685251726,1800.0,19,2,"{""description"":""some"",""duration"":40}" +468,1685251726,700.0,19,2,"{""description"":""some"",""duration"":40}" +469,1685495439,1000.0,19,2,"{""description"":""some"",""duration"":40}" +470,1685495439,700.0,19,2,"{""description"":""some"",""duration"":40}" +471,1685495439,1000.0,19,2,"{""description"":""some"",""duration"":40}" +472,1685495439,300.0,19,2,"{""description"":""some"",""duration"":40}" +473,1685625474,1000.0,19,2,"{""description"":""some"",""duration"":40}" +474,1685625474,200.0,19,2,"{""description"":""some"",""duration"":40}" +475,1685625474,1000.0,19,2,"{""description"":""some"",""duration"":40}" +476,1685625474,300.0,19,2,"{""description"":""some"",""duration"":40}" +477,1685625474,1000.0,19,2,"{""description"":""some"",""duration"":40}" +478,1685625474,1000.0,19,2,"{""description"":""some"",""duration"":40}" +479,1685625474,700.0,19,2,"{""description"":""some"",""duration"":40}" +480,1685625474,1000.0,19,2,"{""description"":""some"",""duration"":40}" +481,1685625474,1800.0,19,2,"{""description"":""some"",""duration"":40}" +482,1685625474,700.0,19,2,"{""description"":""some"",""duration"":40}" +483,1685625474,1000.0,19,2,"{""description"":""some"",""duration"":40}" +484,1685625474,3500.0,19,2,"{""description"":""some"",""duration"":40}" +485,1685625474,3000.0,19,2,"{""description"":""some"",""duration"":40}" +486,1685625474,3000.0,19,2,"{""description"":""some"",""duration"":40}" +487,1685993343,4000.0,19,2,"{""description"":""some"",""duration"":40}" +488,1686060913,1000.0,19,2,"{""description"":""some"",""duration"":40}" +489,1686060926,440.0,19,2,"{""description"":""some"",""duration"":40}" +490,1686060950,3500.0,19,2,"{""description"":""some"",""duration"":40}" +491,1686072609,1000.0,19,2,"{""description"":""some"",""duration"":40}" +492,1686072620,500.0,19,2,"{""description"":""some"",""duration"":40}" +493,1686142826,1800.0,19,2,"{""description"":""some"",""duration"":40}" +494,1686142837,200.0,19,2,"{""description"":""some"",""duration"":40}" +495,1686142846,1000.0,19,2,"{""description"":""some"",""duration"":40}" +496,1686142855,1000.0,19,2,"{""description"":""some"",""duration"":40}" +497,1686231940,1000.0,19,2,"{""description"":""some"",""duration"":40}" +498,1686231949,1000.0,19,2,"{""description"":""some"",""duration"":40}" +499,1686231956,1000.0,19,2,"{""description"":""some"",""duration"":40}" +500,1686231974,0.0,19,2,"{""description"":""some"",""duration"":40}" +501,1686296670,1000.0,19,2,"{""description"":""some"",""duration"":40}" +502,1686316196,1000.0,19,2,"{""description"":""some"",""duration"":40}" +503,1686316218,200.0,19,2,"{""description"":""some"",""duration"":40}" +504,1686316226,1000.0,19,2,"{""description"":""some"",""duration"":40}" +505,1686316234,500.0,19,2,"{""description"":""some"",""duration"":40}" +506,1686842773,15000.0,19,2,"{""description"":""some"",""duration"":40}" +507,1686849138,20000.0,19,2,"{""description"":""some"",""duration"":40}" +508,1686905957,1000.0,19,2,"{""description"":""some"",""duration"":40}" +509,1686905987,500.0,19,2,"{""description"":""some"",""duration"":40}" +510,1686993942,1000.0,19,2,"{""description"":""some"",""duration"":40}" +511,1687510179,1000.0,19,2,"{""description"":""some"",""duration"":40}" +512,1687510190,500.0,19,2,"{""description"":""some"",""duration"":40}" +513,1687510198,1000.0,19,2,"{""description"":""some"",""duration"":40}" +514,1687528818,700.0,19,2,"{""description"":""some"",""duration"":40}" +515,1687528830,1000.0,19,2,"{""description"":""some"",""duration"":40}" +516,1687528844,1000.0,19,2,"{""description"":""some"",""duration"":40}" +517,1687531541,700.0,19,2,"{""description"":""some"",""duration"":40}" +518,1687607686,1500.0,19,2,"{""description"":""some"",""duration"":40}" +519,1687607697,1000.0,19,2,"{""description"":""some"",""duration"":40}" +520,1687607707,300.0,19,2,"{""description"":""some"",""duration"":40}" +521,1687622885,1000.0,19,2,"{""description"":""some"",""duration"":40}" +522,1687622896,300.0,19,2,"{""description"":""some"",""duration"":40}" +523,1687776852,3500.0,19,2,"{""description"":""some"",""duration"":40}" +524,1687776866,800.0,19,2,"{""description"":""some"",""duration"":40}" +525,1687776883,1500.0,19,2,"{""description"":""some"",""duration"":40}" +526,1687786827,3000.0,19,2,"{""description"":""some"",""duration"":40}" +527,1687786836,3000.0,19,2,"{""description"":""some"",""duration"":40}" +528,1687901328,1000.0,19,2,"{""description"":""some"",""duration"":40}" +529,1687939090,1000.0,19,2,"{""description"":""some"",""duration"":40}" +530,1687961907,1000.0,19,2,"{""description"":""some"",""duration"":40}" +531,1688055227,3500.0,19,2,"{""description"":""some"",""duration"":40}" +532,1688203080,1800.0,19,2,"{""description"":""some"",""duration"":40}" +533,1688203088,500.0,19,2,"{""description"":""some"",""duration"":40}" +534,1688212129,1000.0,19,2,"{""description"":""some"",""duration"":40}" +535,1688212138,500.0,19,2,"{""description"":""some"",""duration"":40}" +536,1688212669,300.0,19,2,"{""description"":""some"",""duration"":40}" +537,1688377745,1000.0,19,2,"{""description"":""some"",""duration"":40}" +538,1688377763,1000.0,19,2,"{""description"":""some"",""duration"":40}" +539,1688730464,1000.0,19,2,"{""description"":""some"",""duration"":40}" +540,1688730473,1800.0,19,2,"{""description"":""some"",""duration"":40}" +541,1688805023,4500.0,19,2,"{""description"":""some"",""duration"":40}" +542,1688808683,1000.0,19,2,"{""description"":""some"",""duration"":40}" +543,1689180206,1000.0,19,2,"{""description"":""some"",""duration"":40}" +544,1689237819,3000.0,19,2,"{""description"":""some"",""duration"":40}" +545,1689254432,1000.0,19,2,"{""description"":""some"",""duration"":40}" +546,1689254441,1000.0,19,2,"{""description"":""some"",""duration"":40}" +547,1689254450,1000.0,19,2,"{""description"":""some"",""duration"":40}" +548,1689325295,1500.0,19,2,"{""description"":""some"",""duration"":40}" +549,1689325307,1500.0,19,2,"{""description"":""some"",""duration"":40}" +550,1689590069,1100.0,19,2,"{""description"":""some"",""duration"":40}" +551,1689590089,500.0,19,2,"{""description"":""some"",""duration"":40}" +552,1689590097,1000.0,19,2,"{""description"":""some"",""duration"":40}" +553,1689681790,1000.0,19,2,"{""description"":""some"",""duration"":40}" +554,1689913856,1000.0,19,2,"{""description"":""some"",""duration"":40}" +555,1689913865,1000.0,19,2,"{""description"":""some"",""duration"":40}" +556,1689913877,2000.0,19,2,"{""description"":""some"",""duration"":40}" +557,1689948414,1000.0,19,2,"{""description"":""some"",""duration"":40}" +558,1689948424,500.0,19,2,"{""description"":""some"",""duration"":40}" +559,1689948440,1000.0,19,2,"{""description"":""some"",""duration"":40}" +560,1689948452,1300.0,19,2,"{""description"":""some"",""duration"":40}" +561,1689948469,1300.0,19,2,"{""description"":""some"",""duration"":40}" +562,1690029025,1300.0,19,2,"{""description"":""some"",""duration"":40}" +563,1690029038,1300.0,19,2,"{""description"":""some"",""duration"":40}" +564,1690029045,1000.0,19,2,"{""description"":""some"",""duration"":40}" +565,1690120357,3500.0,19,2,"{""description"":""some"",""duration"":40}" +566,1690120374,4500.0,19,2,"{""description"":""some"",""duration"":40}" +567,1690211540,1300.0,19,2,"{""description"":""some"",""duration"":40}" +568,1690271007,1000.0,19,2,"{""description"":""some"",""duration"":40}" +569,1690471407,700.0,19,2,"{""description"":""some"",""duration"":40}" +570,1690546782,2000.0,19,2,"{""description"":""some"",""duration"":40}" +571,1690618437,1800.0,19,2,"{""description"":""some"",""duration"":40}" +572,1691056244,1000.0,19,2,"{""description"":""some"",""duration"":40}" +573,1691061626,1000.0,19,2,"{""description"":""some"",""duration"":40}" +574,1691061644,500.0,19,2,"{""description"":""some"",""duration"":40}" +575,1691061658,800.0,19,2,"{""description"":""some"",""duration"":40}" +576,1691067178,1000.0,19,2,"{""description"":""some"",""duration"":40}" +577,1691145984,800.0,19,2,"{""description"":""some"",""duration"":40}" +578,1691155256,1000.0,19,2,"{""description"":""some"",""duration"":40}" +579,1691155268,1000.0,19,2,"{""description"":""some"",""duration"":40}" +580,1691155286,300.0,19,2,"{""description"":""some"",""duration"":40}" +581,1691228808,2000.0,19,2,"{""description"":""some"",""duration"":40}" +582,1691325700,2500.0,19,2,"{""description"":""some"",""duration"":40}" +583,1691423922,500.0,19,2,"{""description"":""some"",""duration"":40}" +584,1691568834,1000.0,19,2,"{""description"":""some"",""duration"":40}" +585,1691568841,1000.0,19,2,"{""description"":""some"",""duration"":40}" +586,1691568853,2000.0,19,2,"{""description"":""some"",""duration"":40}" +587,1691659838,1000.0,19,2,"{""description"":""some"",""duration"":40}" +588,1691742970,6000.0,19,2,"{""description"":""some"",""duration"":40}" +589,1691750551,1800.0,19,2,"{""description"":""some"",""duration"":40}" +590,1691750567,700.0,19,2,"{""description"":""some"",""duration"":40}" +591,1691836061,1500.0,19,2,"{""description"":""some"",""duration"":40}" +592,1691836133,1000.0,19,2,"{""description"":""some"",""duration"":40}" +593,1691836142,200.0,19,2,"{""description"":""some"",""duration"":40}" +594,1691838372,1000.0,19,2,"{""description"":""some"",""duration"":40}" +595,1691838383,500.0,19,2,"{""description"":""some"",""duration"":40}" +596,1691849545,1000.0,19,2,"{""description"":""some"",""duration"":40}" +597,1691849552,1000.0,19,2,"{""description"":""some"",""duration"":40}" +598,1691849570,500.0,19,2,"{""description"":""some"",""duration"":40}" +599,1691851915,1000.0,19,2,"{""description"":""some"",""duration"":40}" +600,1691856926,1000.0,19,2,"{""description"":""some"",""duration"":40}" +601,1691913765,1000.0,19,2,"{""description"":""some"",""duration"":40}" +602,1692017490,700.0,19,2,"{""description"":""some"",""duration"":40}" +603,1692108560,1000.0,19,2,"{""description"":""some"",""duration"":40}" +604,1692108571,2000.0,19,2,"{""description"":""some"",""duration"":40}" +605,1692178948,1000.0,19,2,"{""description"":""some"",""duration"":40}" +606,1692178964,500.0,19,2,"{""description"":""some"",""duration"":40}" +607,1692184222,3000.0,19,2,"{""description"":""some"",""duration"":40}" +608,1692277211,700.0,19,2,"{""description"":""some"",""duration"":40}" +609,1692277236,200.0,19,2,"{""description"":""some"",""duration"":40}" +610,1692277257,900.0,19,2,"{""description"":""some"",""duration"":40}" +611,1692277279,2000.0,19,2,"{""description"":""some"",""duration"":40}" +612,1692277294,600.0,19,2,"{""description"":""some"",""duration"":40}" +613,1692277318,2000.0,19,2,"{""description"":""some"",""duration"":40}" +614,1692280467,1000.0,19,2,"{""description"":""some"",""duration"":40}" +615,1692280484,300.0,19,2,"{""description"":""some"",""duration"":40}" +616,1692289428,1000.0,19,2,"{""description"":""some"",""duration"":40}" +617,1692441584,1000.0,19,2,"{""description"":""some"",""duration"":40}" +618,1692447214,2000.0,19,2,"{""description"":""some"",""duration"":40}" +619,1692447222,1000.0,19,2,"{""description"":""some"",""duration"":40}" +620,1692447238,500.0,19,2,"{""description"":""some"",""duration"":40}" +621,1692524397,1000.0,19,2,"{""description"":""some"",""duration"":40}" +622,1692531489,700.0,19,2,"{""description"":""some"",""duration"":40}" +623,1692531649,500.0,19,2,"{""description"":""some"",""duration"":40}" +624,1692531657,900.0,19,2,"{""description"":""some"",""duration"":40}" +625,1692603771,1000.0,19,2,"{""description"":""some"",""duration"":40}" +626,1692618584,600.0,19,2,"{""description"":""some"",""duration"":40}" +627,1692618591,500.0,19,2,"{""description"":""some"",""duration"":40}" +628,1692618599,700.0,19,2,"{""description"":""some"",""duration"":40}" +629,1692806265,2000.0,19,2,"{""description"":""some"",""duration"":40}" +630,1692866389,1000.0,19,2,"{""description"":""some"",""duration"":40}" +631,1692866397,1000.0,19,2,"{""description"":""some"",""duration"":40}" +632,1692866408,1000.0,19,2,"{""description"":""some"",""duration"":40}" +633,1692870155,1000.0,19,2,"{""description"":""some"",""duration"":40}" +634,1692870166,500.0,19,2,"{""description"":""some"",""duration"":40}" +635,1692967112,1000.0,19,2,"{""description"":""some"",""duration"":40}" +636,1692967127,300.0,19,2,"{""description"":""some"",""duration"":40}" +637,1692971743,1000.0,19,2,"{""description"":""some"",""duration"":40}" +638,1692971762,500.0,19,2,"{""description"":""some"",""duration"":40}" +639,1692971780,500.0,19,2,"{""description"":""some"",""duration"":40}" +640,1692971791,1000.0,19,2,"{""description"":""some"",""duration"":40}" +641,1692971826,500.0,19,2,"{""description"":""some"",""duration"":40}" +642,1692971834,300.0,19,2,"{""description"":""some"",""duration"":40}" +643,1692974608,1000.0,19,2,"{""description"":""some"",""duration"":40}" +644,1693036293,1000.0,19,2,"{""description"":""some"",""duration"":40}" +645,1693036306,500.0,19,2,"{""description"":""some"",""duration"":40}" +646,1693040042,1000.0,19,2,"{""description"":""some"",""duration"":40}" +647,1693043292,1000.0,19,2,"{""description"":""some"",""duration"":40}" +648,1693043300,500.0,19,2,"{""description"":""some"",""duration"":40}" +649,1693226064,2000.0,19,2,"{""description"":""some"",""duration"":40}" +650,1693226076,700.0,19,2,"{""description"":""some"",""duration"":40}" +651,1693226086,300.0,19,2,"{""description"":""some"",""duration"":40}" +652,1693226117,2000.0,19,2,"{""description"":""some"",""duration"":40}" +653,1693303885,1000.0,19,2,"{""description"":""some"",""duration"":40}" +654,1693303893,1000.0,19,2,"{""description"":""some"",""duration"":40}" +655,1693385143,1000.0,19,2,"{""description"":""some"",""duration"":40}" +656,1693385155,300.0,19,2,"{""description"":""some"",""duration"":40}" +657,1693385176,500.0,19,2,"{""description"":""some"",""duration"":40}" +658,1693483053,600.0,19,2,"{""description"":""some"",""duration"":40}" +659,1693483065,700.0,19,2,"{""description"":""some"",""duration"":40}" +660,1693483081,500.0,19,2,"{""description"":""some"",""duration"":40}" +661,1693572936,1000.0,19,2,"{""description"":""some"",""duration"":40}" +662,1693578938,1000.0,19,2,"{""description"":""some"",""duration"":40}" +663,1693641451,1000.0,19,2,"{""description"":""some"",""duration"":40}" +664,1693641462,300.0,19,2,"{""description"":""some"",""duration"":40}" +665,1693657446,2000.0,19,2,"{""description"":""some"",""duration"":40}" +666,1693657467,2000.0,19,2,"{""description"":""some"",""duration"":40}" +667,1693850745,1000.0,19,2,"{""description"":""some"",""duration"":40}" +668,1693850758,1500.0,19,2,"{""description"":""some"",""duration"":40}" +669,1693985700,500.0,19,2,"{""description"":""some"",""duration"":40}" +670,1694018421,500.0,19,2,"{""description"":""some"",""duration"":40}" +671,1694087017,1000.0,19,2,"{""description"":""some"",""duration"":40}" +672,1694087028,500.0,19,2,"{""description"":""some"",""duration"":40}" +673,1694180430,1000.0,19,2,"{""description"":""some"",""duration"":40}" +674,1694260565,1000.0,19,2,"{""description"":""some"",""duration"":40}" +675,1694260574,500.0,19,2,"{""description"":""some"",""duration"":40}" +676,1694268371,1000.0,19,2,"{""description"":""some"",""duration"":40}" +677,1694425420,1000.0,19,2,"{""description"":""some"",""duration"":40}" +678,1694425434,2000.0,19,2,"{""description"":""some"",""duration"":40}" +679,1694437840,1000.0,19,2,"{""description"":""some"",""duration"":40}" +680,1694595492,1500.0,19,2,"{""description"":""some"",""duration"":40}" +681,1694607612,1000.0,19,2,"{""description"":""some"",""duration"":40}" +682,1694611951,1000.0,19,2,"{""description"":""some"",""duration"":40}" +683,1694689945,3500.0,19,2,"{""description"":""some"",""duration"":40}" +684,1695030060,1000.0,19,2,"{""description"":""some"",""duration"":40}" +685,1695044951,2000.0,19,2,"{""description"":""some"",""duration"":40}" +686,1695115942,1000.0,19,2,"{""description"":""some"",""duration"":40}" +687,1695126728,2000.0,19,2,"{""description"":""some"",""duration"":40}" +688,1695131546,1000.0,19,2,"{""description"":""some"",""duration"":40}" +689,1695131557,500.0,19,2,"{""description"":""some"",""duration"":40}" +690,1695131566,1000.0,19,2,"{""description"":""some"",""duration"":40}" +691,1695133639,1000.0,19,2,"{""description"":""some"",""duration"":40}" +692,1695204170,1000.0,19,2,"{""description"":""some"",""duration"":40}" +693,1695208128,1000.0,19,2,"{""description"":""some"",""duration"":40}" +694,1695208140,1000.0,19,2,"{""description"":""some"",""duration"":40}" +695,1695208152,300.0,19,2,"{""description"":""some"",""duration"":40}" +696,1695213265,1000.0,19,2,"{""description"":""some"",""duration"":40}" +697,1695213275,300.0,19,2,"{""description"":""some"",""duration"":40}" +698,1695226073,900.0,19,2,"{""description"":""some"",""duration"":40}" +699,1695369878,3000.0,19,2,"{""description"":""some"",""duration"":40}" +700,1695447410,1000.0,19,2,"{""description"":""some"",""duration"":40}" +701,1695447420,300.0,19,2,"{""description"":""some"",""duration"":40}" +702,1695462527,1000.0,19,2,"{""description"":""some"",""duration"":40}" +703,1695542006,4500.0,19,2,"{""description"":""some"",""duration"":40}" +704,1695724915,1000.0,19,2,"{""description"":""some"",""duration"":40}" +705,1695724929,300.0,19,2,"{""description"":""some"",""duration"":40}" +706,1695724960,1000.0,19,2,"{""description"":""some"",""duration"":40}" +707,1695724970,1000.0,19,2,"{""description"":""some"",""duration"":40}" +708,1695812407,1000.0,19,2,"{""description"":""some"",""duration"":40}" +709,1695977012,2000.0,19,2,"{""description"":""some"",""duration"":40}" +710,1695977084,1000.0,19,2,"{""description"":""some"",""duration"":40}" +711,1695977099,500.0,19,2,"{""description"":""some"",""duration"":40}" +712,1695978463,700.0,19,2,"{""description"":""some"",""duration"":40}" +713,1695989760,3000.0,19,2,"{""description"":""some"",""duration"":40}" +714,1695994538,2000.0,19,2,"{""description"":""some"",""duration"":40}" +715,1696064713,1000.0,19,2,"{""description"":""some"",""duration"":40}" +716,1696064725,700.0,19,2,"{""description"":""some"",""duration"":40}" +717,1696064734,500.0,19,2,"{""description"":""some"",""duration"":40}" +718,1696237522,2000.0,19,2,"{""description"":""some"",""duration"":40}" +719,1696237537,700.0,19,2,"{""description"":""some"",""duration"":40}" +720,1696409137,1000.0,19,2,"{""description"":""some"",""duration"":40}" +721,1696413405,1000.0,19,2,"{""description"":""some"",""duration"":40}" +722,1696413414,300.0,19,2,"{""description"":""some"",""duration"":40}" +723,1696419260,700.0,19,2,"{""description"":""some"",""duration"":40}" +724,1696419270,500.0,19,2,"{""description"":""some"",""duration"":40}" +725,1696498224,3500.0,19,2,"{""description"":""some"",""duration"":40}" +726,1696512272,600.0,19,2,"{""description"":""some"",""duration"":40}" +727,1696512306,600.0,19,2,"{""description"":""some"",""duration"":40}" +728,1696512314,1000.0,19,2,"{""description"":""some"",""duration"":40}" +729,1696584198,2000.0,19,2,"{""description"":""some"",""duration"":40}" +730,1696602696,2000.0,19,2,"{""description"":""some"",""duration"":40}" +731,1696763281,3000.0,19,2,"{""description"":""some"",""duration"":40}" +732,1696842215,1000.0,19,2,"{""description"":""some"",""duration"":40}" +733,1696842227,500.0,19,2,"{""description"":""some"",""duration"":40}" +734,1696863818,1000.0,19,2,"{""description"":""some"",""duration"":40}" +735,1696951368,2000.0,19,2,"{""description"":""some"",""duration"":40}" +736,1697016706,1000.0,19,2,"{""description"":""some"",""duration"":40}" +737,1697026949,2000.0,19,2,"{""description"":""some"",""duration"":40}" +738,1697193136,800.0,19,2,"{""description"":""some"",""duration"":40}" +739,1697193145,1000.0,19,2,"{""description"":""some"",""duration"":40}" +740,1697193156,500.0,19,2,"{""description"":""some"",""duration"":40}" +741,1697200701,2000.0,19,2,"{""description"":""some"",""duration"":40}" +742,1697200711,1000.0,19,2,"{""description"":""some"",""duration"":40}" +743,1697204569,1000.0,19,2,"{""description"":""some"",""duration"":40}" +744,1697446745,2000.0,19,2,"{""description"":""some"",""duration"":40}" +745,1697446756,300.0,19,2,"{""description"":""some"",""duration"":40}" +746,1697446765,1000.0,19,2,"{""description"":""some"",""duration"":40}" +747,1697618988,1000.0,19,2,"{""description"":""some"",""duration"":40}" +748,1697631857,1000.0,19,2,"{""description"":""some"",""duration"":40}" +749,1697708072,1000.0,19,2,"{""description"":""some"",""duration"":40}" +750,1697708180,1000.0,19,2,"{""description"":""some"",""duration"":40}" +751,1697708189,1300.0,19,2,"{""description"":""some"",""duration"":40}" +752,1697799272,20000.0,19,2,"{""description"":""some"",""duration"":40}" +753,1697799287,1000.0,19,2,"{""description"":""some"",""duration"":40}" +754,1698042003,2000.0,19,2,"{""description"":""some"",""duration"":40}" +755,1698042014,1000.0,19,2,"{""description"":""some"",""duration"":40}" +756,1698064325,1000.0,19,2,"{""description"":""some"",""duration"":40}" +757,1698069302,2000.0,19,2,"{""description"":""some"",""duration"":40}" +758,1698073704,1000.0,19,2,"{""description"":""some"",""duration"":40}" +759,1698073712,1000.0,19,2,"{""description"":""some"",""duration"":40}" +760,1698132322,3000.0,19,2,"{""description"":""some"",""duration"":40}" +761,1698132335,1000.0,19,2,"{""description"":""some"",""duration"":40}" +762,1698132352,500.0,19,2,"{""description"":""some"",""duration"":40}" +763,1698232445,1000.0,19,2,"{""description"":""some"",""duration"":40}" +764,1698232455,1000.0,19,2,"{""description"":""some"",""duration"":40}" +765,1698232465,500.0,19,2,"{""description"":""some"",""duration"":40}" +766,1698233758,700.0,19,2,"{""description"":""some"",""duration"":40}" +767,1698493234,1000.0,19,2,"{""description"":""some"",""duration"":40}" +768,1698493243,2000.0,19,2,"{""description"":""some"",""duration"":40}" +769,1698497541,1000.0,19,2,"{""description"":""some"",""duration"":40}" +770,1698501209,1000.0,19,2,"{""description"":""some"",""duration"":40}" +771,1698572939,5000.0,19,2,"{""description"":""some"",""duration"":40}" +772,1698831768,1000.0,19,2,"{""description"":""some"",""duration"":40}" +773,1698864277,2000.0,19,2,"{""description"":""some"",""duration"":40}" +774,1699270941,1000.0,19,2,"{""description"":""some"",""duration"":40}" +775,1699270965,2000.0,19,2,"{""description"":""some"",""duration"":40}" +776,1699270982,300.0,19,2,"{""description"":""some"",""duration"":40}" +777,1699270993,1000.0,19,2,"{""description"":""some"",""duration"":40}" +778,1699271062,700.0,19,2,"{""description"":""some"",""duration"":40}" +779,1699345725,1000.0,19,2,"{""description"":""some"",""duration"":40}" +780,1699364573,1000.0,19,2,"{""description"":""some"",""duration"":40}" +781,1699436162,1000.0,19,2,"{""description"":""some"",""duration"":40}" +782,1699444375,1000.0,19,2,"{""description"":""some"",""duration"":40}" +783,1699444389,500.0,19,2,"{""description"":""some"",""duration"":40}" +784,1699642609,2000.0,19,2,"{""description"":""some"",""duration"":40}" +785,1699642619,1000.0,19,2,"{""description"":""some"",""duration"":40}" +786,1700038094,1000.0,19,2,"{""description"":""some"",""duration"":40}" +787,1700038107,2000.0,19,2,"{""description"":""some"",""duration"":40}" +788,1700042242,1000.0,19,2,"{""description"":""some"",""duration"":40}" +789,1700042256,1000.0,19,2,"{""description"":""some"",""duration"":40}" +790,1700042267,1000.0,19,2,"{""description"":""some"",""duration"":40}" +791,1700056633,1000.0,19,2,"{""description"":""some"",""duration"":40}" +792,1700056654,300.0,19,2,"{""description"":""some"",""duration"":40}" +793,1700056664,1000.0,19,2,"{""description"":""some"",""duration"":40}" +794,1700205890,700.0,19,2,"{""description"":""some"",""duration"":40}" +795,1700229914,3500.0,19,2,"{""description"":""some"",""duration"":40}" +796,1700565540,20000.0,19,2,"{""description"":""some"",""duration"":40}" +797,1700565558,1000.0,19,2,"{""description"":""some"",""duration"":40}" +798,1700656546,5000.0,19,2,"{""description"":""some"",""duration"":40}" +799,1700736068,1500.0,19,2,"{""description"":""some"",""duration"":40}" +800,1700739329,1500.0,19,2,"{""description"":""some"",""duration"":40}" +801,1700739341,300.0,19,2,"{""description"":""some"",""duration"":40}" +802,1700902222,1000.0,19,2,"{""description"":""some"",""duration"":40}" +803,1700902246,1000.0,19,2,"{""description"":""some"",""duration"":40}" +804,1700902259,500.0,19,2,"{""description"":""some"",""duration"":40}" +805,1700902272,4000.0,19,2,"{""description"":""some"",""duration"":40}" +806,1700926026,1500.0,19,2,"{""description"":""some"",""duration"":40}" +807,1701097123,1000.0,19,2,"{""description"":""some"",""duration"":40}" +808,1701097139,2400.0,19,2,"{""description"":""some"",""duration"":40}" +809,1701097153,800.0,19,2,"{""description"":""some"",""duration"":40}" +810,1701245408,1000.0,19,2,"{""description"":""some"",""duration"":40}" +811,1701245421,2000.0,19,2,"{""description"":""some"",""duration"":40}" +812,1701251764,1000.0,19,2,"{""description"":""some"",""duration"":40}" +813,1701265932,7500.0,19,2,"{""description"":""some"",""duration"":40}" +814,1701418989,1000.0,19,2,"{""description"":""some"",""duration"":40}" +815,1701419000,300.0,19,2,"{""description"":""some"",""duration"":40}" +816,1701419014,1000.0,19,2,"{""description"":""some"",""duration"":40}" +817,1701419035,600.0,19,2,"{""description"":""some"",""duration"":40}" +818,1701419048,400.0,19,2,"{""description"":""some"",""duration"":40}" +819,1701419068,1000.0,19,2,"{""description"":""some"",""duration"":40}" +820,1701703329,2000.0,19,2,"{""description"":""some"",""duration"":40}" +821,1701703340,500.0,19,2,"{""description"":""some"",""duration"":40}" +822,1701770749,1500.0,19,2,"{""description"":""some"",""duration"":40}" +823,1701859469,1000.0,19,2,"{""description"":""some"",""duration"":40}" +824,1701947352,2600.0,19,2,"{""description"":""some"",""duration"":40}" +825,1701947366,1500.0,19,2,"{""description"":""some"",""duration"":40}" +826,1701950073,1500.0,19,2,"{""description"":""some"",""duration"":40}" +827,1701953358,1500.0,19,2,"{""description"":""some"",""duration"":40}" +828,1701953673,500.0,19,2,"{""description"":""some"",""duration"":40}" +829,1702024426,1000.0,19,2,"{""description"":""some"",""duration"":40}" +830,1702024439,2000.0,19,2,"{""description"":""some"",""duration"":40}" +831,1702026136,3500.0,19,2,"{""description"":""some"",""duration"":40}" +832,1702298711,1000.0,19,2,"{""description"":""some"",""duration"":40}" +833,1702298734,1000.0,19,2,"{""description"":""some"",""duration"":40}" +834,1702298739,300.0,19,2,"{""description"":""some"",""duration"":40}" +835,1702298744,400.0,19,2,"{""description"":""some"",""duration"":40}" +836,1702453750,6000.0,19,2,"{""description"":""some"",""duration"":40}" +837,1702466139,2000.0,19,2,"{""description"":""some"",""duration"":40}" +838,1702466153,2000.0,19,2,"{""description"":""some"",""duration"":40}" +839,1702636904,2000.0,19,2,"{""description"":""some"",""duration"":40}" +840,1702636912,1000.0,19,2,"{""description"":""some"",""duration"":40}" +841,1702636920,1000.0,19,2,"{""description"":""some"",""duration"":40}" +842,1702639560,1000.0,19,2,"{""description"":""some"",""duration"":40}" +843,1702888560,800.0,19,2,"{""description"":""some"",""duration"":40}" +844,1702888571,2000.0,19,2,"{""description"":""some"",""duration"":40}" +845,1702888578,1000.0,19,2,"{""description"":""some"",""duration"":40}" +846,1702982449,1500.0,19,2,"{""description"":""some"",""duration"":40}" +847,1702982460,500.0,19,2,"{""description"":""some"",""duration"":40}" +848,1702982476,1500.0,19,2,"{""description"":""some"",""duration"":40}" +849,1702982489,500.0,19,2,"{""description"":""some"",""duration"":40}" +850,1703071032,700.0,19,2,"{""description"":""some"",""duration"":40}" +851,1703071040,1000.0,19,2,"{""description"":""some"",""duration"":40}" +852,1703071050,1000.0,19,2,"{""description"":""some"",""duration"":40}" +853,1703071058,700.0,19,2,"{""description"":""some"",""duration"":40}" +854,1703071071,2450.0,19,2,"{""description"":""some"",""duration"":40}" +855,1703071087,1000.0,19,2,"{""description"":""some"",""duration"":40}" +856,1703071094,300.0,19,2,"{""description"":""some"",""duration"":40}" +857,1703071104,1000.0,19,2,"{""description"":""some"",""duration"":40}" +858,1703083455,1000.0,19,2,"{""description"":""some"",""duration"":40}" +859,1703083467,2500.0,19,2,"{""description"":""some"",""duration"":40}" +860,1703083475,1000.0,19,2,"{""description"":""some"",""duration"":40}" +861,1703083500,1000.0,19,2,"{""description"":""some"",""duration"":40}" +862,1703083511,500.0,19,2,"{""description"":""some"",""duration"":40}" +863,1703228194,2000.0,19,2,"{""description"":""some"",""duration"":40}" +864,1703228200,1000.0,19,2,"{""description"":""some"",""duration"":40}" +865,1703228220,200.0,19,2,"{""description"":""some"",""duration"":40}" +866,1703232931,1000.0,19,2,"{""description"":""some"",""duration"":40}" +867,1703232948,500.0,19,2,"{""description"":""some"",""duration"":40}" +868,1703241826,3500.0,19,2,"{""description"":""some"",""duration"":40}" +869,1703254522,1000.0,19,2,"{""description"":""some"",""duration"":40}" +870,1703254531,700.0,19,2,"{""description"":""some"",""duration"":40}" +871,1703254540,1200.0,19,2,"{""description"":""some"",""duration"":40}" +872,1703254551,2000.0,19,2,"{""description"":""some"",""duration"":40}" +873,1703254579,1000.0,19,2,"{""description"":""some"",""duration"":40}" +874,1703254590,500.0,19,2,"{""description"":""some"",""duration"":40}" +875,1703260577,900.0,19,2,"{""description"":""some"",""duration"":40}" +876,1703260587,1000.0,19,2,"{""description"":""some"",""duration"":40}" +877,1703260647,1000.0,19,2,"{""description"":""some"",""duration"":40}" +878,1703261966,200.0,19,2,"{""description"":""some"",""duration"":40}" +879,1703261984,2000.0,19,2,"{""description"":""some"",""duration"":40}" +880,1703504838,2000.0,19,2,"{""description"":""some"",""duration"":40}" +881,1703504841,1000.0,19,2,"{""description"":""some"",""duration"":40}" +882,1703504845,500.0,19,2,"{""description"":""some"",""duration"":40}" +883,1703507231,1000.0,19,2,"{""description"":""some"",""duration"":40}" +884,1703507239,1000.0,19,2,"{""description"":""some"",""duration"":40}" +885,1703511169,2000.0,19,2,"{""description"":""some"",""duration"":40}" +886,1703516553,1000.0,19,2,"{""description"":""some"",""duration"":40}" +887,1703516560,1000.0,19,2,"{""description"":""some"",""duration"":40}" +888,1703586431,2600.0,19,2,"{""description"":""some"",""duration"":40}" +889,1703586441,1500.0,19,2,"{""description"":""some"",""duration"":40}" +890,1703589397,1500.0,19,2,"{""description"":""some"",""duration"":40}" +891,1703589405,300.0,19,2,"{""description"":""some"",""duration"":40}" +892,1703669545,1000.0,19,2,"{""description"":""some"",""duration"":40}" +893,1703669555,500.0,19,2,"{""description"":""some"",""duration"":40}" +894,1703669564,1000.0,19,2,"{""description"":""some"",""duration"":40}" +895,1703669578,700.0,19,2,"{""description"":""some"",""duration"":40}" +896,1703669586,1000.0,19,2,"{""description"":""some"",""duration"":40}" +897,1703675009,2500.0,19,2,"{""description"":""some"",""duration"":40}" +898,1703680550,1000.0,19,2,"{""description"":""some"",""duration"":40}" +899,1703680561,500.0,19,2,"{""description"":""some"",""duration"":40}" +900,1703690736,1000.0,19,2,"{""description"":""some"",""duration"":40}" +901,1703690757,400.0,19,2,"{""description"":""some"",""duration"":40}" +902,1703690851,800.0,19,2,"{""description"":""some"",""duration"":40}" +903,1703690862,1000.0,19,2,"{""description"":""some"",""duration"":40}" +904,1703690880,500.0,19,2,"{""description"":""some"",""duration"":40}" +905,1703753876,2600.0,19,2,"{""description"":""some"",""duration"":40}" +906,1703753921,2400.0,19,2,"{""description"":""some"",""duration"":40}" +907,1703846567,2000.0,19,2,"{""description"":""some"",""duration"":40}" +908,1703846578,1000.0,19,2,"{""description"":""some"",""duration"":40}" +909,1703851354,2300.0,19,2,"{""description"":""some"",""duration"":40}" +910,1703851372,1000.0,19,2,"{""description"":""some"",""duration"":40}" +911,1703851386,1000.0,19,2,"{""description"":""some"",""duration"":40}" +912,1703851405,1000.0,19,2,"{""description"":""some"",""duration"":40}" +913,1703855103,1000.0,19,2,"{""description"":""some"",""duration"":40}" +914,1703855130,600.0,19,2,"{""description"":""some"",""duration"":40}" +915,1703858287,1000.0,19,2,"{""description"":""some"",""duration"":40}" +916,1703858297,300.0,19,2,"{""description"":""some"",""duration"":40}" +917,1704475114,3500.0,19,2,"{""description"":""some"",""duration"":40}" +918,1704946608,1000.0,19,2,"{""description"":""some"",""duration"":40}" +919,1704946615,1000.0,19,2,"{""description"":""some"",""duration"":40}" +920,1704946637,1000.0,19,2,"{""description"":""some"",""duration"":40}" +921,1705308258,2000.0,19,2,"{""description"":""some"",""duration"":40}" +922,1705308266,1000.0,19,2,"{""description"":""some"",""duration"":40}" +923,1705308277,500.0,19,2,"{""description"":""some"",""duration"":40}" +924,1705308284,1200.0,19,2,"{""description"":""some"",""duration"":40}" +925,1705340107,1200.0,19,2,"{""description"":""some"",""duration"":40}" +926,1705340119,300.0,19,2,"{""description"":""some"",""duration"":40}" +927,1705498948,3500.0,19,2,"{""description"":""some"",""duration"":40}" +928,1705498961,1000.0,19,2,"{""description"":""some"",""duration"":40}" +929,1705498970,300.0,19,2,"{""description"":""some"",""duration"":40}" +930,1705498977,700.0,19,2,"{""description"":""some"",""duration"":40}" +931,1705498986,1000.0,19,2,"{""description"":""some"",""duration"":40}" +932,1705499003,500.0,19,2,"{""description"":""some"",""duration"":40}" +933,1705521011,2400.0,19,2,"{""description"":""some"",""duration"":40}" +934,1705659907,1000.0,19,2,"{""description"":""some"",""duration"":40}" +935,1705672044,1000.0,19,2,"{""description"":""some"",""duration"":40}" +936,1705672058,1000.0,19,2,"{""description"":""some"",""duration"":40}" +937,1705685251,1000.0,19,2,"{""description"":""some"",""duration"":40}" +938,1706009980,2600.0,19,2,"{""description"":""some"",""duration"":40}" +939,1706009992,1500.0,19,2,"{""description"":""some"",""duration"":40}" +940,1706010004,1500.0,19,2,"{""description"":""some"",""duration"":40}" +941,1706010025,300.0,19,2,"{""description"":""some"",""duration"":40}" +942,1706091955,600.0,19,2,"{""description"":""some"",""duration"":40}" +943,1706091964,500.0,19,2,"{""description"":""some"",""duration"":40}" +944,1706091978,1000.0,19,2,"{""description"":""some"",""duration"":40}" +945,1706091991,500.0,19,2,"{""description"":""some"",""duration"":40}" +946,1706122818,700.0,19,2,"{""description"":""some"",""duration"":40}" +947,1706122826,700.0,19,2,"{""description"":""some"",""duration"":40}" +948,1706122840,1000.0,19,2,"{""description"":""some"",""duration"":40}" +949,1706122864,300.0,19,2,"{""description"":""some"",""duration"":40}" +950,1706122884,2600.0,19,2,"{""description"":""some"",""duration"":40}" +951,1706122904,2000.0,19,2,"{""description"":""some"",""duration"":40}" +952,1706122915,500.0,19,2,"{""description"":""some"",""duration"":40}" +953,1706122926,800.0,19,2,"{""description"":""some"",""duration"":40}" +954,1706288328,1000.0,19,2,"{""description"":""some"",""duration"":40}" +955,1706288335,500.0,19,2,"{""description"":""some"",""duration"":40}" +956,1706288342,1000.0,19,2,"{""description"":""some"",""duration"":40}" +957,1706288349,300.0,19,2,"{""description"":""some"",""duration"":40}" +958,1706288358,1500.0,19,2,"{""description"":""some"",""duration"":40}" +959,1706288367,1000.0,19,2,"{""description"":""some"",""duration"":40}" +960,1706288375,800.0,19,2,"{""description"":""some"",""duration"":40}" +961,1706288389,4000.0,19,2,"{""description"":""some"",""duration"":40}" +962,1706688568,2400.0,19,2,"{""description"":""some"",""duration"":40}" +963,1706688585,1000.0,19,2,"{""description"":""some"",""duration"":40}" +964,1706688593,4000.0,19,2,"{""description"":""some"",""duration"":40}" +965,1706693372,2000.0,19,2,"{""description"":""some"",""duration"":40}" +966,1706695183,1000.0,19,2,"{""description"":""some"",""duration"":40}" +967,1706696643,1300.0,19,2,"{""description"":""some"",""duration"":40}" +968,1706699548,1300.0,19,2,"{""description"":""some"",""duration"":40}" +969,1706701507,700.0,19,2,"{""description"":""some"",""duration"":40}" +970,1706705449,1500.0,19,2,"{""description"":""some"",""duration"":40}" +971,1706717659,1000.0,19,2,"{""description"":""some"",""duration"":40}" +972,1706870733,1300.0,19,2,"{""description"":""some"",""duration"":40}" +973,1706870744,1300.0,19,2,"{""description"":""some"",""duration"":40}" +974,1706870753,2400.0,19,2,"{""description"":""some"",""duration"":40}" +975,1706870762,1100.0,19,2,"{""description"":""some"",""duration"":40}" +976,1706878834,2000.0,19,2,"{""description"":""some"",""duration"":40}" +977,1708108933,1000.0,19,2,"{""description"":""some"",""duration"":40}" +978,1708108941,2000.0,19,2,"{""description"":""some"",""duration"":40}" +979,1708108945,1000.0,19,2,"{""description"":""some"",""duration"":40}" +980,1708108949,500.0,19,2,"{""description"":""some"",""duration"":40}" +981,1708108953,2400.0,19,2,"{""description"":""some"",""duration"":40}" +982,1708108956,1300.0,19,2,"{""description"":""some"",""duration"":40}" +983,1708108960,1000.0,19,2,"{""description"":""some"",""duration"":40}" +984,1708108963,1300.0,19,2,"{""description"":""some"",""duration"":40}" +985,1708108966,1000.0,19,2,"{""description"":""some"",""duration"":40}" +986,1708348230,1300.0,19,2,"{""description"":""some"",""duration"":40}" +987,1708348240,900.0,19,2,"{""description"":""some"",""duration"":40}" +988,1708348254,900.0,19,2,"{""description"":""some"",""duration"":40}" +989,1708348264,1300.0,19,2,"{""description"":""some"",""duration"":40}" +990,1708429295,1000.0,19,2,"{""description"":""some"",""duration"":40}" +991,1708429306,1300.0,19,2,"{""description"":""some"",""duration"":40}" +992,1708429318,500.0,19,2,"{""description"":""some"",""duration"":40}" +993,1708429327,1500.0,19,2,"{""description"":""some"",""duration"":40}" +994,1708429337,300.0,19,2,"{""description"":""some"",""duration"":40}" +995,1708438299,1500.0,19,2,"{""description"":""some"",""duration"":40}" +996,1708524076,1300.0,19,2,"{""description"":""some"",""duration"":40}" +997,1708524086,900.0,19,2,"{""description"":""some"",""duration"":40}" +998,1708524105,1300.0,19,2,"{""description"":""some"",""duration"":40}" +999,1708524114,300.0,19,2,"{""description"":""some"",""duration"":40}" +1000,1708524124,900.0,19,2,"{""description"":""some"",""duration"":40}" +1001,1708524133,300.0,19,2,"{""description"":""some"",""duration"":40}" +1002,1708527667,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1003,1708937921,1000.0,19,2,"{""description"":""some"",""duration"":40}" +1005,1708937946,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1006,1708938010,500.0,19,2,"{""description"":""some"",""duration"":40}" +1007,1708956931,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1008,1708956938,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1009,1709114657,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1010,1709114675,500.0,19,2,"{""description"":""some"",""duration"":40}" +1011,1709114706,900.0,19,2,"{""description"":""some"",""duration"":40}" +1012,1709114718,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1013,1709287407,1000.0,19,2,"{""description"":""some"",""duration"":40}" +1014,1709287419,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1015,1709287440,300.0,19,2,"{""description"":""some"",""duration"":40}" +1016,1709287454,2400.0,19,2,"{""description"":""some"",""duration"":40}" +1017,1709292049,1000.0,19,2,"{""description"":""some"",""duration"":40}" +1018,1709295355,1000.0,19,2,"{""description"":""some"",""duration"":40}" +1019,1709450798,5000.0,19,2,"{""description"":""some"",""duration"":40}" +1020,1709538152,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1021,1709542187,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1022,1709542200,100.0,19,2,"{""description"":""some"",""duration"":40}" +1023,1709550792,2000.0,19,2,"{""description"":""some"",""duration"":40}" +1024,1709732984,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1025,1709732996,2600.0,19,2,"{""description"":""some"",""duration"":40}" +1026,1709733004,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1027,1709733014,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1028,1709733025,500.0,19,2,"{""description"":""some"",""duration"":40}" +1029,1709822257,3000.0,19,2,"{""description"":""some"",""duration"":40}" +1030,1710347707,1000.0,19,2,"{""description"":""some"",""duration"":40}" +1031,1710347725,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1032,1710347733,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1033,1710347741,300.0,19,2,"{""description"":""some"",""duration"":40}" +1034,1710347748,900.0,19,2,"{""description"":""some"",""duration"":40}" +1035,1710347776,300.0,19,2,"{""description"":""some"",""duration"":40}" +1036,1710492247,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1037,1710499630,20000.0,19,2,"{""description"":""some"",""duration"":40}" +1038,1710754282,900.0,19,2,"{""description"":""some"",""duration"":40}" +1039,1710757850,600.0,19,2,"{""description"":""some"",""duration"":40}" +1040,1710757859,2000.0,19,2,"{""description"":""some"",""duration"":40}" +1041,1710775397,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1042,1710775427,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1043,1710930870,2000.0,19,2,"{""description"":""some"",""duration"":40}" +1044,1710930882,1000.0,19,2,"{""description"":""some"",""duration"":40}" +1045,1710938249,900.0,19,2,"{""description"":""some"",""duration"":40}" +1046,1710938315,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1047,1710938333,700.0,19,2,"{""description"":""some"",""duration"":40}" +1048,1710944451,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1049,1711010996,2000.0,19,2,"{""description"":""some"",""duration"":40}" +1050,1711011005,1000.0,19,2,"{""description"":""some"",""duration"":40}" +1051,1711014391,1500.0,19,2,"{""description"":""some"",""duration"":40}" +1052,1711014402,300.0,19,2,"{""description"":""some"",""duration"":40}" +1053,1711094574,2400.0,19,2,"{""description"":""some"",""duration"":40}" +1054,1711094585,1100.0,19,2,"{""description"":""some"",""duration"":40}" +1055,1711096245,900.0,19,2,"{""description"":""some"",""duration"":40}" +1056,1711096260,300.0,19,2,"{""description"":""some"",""duration"":40}" +1057,1711177704,7000.0,19,2,"{""description"":""some"",""duration"":40}" +1058,1711185522,2600.0,19,2,"{""description"":""some"",""duration"":40}" +1059,1711185534,600.0,19,2,"{""description"":""some"",""duration"":40}" +1060,1711375163,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1061,1711375189,500.0,19,2,"{""description"":""some"",""duration"":40}" +1062,1711375224,2400.0,19,2,"{""description"":""some"",""duration"":40}" +1063,1711375239,1500.0,19,2,"{""description"":""some"",""duration"":40}" diff --git a/crates/duckdb/examples/appender.rs b/crates/duckdb/examples/appender.rs index c159829d..764b6fef 100644 --- a/crates/duckdb/examples/appender.rs +++ b/crates/duckdb/examples/appender.rs @@ -1,9 +1,20 @@ extern crate duckdb; -use duckdb::{params, Connection, DropBehavior, Result}; +use std::time::Instant; + +use arrow_convert::{ArrowDeserialize, ArrowField, ArrowSerialize}; +use duckdb::{Connection, DropBehavior, Result}; + +#[derive(ArrowField, ArrowSerialize, ArrowDeserialize)] +struct User { + id: i32, + area: Option, + age: i8, + active: i8, +} fn main() -> Result<()> { - //let mut db = Connection::open("10m.db")?; + // let mut db = Connection::open("10m.db")?; let mut db = Connection::open_in_memory()?; let create_table_sql = " @@ -15,25 +26,20 @@ fn main() -> Result<()> { active TINYINT not null );"; db.execute_batch(create_table_sql)?; + let row_count = 10_000_000u32; + let data = firstn(row_count).collect_vec(); - let row_count = 10_000_000; { + let start = Instant::now(); let mut tx = db.transaction()?; tx.set_drop_behavior(DropBehavior::Commit); let mut app = tx.appender("test")?; // use generator - // for u in firstn(1_000_000) { - // app.append_row(params![u.id, u.area, u.age, u.active])?; - // } + app.append_rows_arrow(&data, true)?; - for i in 0..row_count { - app.append_row(params![ - i, - get_random_area_code(), - get_random_age(), - get_random_active(), - ])?; - } + let duration = start.elapsed(); + + println!("Time elapsed in transaction is: {:?}", duration); } let val = db.query_row("SELECT count(1) FROM test", [], |row| <(u32,)>::try_from(row))?; @@ -42,15 +48,7 @@ fn main() -> Result<()> { } #[allow(dead_code)] -struct User { - id: i32, - area: Option, - age: i8, - active: i8, -} - -#[allow(dead_code)] -fn firstn(n: i32) -> impl std::iter::Iterator { +fn firstn(n: u32) -> impl std::iter::Iterator { let mut id = 0; std::iter::from_fn(move || { if id >= n { @@ -58,7 +56,7 @@ fn firstn(n: i32) -> impl std::iter::Iterator { } id += 1; Some(User { - id, + id: id as i32, area: get_random_area_code(), age: get_random_age(), active: get_random_active(), @@ -66,8 +64,10 @@ fn firstn(n: i32) -> impl std::iter::Iterator { }) } +use itertools::Itertools; // Modified from https://github.com/avinassh/fast-sqlite3-inserts/blob/master/src/bin/common.rs -use rand::{prelude::SliceRandom, Rng}; +use rand::prelude::SliceRandom; +use rand::Rng; #[inline] fn get_random_age() -> i8 { diff --git a/crates/duckdb/examples/hello-ext/main.rs b/crates/duckdb/examples/hello-ext/main.rs index 5409dff1..28ef6c6a 100644 --- a/crates/duckdb/examples/hello-ext/main.rs +++ b/crates/duckdb/examples/hello-ext/main.rs @@ -3,7 +3,7 @@ extern crate duckdb_loadable_macros; extern crate libduckdb_sys; use duckdb::{ - vtab::{BindInfo, DataChunk, Free, FunctionInfo, InitInfo, Inserter, LogicalType, LogicalTypeId, VTab}, + vtab::{BindInfo, DataChunkHandle, Free, FunctionInfo, InitInfo, Inserter, LogicalTypeHandle, LogicalTypeId, VTab}, Connection, Result, }; use duckdb_loadable_macros::duckdb_entrypoint; @@ -42,8 +42,8 @@ impl VTab for HelloVTab { type InitData = HelloInitData; type BindData = HelloBindData; - unsafe fn bind(bind: &BindInfo, data: *mut HelloBindData) -> Result<(), Box> { - bind.add_result_column("column0", LogicalType::new(LogicalTypeId::Varchar)); + fn bind(bind: &BindInfo, data: *mut HelloBindData) -> Result<(), Box> { + bind.add_result_column("column0", LogicalTypeHandle::from(LogicalTypeId::Varchar)); let param = bind.get_parameter(0).to_string(); unsafe { (*data).name = CString::new(param).unwrap().into_raw(); @@ -51,14 +51,14 @@ impl VTab for HelloVTab { Ok(()) } - unsafe fn init(_: &InitInfo, data: *mut HelloInitData) -> Result<(), Box> { + fn init(_: &InitInfo, data: *mut HelloInitData) -> Result<(), Box> { unsafe { (*data).done = false; } Ok(()) } - unsafe fn func(func: &FunctionInfo, output: &mut DataChunk) -> Result<(), Box> { + fn func(func: &FunctionInfo, output: DataChunkHandle) -> Result<(), Box> { let init_info = func.get_init_data::(); let bind_info = func.get_bind_data::(); @@ -79,8 +79,8 @@ impl VTab for HelloVTab { Ok(()) } - fn parameters() -> Option> { - Some(vec![LogicalType::new(LogicalTypeId::Varchar)]) + fn parameters() -> Option> { + Some(vec![LogicalTypeHandle::from(LogicalTypeId::Varchar)]) } } diff --git a/crates/duckdb/src/appender/arrow.rs b/crates/duckdb/src/appender/arrow.rs deleted file mode 100644 index 19d79bb4..00000000 --- a/crates/duckdb/src/appender/arrow.rs +++ /dev/null @@ -1,82 +0,0 @@ -use super::{ffi, Appender, Result}; -use crate::{ - error::result_from_duckdb_appender, - vtab::{record_batch_to_duckdb_data_chunk, to_duckdb_logical_type, DataChunk, LogicalType}, - Error, -}; -use arrow::record_batch::RecordBatch; -use ffi::duckdb_append_data_chunk; - -impl Appender<'_> { - /// Append one record_batch - /// - /// ## Example - /// - /// ```rust,no_run - /// # use duckdb::{Connection, Result, params}; - /// use arrow::record_batch::RecordBatch; - /// fn insert_record_batch(conn: &Connection,record_batch:RecordBatch) -> Result<()> { - /// let mut app = conn.appender("foo")?; - /// app.append_record_batch(record_batch)?; - /// Ok(()) - /// } - /// ``` - /// - /// # Failure - /// - /// Will return `Err` if append column count not the same with the table schema - #[inline] - pub fn append_record_batch(&mut self, record_batch: RecordBatch) -> Result<()> { - let schema = record_batch.schema(); - let mut logical_type: Vec = vec![]; - for field in schema.fields() { - let logical_t = to_duckdb_logical_type(field.data_type()) - .map_err(|_op| Error::ArrowTypeToDuckdbType(field.to_string(), field.data_type().clone()))?; - logical_type.push(logical_t); - } - - let mut data_chunk = DataChunk::new(&logical_type); - record_batch_to_duckdb_data_chunk(&record_batch, &mut data_chunk).map_err(|_op| Error::AppendError)?; - - let rc = unsafe { duckdb_append_data_chunk(self.app, data_chunk.get_ptr()) }; - result_from_duckdb_appender(rc, &mut self.app) - } -} - -#[cfg(test)] -mod test { - use crate::{Connection, Result}; - use arrow::{ - array::{Int8Array, StringArray}, - datatypes::{DataType, Field, Schema}, - record_batch::RecordBatch, - }; - use std::sync::Arc; - - #[test] - fn test_append_record_batch() -> Result<()> { - let db = Connection::open_in_memory()?; - db.execute_batch("CREATE TABLE foo(id TINYINT not null,area TINYINT not null,name Varchar)")?; - { - let id_array = Int8Array::from(vec![1, 2, 3, 4, 5]); - let area_array = Int8Array::from(vec![11, 22, 33, 44, 55]); - let name_array = StringArray::from(vec![Some("11"), None, None, Some("44"), None]); - let schema = Schema::new(vec![ - Field::new("id", DataType::Int8, true), - Field::new("area", DataType::Int8, true), - Field::new("area", DataType::Utf8, true), - ]); - let record_batch = RecordBatch::try_new( - Arc::new(schema), - vec![Arc::new(id_array), Arc::new(area_array), Arc::new(name_array)], - ) - .unwrap(); - let mut app = db.appender("foo")?; - app.append_record_batch(record_batch)?; - } - let mut stmt = db.prepare("SELECT id, area,name FROM foo")?; - let rbs: Vec = stmt.query_arrow([])?.collect(); - assert_eq!(rbs.iter().map(|op| op.num_rows()).sum::(), 5); - Ok(()) - } -} diff --git a/crates/duckdb/src/appender/mod.rs b/crates/duckdb/src/appender/mod.rs index 488db82f..1f238ec7 100644 --- a/crates/duckdb/src/appender/mod.rs +++ b/crates/duckdb/src/appender/mod.rs @@ -1,22 +1,142 @@ -use super::{ffi, AppenderParams, Connection, Result, ValueRef}; -use std::{ffi::c_void, fmt, os::raw::c_char}; - -use crate::{ - error::result_from_duckdb_appender, - types::{ToSql, ToSqlOutput}, - Error, -}; +use std::ffi::{c_char, c_void}; +use std::fmt; + +use ::arrow::array::RecordBatch; +use arrow_convert::field::ArrowField; +use arrow_convert::serialize::{ArrowSerialize, FlattenRecordBatch, TryIntoArrow}; +use ffi::{duckdb_append_data_chunk, duckdb_appender_flush}; +use itertools::Itertools; + +use super::{ffi, Connection, Result}; +use crate::core::column_info::ColumnInfo; +use crate::core::{DataChunk, DataChunkHandle, LogicalType, LogicalTypeHandle}; +use crate::error::result_from_duckdb_appender; +use crate::types::{ToSqlOutput, ValueRef}; +#[cfg(feature = "appender-arrow")] +use crate::vtab::arrow::record_batch_to_duckdb_data_chunk; +use crate::{AppenderParams, Error, ToSql}; /// Appender for fast import data pub struct Appender<'conn> { conn: &'conn Connection, app: ffi::duckdb_appender, -} -#[cfg(feature = "appender-arrow")] -mod arrow; + /// column layout stored as tree, for fast access and to create data chunks + columns: Vec, + + /// the type of the columns this table stores + column_types: Vec, + + /// chunks that have not been flushed + chunks: Vec, +} impl Appender<'_> { + #[inline] + pub(super) fn new(conn: &Connection, app: ffi::duckdb_appender) -> Appender<'_> { + let column_count = unsafe { ffi::duckdb_appender_column_count(app) }; + + // initialize column_types + let column_types = (0..column_count) + .map(|i| { + let handle = LogicalTypeHandle::from(unsafe { ffi::duckdb_appender_column_type(app, i) }); + LogicalType::from(handle) + }) + .collect::>(); + + // initialize columns + let columns: Vec = (0..column_count) + .map(|i| ColumnInfo::new(&column_types[i as usize])) + .collect(); + + let chunks = vec![]; + + Appender { + conn, + app, + columns, + column_types, + chunks, + } + } + + #[cfg(feature = "appender-arrow")] + fn append_record_batch_inner(&mut self, record_batch: Vec) -> Result<()> { + for batch in record_batch { + let mut data_chunk = DataChunk::new(batch.num_rows(), &self.column_types, &self.columns); + record_batch_to_duckdb_data_chunk(&batch, &mut data_chunk).map_err(|_op| Error::AppendError)?; + self.chunks.push(data_chunk); + } + Ok(()) + } + + /// Append one record_batch + /// + /// ## Example + /// + /// ```rust,no_run + /// # use duckdb::{Connection, Result, params}; + /// use arrow::record_batch::RecordBatch; + /// fn insert_record_batch(conn: &Connection,record_batch:RecordBatch) -> Result<()> { + /// let mut app = conn.appender("foo")?; + /// app.append_record_batch(record_batch)?; + /// Ok(()) + /// } + /// ``` + /// + /// # Failure + /// + /// Will return `Err` if append column count not the same with the table schema + #[cfg(feature = "appender-arrow")] + pub fn append_record_batch(&mut self, record_batch: RecordBatch) -> Result<()> { + let batches = split_batch_for_data_chunk_capacity(record_batch, DataChunkHandle::max_capacity()); + self.append_record_batch_inner(batches) + } + + /// Append rows to the appender. + /// + /// # Arguments + /// + /// * `records` - An iterator over the records to be appended. + /// * `expand_struct` - A boolean indicating whether to expand the struct. + /// + /// # Example + /// + /// ```rust,no_run + /// # use duckdb::{Connection, Result, params}; + /// use arrow::record_batch::RecordBatch; + /// fn append_rows_example(conn: &Connection, records: &[RecordBatch], expand_struct: bool) -> Result<()> { + /// let mut app = conn.appender("foo")?; + /// app.append_rows_arrow(records, expand_struct)?; + /// Ok(()) + /// } + /// ``` + #[cfg(feature = "appender-arrow")] + pub fn append_rows_arrow<'a, T>( + &mut self, + records: impl IntoIterator, + expand_struct: bool, + ) -> Result<()> + where + T: ArrowSerialize + ArrowField + 'static, + { + let batches = records + .into_iter() + .chunks(DataChunkHandle::max_capacity()) + .into_iter() + .map(|chunk| { + let batch: RecordBatch = chunk.try_into_arrow()?; + if expand_struct { + batch.flatten() + } else { + Ok(batch) + } + }) + .collect::, _>>()?; + + self.append_record_batch_inner(batches) + } + /// Append multiple rows from Iterator /// /// ## Example @@ -143,25 +263,24 @@ impl Appender<'_> { Ok(()) } - #[inline] - pub(super) fn new(conn: &Connection, app: ffi::duckdb_appender) -> Appender<'_> { - Appender { conn, app } - } - /// Flush data into DB #[inline] pub fn flush(&mut self) -> Result<()> { - unsafe { - let res = ffi::duckdb_appender_flush(self.app); - result_from_duckdb_appender(res, &mut self.app) + // append data chunks + for chunk in self.chunks.drain(..) { + let rc = unsafe { duckdb_append_data_chunk(self.app, chunk.get_handle().ptr) }; + result_from_duckdb_appender(rc, &mut self.app)?; } + + let rc = unsafe { duckdb_appender_flush(self.app) }; + result_from_duckdb_appender(rc, &mut self.app) } } impl Drop for Appender<'_> { fn drop(&mut self) { if !self.app.is_null() { - let _ = self.flush(); // can't safely handle failures here + self.flush().expect("Failed to flush appender"); unsafe { ffi::duckdb_appender_close(self.app); ffi::duckdb_appender_destroy(&mut self.app); @@ -176,10 +295,61 @@ impl fmt::Debug for Appender<'_> { } } +/// Splits a given `RecordBatch` into multiple smaller batches based on the maximum number of rows per batch. +fn split_batch_for_data_chunk_capacity(batch: RecordBatch, data_chunk_max_rows: usize) -> Vec { + let rows_per_batch = data_chunk_max_rows.max(1); + let n_batches = (batch.num_rows() / rows_per_batch).max(1); + let mut out = Vec::with_capacity(n_batches + 1); + + let mut offset = 0; + while offset < batch.num_rows() { + let length = (rows_per_batch).min(batch.num_rows() - offset); + out.push(batch.slice(offset, length)); + + offset += length; + } + + out +} + #[cfg(test)] mod test { + use std::sync::Arc; + + use arrow::array::{Int8Array, StringArray}; + use arrow::datatypes::{DataType, Field, Schema}; + use arrow::record_batch::RecordBatch; + use crate::{Connection, Result}; + #[cfg(feature = "appender-arrow")] + #[test] + fn test_append_record_batch() -> Result<()> { + let db = Connection::open_in_memory()?; + db.execute_batch("CREATE TABLE foo(id TINYINT not null,area TINYINT not null,name Varchar)")?; + { + let id_array = Int8Array::from(vec![1, 2, 3, 4, 5]); + let area_array = Int8Array::from(vec![11, 22, 33, 44, 55]); + let name_array = StringArray::from(vec![Some("11"), None, None, Some("44"), None]); + let schema = Schema::new(vec![ + Field::new("id", DataType::Int8, true), + Field::new("area", DataType::Int8, true), + Field::new("area", DataType::Utf8, true), + ]); + let record_batch = RecordBatch::try_new( + Arc::new(schema), + vec![Arc::new(id_array), Arc::new(area_array), Arc::new(name_array)], + ) + .unwrap(); + let mut app = db.appender("foo")?; + app.append_record_batch(record_batch)?; + } + let mut stmt = db.prepare("SELECT id, area,name FROM foo")?; + let rbs: Vec = stmt.query_arrow([])?.collect(); + assert_eq!(rbs.iter().map(|op| op.num_rows()).sum::(), 5); + Ok(()) + } + #[test] fn test_append_one_row() -> Result<()> { let db = Connection::open_in_memory()?; @@ -266,9 +436,10 @@ mod test { #[test] #[cfg(feature = "chrono")] fn test_append_datetime() -> Result<()> { - use crate::params; use chrono::{NaiveDate, NaiveDateTime}; + use crate::params; + let db = Connection::open_in_memory()?; db.execute_batch("CREATE TABLE foo(x DATE, y TIMESTAMP)")?; diff --git a/crates/duckdb/src/core/column_info.rs b/crates/duckdb/src/core/column_info.rs new file mode 100644 index 00000000..e40a6dce --- /dev/null +++ b/crates/duckdb/src/core/column_info.rs @@ -0,0 +1,133 @@ +use arrow::datatypes::DataType; +use derive_more::Unwrap; + +use super::LogicalType; + +#[derive(Debug)] +pub struct ListColumnInfo { + pub child: Box, +} + +// note: Also works for fixed size length +#[derive(Debug)] +pub struct ArrayColumnInfo { + pub array_size: usize, + pub child: Box, +} + +#[derive(Debug)] +pub struct StructColumnInfo { + pub children: Vec, +} + +#[derive(Debug)] +pub struct UnionColumnInfo { + pub members: Vec, +} + +/// Represents the duckdb schema for error handling and caching purposes. +#[derive(Unwrap, Debug)] +pub enum ColumnInfo { + Flat, + List(ListColumnInfo), + Array(ArrayColumnInfo), + Struct(StructColumnInfo), + Union(UnionColumnInfo), +} + +impl ColumnInfo { + pub fn new(logical_type: &LogicalType) -> Self { + match logical_type { + LogicalType::General { .. } | LogicalType::Decimal(_) => ColumnInfo::Flat, + LogicalType::List(ty) => { + let child_type = ty.child_type(); + let child_info = Self::new(&child_type); + + ColumnInfo::List(ListColumnInfo { + child: Box::new(child_info), + }) + } + LogicalType::Struct(ty) => { + let child_count = ty.child_count(); + + let child_column_infos = (0..child_count) + .map(|i| { + let child_type = ty.child_type(i); + Self::new(&child_type) + }) + .collect::>(); + + ColumnInfo::Struct(StructColumnInfo { + children: child_column_infos, + }) + } + LogicalType::Map(_) => unimplemented!(), + LogicalType::Union(ty) => { + let child_count = ty.member_count(); + + let member_column_infos = (0..child_count) + .map(|i| { + let child_type = ty.member_type(i); + Self::new(&child_type) + }) + .collect::>(); + + ColumnInfo::Union(UnionColumnInfo { + members: member_column_infos, + }) + } + LogicalType::Array(ty) => { + let array_size = ty.size(); + let child_type = ty.child_type(); + let child_info = Self::new(&child_type); + + ColumnInfo::Array(ArrayColumnInfo { + array_size, + child: Box::new(child_info), + }) + } + } + } + + pub fn new_from_data_type(data_type: &DataType) -> Self { + match data_type { + dt if dt.is_primitive() => ColumnInfo::Flat, + DataType::Binary | DataType::Utf8 => ColumnInfo::Flat, + DataType::List(ty) => { + let child_info = Self::new_from_data_type(ty.data_type()); + ColumnInfo::List(ListColumnInfo { + child: Box::new(child_info), + }) + } + DataType::FixedSizeList(field, size) => { + let child_info = Self::new_from_data_type(field.data_type()); + ColumnInfo::Array(ArrayColumnInfo { + array_size: *size as usize, + child: Box::new(child_info), + }) + } + DataType::Struct(fields) => { + let children = fields + .iter() + .map(|field| { + let child_info = Self::new_from_data_type(field.data_type()); + child_info + }) + .collect::>(); + ColumnInfo::Struct(StructColumnInfo { children }) + } + DataType::Union(fields, _) => { + let members = fields + .iter() + .map(|(_, field)| { + let child_info = Self::new_from_data_type(field.data_type()); + child_info + }) + .collect::>(); + ColumnInfo::Union(UnionColumnInfo { members }) + } + DataType::Map(_, _) => todo!(), + dt => unimplemented!("data type: {} is not implemented yet.", dt), + } + } +} diff --git a/crates/duckdb/src/core/data_chunk.rs b/crates/duckdb/src/core/data_chunk.rs new file mode 100644 index 00000000..db39a45d --- /dev/null +++ b/crates/duckdb/src/core/data_chunk.rs @@ -0,0 +1,144 @@ +use std::sync::Arc; + +use libduckdb_sys::duckdb_vector_size; + +use super::column_info::ColumnInfo; +use crate::core::{LogicalType, Vector, VectorHandle}; +use crate::ffi::{ + duckdb_create_data_chunk, duckdb_data_chunk, duckdb_data_chunk_get_column_count, duckdb_data_chunk_get_size, + duckdb_data_chunk_get_vector, duckdb_data_chunk_set_size, duckdb_destroy_data_chunk, +}; + +/// DataChunk in DuckDB. +pub struct DataChunkHandle { + /// Pointer to the DataChunk in duckdb C API. + pub(crate) ptr: duckdb_data_chunk, + + /// Whether this [DataChunk] own the [DataChunk::ptr]. + owned: bool, +} + +impl Drop for DataChunkHandle { + fn drop(&mut self) { + if self.owned && !self.ptr.is_null() { + unsafe { duckdb_destroy_data_chunk(&mut self.ptr) } + } + self.ptr = std::ptr::null_mut(); + } +} + +impl DataChunkHandle { + pub(crate) unsafe fn new_unowned(ptr: duckdb_data_chunk) -> Self { + Self { ptr, owned: false } + } + + /// Create a new [DataChunk] with the given [LogicalTypeHandle]s. + pub fn new(logical_types: &[LogicalType]) -> Self { + let num_columns = logical_types.len(); + let mut c_types = logical_types.iter().map(|t| unsafe { t.ptr() }).collect::>(); + let ptr = unsafe { duckdb_create_data_chunk(c_types.as_mut_ptr(), num_columns as u64) }; + Self { ptr, owned: true } + } + + /// Get the vector at the specific column index: `idx`. + pub(crate) fn get_vector(&self, idx: usize) -> VectorHandle { + unsafe { VectorHandle::new_unchecked(duckdb_data_chunk_get_vector(self.ptr, idx as u64)) } + } + + /// Get the maximum capacity of the data chunk. + /// Data should be capped at this size. + /// If the data exceeds this size, it should be split into multiple data chunks. + pub fn max_capacity() -> usize { + unsafe { duckdb_vector_size() as usize } + } + + /// Set the size of the data chunk + pub fn set_len(&self, new_len: usize) { + unsafe { duckdb_data_chunk_set_size(self.ptr, new_len as u64) }; + } + + pub fn reserve(&self, size: usize) { + if size > self.len() { + self.set_len(size.next_power_of_two()); + } + } + + pub fn with_capacity(self, size: usize) -> Self { + self.reserve(size); + self + } + + /// Get the length / the number of rows in this [DataChunk]. + pub fn len(&self) -> usize { + unsafe { duckdb_data_chunk_get_size(self.ptr) as usize } + } + + pub fn num_columns(&self) -> usize { + unsafe { duckdb_data_chunk_get_column_count(self.ptr) as usize } + } + + /// Check whether this [DataChunk] is empty. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +pub struct DataChunk { + handle: Arc, + pub columns: Vec, +} + +impl DataChunk { + // Add data chunk construction from arrow::RecordBatch + pub fn new(size: usize, logical_types: &[LogicalType], columns: &[ColumnInfo]) -> Self { + let handle = Arc::new(DataChunkHandle::new(logical_types).with_capacity(size)); + let columns = Vector::new_from_column_infos(columns, handle.clone()); + Self { handle, columns } + } + + pub fn new_from_record_batch_handle(schema: &arrow::datatypes::SchemaRef, handle: DataChunkHandle) -> Self { + let handle = Arc::new(handle); + let columns = Vector::new_from_record_batch_schema(schema, handle.clone()); + Self { handle, columns } + } + + /// Get the ptr of duckdb_data_chunk in this [DataChunk]. + pub fn get_handle(&self) -> Arc { + self.handle.clone() + } + + /// Get the number of columns in this [DataChunk]. + pub fn num_columns(&self) -> usize { + self.columns.len() + } + + pub fn set_len(&mut self, size: usize) { + self.handle.set_len(size); + } +} + +#[cfg(test)] +mod test { + use super::super::logical_type::LogicalTypeId; + use super::*; + use crate::core::{FlatVector, LogicalTypeHandle}; + + #[test] + fn test_data_chunk_construction() { + let dc = DataChunkHandle::new(&[LogicalTypeHandle::from(LogicalTypeId::Integer).into()]); + + assert_eq!(dc.num_columns(), 1); + + drop(dc); + } + + #[test] + fn test_vector() { + let datachunk = DataChunkHandle::new(&[LogicalTypeHandle::from(LogicalTypeId::Bigint).into()]); + datachunk.reserve(1); + let mut vector = FlatVector::new(datachunk.get_vector(0), Arc::new(move || datachunk.len())); + let data = vector.as_mut_slice::(); + + data[0] = 42; + } +} diff --git a/crates/duckdb/src/core/logical_type.rs b/crates/duckdb/src/core/logical_type.rs new file mode 100644 index 00000000..83e63a2c --- /dev/null +++ b/crates/duckdb/src/core/logical_type.rs @@ -0,0 +1,524 @@ +use std::ffi::{c_char, CString}; +use std::fmt::Debug; + +use derive_more::{Constructor, From}; +use strum::EnumTryAs; + +use crate::ffi::*; + +/// Logical Type Id +/// https://duckdb.org/docs/api/c/types +#[repr(u32)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LogicalTypeId { + /// Boolean + Boolean = DUCKDB_TYPE_DUCKDB_TYPE_BOOLEAN, + /// Tinyint + Tinyint = DUCKDB_TYPE_DUCKDB_TYPE_TINYINT, + /// Smallint + Smallint = DUCKDB_TYPE_DUCKDB_TYPE_SMALLINT, + /// Integer + Integer = DUCKDB_TYPE_DUCKDB_TYPE_INTEGER, + /// Bigint + Bigint = DUCKDB_TYPE_DUCKDB_TYPE_BIGINT, + /// Unsigned Tinyint + UTinyint = DUCKDB_TYPE_DUCKDB_TYPE_UTINYINT, + /// Unsigned Smallint + USmallint = DUCKDB_TYPE_DUCKDB_TYPE_USMALLINT, + /// Unsigned Integer + UInteger = DUCKDB_TYPE_DUCKDB_TYPE_UINTEGER, + /// Unsigned Bigint + UBigint = DUCKDB_TYPE_DUCKDB_TYPE_UBIGINT, + /// Float + Float = DUCKDB_TYPE_DUCKDB_TYPE_FLOAT, + /// Double + Double = DUCKDB_TYPE_DUCKDB_TYPE_DOUBLE, + /// Timestamp + Timestamp = DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP, + /// Date + Date = DUCKDB_TYPE_DUCKDB_TYPE_DATE, + /// Time + Time = DUCKDB_TYPE_DUCKDB_TYPE_TIME, + /// Interval + Interval = DUCKDB_TYPE_DUCKDB_TYPE_INTERVAL, + /// Hugeint + Hugeint = DUCKDB_TYPE_DUCKDB_TYPE_HUGEINT, + /// Varchar + Varchar = DUCKDB_TYPE_DUCKDB_TYPE_VARCHAR, + /// Blob + Blob = DUCKDB_TYPE_DUCKDB_TYPE_BLOB, + /// Decimal + Decimal = DUCKDB_TYPE_DUCKDB_TYPE_DECIMAL, + /// Timestamp S + TimestampS = DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_S, + /// Timestamp MS + TimestampMs = DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_MS, + /// Timestamp NS + TimestampNs = DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_NS, + /// Enum + Enum = DUCKDB_TYPE_DUCKDB_TYPE_ENUM, + /// List + List = DUCKDB_TYPE_DUCKDB_TYPE_LIST, + /// Struct + Struct = DUCKDB_TYPE_DUCKDB_TYPE_STRUCT, + /// Map + Map = DUCKDB_TYPE_DUCKDB_TYPE_MAP, + /// Uuid + Uuid = DUCKDB_TYPE_DUCKDB_TYPE_UUID, + /// Union + Union = DUCKDB_TYPE_DUCKDB_TYPE_UNION, + /// Array + Array = DUCKDB_TYPE_DUCKDB_TYPE_ARRAY, + /// Timestamp TZ + TimestampTZ = DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_TZ, +} + +impl From for LogicalTypeId { + /// Convert from u32 to LogicalTypeId + fn from(value: u32) -> Self { + match value { + DUCKDB_TYPE_DUCKDB_TYPE_BOOLEAN => Self::Boolean, + DUCKDB_TYPE_DUCKDB_TYPE_TINYINT => Self::Tinyint, + DUCKDB_TYPE_DUCKDB_TYPE_SMALLINT => Self::Smallint, + DUCKDB_TYPE_DUCKDB_TYPE_INTEGER => Self::Integer, + DUCKDB_TYPE_DUCKDB_TYPE_BIGINT => Self::Bigint, + DUCKDB_TYPE_DUCKDB_TYPE_UTINYINT => Self::UTinyint, + DUCKDB_TYPE_DUCKDB_TYPE_USMALLINT => Self::USmallint, + DUCKDB_TYPE_DUCKDB_TYPE_UINTEGER => Self::UInteger, + DUCKDB_TYPE_DUCKDB_TYPE_UBIGINT => Self::UBigint, + DUCKDB_TYPE_DUCKDB_TYPE_FLOAT => Self::Float, + DUCKDB_TYPE_DUCKDB_TYPE_DOUBLE => Self::Double, + DUCKDB_TYPE_DUCKDB_TYPE_VARCHAR => Self::Varchar, + DUCKDB_TYPE_DUCKDB_TYPE_BLOB => Self::Blob, + DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP => Self::Timestamp, + DUCKDB_TYPE_DUCKDB_TYPE_DATE => Self::Date, + DUCKDB_TYPE_DUCKDB_TYPE_TIME => Self::Time, + DUCKDB_TYPE_DUCKDB_TYPE_INTERVAL => Self::Interval, + DUCKDB_TYPE_DUCKDB_TYPE_HUGEINT => Self::Hugeint, + DUCKDB_TYPE_DUCKDB_TYPE_DECIMAL => Self::Decimal, + DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_S => Self::TimestampS, + DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_MS => Self::TimestampMs, + DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_NS => Self::TimestampNs, + DUCKDB_TYPE_DUCKDB_TYPE_ENUM => Self::Enum, + DUCKDB_TYPE_DUCKDB_TYPE_LIST => Self::List, + DUCKDB_TYPE_DUCKDB_TYPE_ARRAY => Self::Array, + DUCKDB_TYPE_DUCKDB_TYPE_STRUCT => Self::Struct, + DUCKDB_TYPE_DUCKDB_TYPE_MAP => Self::Map, + DUCKDB_TYPE_DUCKDB_TYPE_UUID => Self::Uuid, + DUCKDB_TYPE_DUCKDB_TYPE_UNION => Self::Union, + DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_TZ => Self::TimestampTZ, + _ => unimplemented!("{value} not implemented"), + } + } +} + +#[derive(Constructor, Debug)] +pub struct LogicalTypeHandle { + pub(crate) ptr: duckdb_logical_type, +} + +#[derive(From, Constructor, Debug)] +pub struct DecimalLogicalType(LogicalTypeHandle); + +impl DecimalLogicalType { + /// Retrieves the decimal width + /// Returns 0 if the LogicalType is not a decimal + pub fn decimal_width(&self) -> u8 { + unsafe { duckdb_decimal_width(self.0.ptr) } + } + + /// Retrieves the decimal scale + /// Returns 0 if the LogicalType is not a decimal + pub fn decimal_scale(&self) -> u8 { + unsafe { duckdb_decimal_scale(self.0.ptr) } + } +} + +#[derive(From, Constructor, Debug)] +pub struct ListLogicalType(LogicalTypeHandle); + +impl ListLogicalType { + pub fn child_type(&self) -> LogicalType { + let handle = LogicalTypeHandle::new(unsafe { duckdb_list_type_child_type(self.0.ptr) }); + + LogicalType::from(handle) + } +} + +#[derive(From, Constructor, Debug)] +pub struct UnionLogicalType(LogicalTypeHandle); + +impl UnionLogicalType { + pub fn member_count(&self) -> usize { + unsafe { duckdb_union_type_member_count(self.0.ptr) as usize } + } + + /// Logical type child by idx + pub fn member_type(&self, idx: usize) -> LogicalType { + let handle = LogicalTypeHandle::new(unsafe { duckdb_union_type_member_type(self.0.ptr, idx as u64) }); + LogicalType::from(handle) + } + + /// Logical type child name by idx + pub fn member_name(&self, idx: usize) -> String { + unsafe { + let child_name_ptr = duckdb_union_type_member_name(self.0.ptr, idx as u64); + let c_str = CString::from_raw(child_name_ptr); + let name = c_str.to_str().unwrap(); + name.to_string() + } + } +} + +#[derive(From, Constructor, Debug)] +pub struct StructLogicalType(LogicalTypeHandle); + +impl StructLogicalType { + /// Logical type child by idx + pub fn child_type(&self, idx: usize) -> LogicalType { + let handle = LogicalTypeHandle::new(unsafe { duckdb_struct_type_child_type(self.0.ptr, idx as u64) }); + LogicalType::from(handle) + } + + pub fn child_count(&self) -> usize { + unsafe { duckdb_struct_type_child_count(self.0.ptr) as usize } + } + + /// Logical type child name by idx + /// + /// Panics if the logical type is not a struct or union + pub fn child_name(&self, idx: usize) -> String { + unsafe { + let child_name_ptr = duckdb_struct_type_child_name(self.0.ptr, idx as u64); + let c_str = CString::from_raw(child_name_ptr); + let name = c_str.to_str().unwrap(); + name.to_string() + } + } +} + +#[derive(From, Constructor, Debug)] +pub struct MapLogicalType(LogicalTypeHandle); + +impl MapLogicalType { + pub fn key_type(&self) -> LogicalType { + let handle = LogicalTypeHandle::new(unsafe { duckdb_map_type_key_type(self.0.ptr) }); + handle.into() + } + + pub fn value_type(&self) -> LogicalType { + let handle = LogicalTypeHandle::new(unsafe { duckdb_map_type_value_type(self.0.ptr) }); + handle.into() + } +} + +#[derive(From, Constructor, Debug)] +pub struct ArrayLogicalType(LogicalTypeHandle); + +impl ArrayLogicalType { + pub fn size(&self) -> usize { + unsafe { duckdb_array_type_array_size(self.0.ptr) as usize } + } + + pub fn child_type(&self) -> LogicalType { + let handle = LogicalTypeHandle::new(unsafe { duckdb_array_type_child_type(self.0.ptr) }); + LogicalType::from(handle) + } +} + +/// DuckDB Logical Type. +/// Modelled from https://duckdb.org/docs/sql/data_types/overview +#[derive(EnumTryAs, Debug)] +pub enum LogicalType { + // TODO: Possibly break this down further + General { + id: LogicalTypeId, + handle: LogicalTypeHandle, + }, + List(ListLogicalType), + Decimal(DecimalLogicalType), + Union(UnionLogicalType), + Struct(StructLogicalType), + Array(ArrayLogicalType), + Map(MapLogicalType), +} + +impl From for LogicalType { + fn from(handle: LogicalTypeHandle) -> Self { + let id = handle.id(); + match id { + LogicalTypeId::Boolean + | LogicalTypeId::Tinyint + | LogicalTypeId::Smallint + | LogicalTypeId::Integer + | LogicalTypeId::Bigint + | LogicalTypeId::UTinyint + | LogicalTypeId::USmallint + | LogicalTypeId::UInteger + | LogicalTypeId::UBigint + | LogicalTypeId::Float + | LogicalTypeId::Double + | LogicalTypeId::Timestamp + | LogicalTypeId::TimestampTZ + | LogicalTypeId::Date + | LogicalTypeId::Time + | LogicalTypeId::Interval + | LogicalTypeId::Hugeint + | LogicalTypeId::Varchar + | LogicalTypeId::Blob + | LogicalTypeId::TimestampS + | LogicalTypeId::TimestampMs + | LogicalTypeId::TimestampNs + | LogicalTypeId::Uuid + | LogicalTypeId::Enum => LogicalType::General { id, handle }, + LogicalTypeId::Decimal => Self::Decimal(handle.into()), + LogicalTypeId::List => Self::List(handle.into()), + LogicalTypeId::Struct => Self::Struct(handle.into()), + LogicalTypeId::Map => Self::Map(handle.into()), + LogicalTypeId::Union => Self::Union(handle.into()), + LogicalTypeId::Array => Self::Array(handle.into()), + } + } +} + +impl LogicalType { + /// Creates a map type from its child type. + pub fn new_map(key: &LogicalTypeHandle, value: &LogicalTypeHandle) -> Self { + Self::Map(LogicalTypeHandle::map(key, value).into()) + } + + /// Creates a list type from its child type. + pub fn new_list(child_type: &LogicalTypeHandle) -> Self { + Self::List(LogicalTypeHandle::list(child_type).into()) + } + + /// Creates a array type from its child type. + pub fn new_array(child_type: &LogicalTypeHandle, size: i32) -> Self { + Self::Array(LogicalTypeHandle::array(child_type, size).into()) + } + + /// Creates a decimal type from its `width` and `scale`. + pub fn new_decimal(width: u8, scale: u8) -> Self { + Self::Decimal(LogicalTypeHandle::decimal(width, scale).into()) + } + + /// Make a `LogicalType` for `struct` + pub fn new_struct_type(fields: &[(&str, LogicalTypeHandle)]) -> Self { + Self::Struct(LogicalTypeHandle::struct_type(fields).into()) + } + + /// Make a `LogicalType` for `union` + pub fn new_union_type(fields: &[(&str, LogicalTypeHandle)]) -> Self { + Self::Union(LogicalTypeHandle::union_type(fields).into()) + } + + pub fn logical_id(&self) -> LogicalTypeId { + match self { + LogicalType::General { id, .. } => *id, + LogicalType::Decimal(_) => LogicalTypeId::Decimal, + LogicalType::List(_) => LogicalTypeId::List, + LogicalType::Struct(_) => LogicalTypeId::Struct, + LogicalType::Map(_) => LogicalTypeId::Map, + LogicalType::Union(_) => LogicalTypeId::Union, + LogicalType::Array(_) => LogicalTypeId::Array, + } + } + + /// This method returns the number of children in the struct or union. + /// This is unsafe because dropping the LogicalTypeHandle will invalidate the pointer. + pub unsafe fn ptr(&self) -> duckdb_logical_type { + match self { + LogicalType::General { handle, .. } => handle.ptr, + LogicalType::Decimal(ty) => ty.0.ptr, + LogicalType::List(ty) => ty.0.ptr, + LogicalType::Struct(ty) => ty.0.ptr, + LogicalType::Map(ty) => ty.0.ptr, + LogicalType::Union(ty) => ty.0.ptr, + LogicalType::Array(ty) => ty.0.ptr, + } + } +} + +// impl Debug for LogicalTypeHandle { +// /// Debug implementation for LogicalType +// fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { +// let id = self.id(); +// match id { +// LogicalTypeId::Struct => { +// write!(f, "struct<")?; +// for i in 0..self.num_children() { +// if i > 0 { +// write!(f, ", ")?; +// } +// write!(f, "{}: {:?}", self.child_name(i), self.struct_or_union_child(i))?; +// } +// write!(f, ">") +// } +// _ => write!(f, "{:?}", self.id()), +// } +// } +// } + +impl Drop for LogicalTypeHandle { + /// Drop implementation for LogicalType + fn drop(&mut self) { + if !self.ptr.is_null() { + unsafe { + duckdb_destroy_logical_type(&mut self.ptr); + } + } + + self.ptr = std::ptr::null_mut(); + } +} + +impl From for LogicalTypeHandle { + /// Wrap a DuckDB logical type from C API + fn from(ptr: duckdb_logical_type) -> Self { + Self { ptr } + } +} + +impl From for LogicalTypeHandle { + /// Create a new [LogicalTypeHandle] from [LogicalTypeId] + fn from(id: LogicalTypeId) -> Self { + unsafe { + Self { + ptr: duckdb_create_logical_type(id as u32), + } + } + } +} + +// constructors +impl LogicalTypeHandle { + /// Creates a map type from its child type. + pub fn map(key: &LogicalTypeHandle, value: &LogicalTypeHandle) -> Self { + unsafe { + Self { + ptr: duckdb_create_map_type(key.ptr, value.ptr), + } + } + } + + /// Creates a list type from its child type. + pub fn list(child_type: &LogicalTypeHandle) -> Self { + unsafe { + Self { + ptr: duckdb_create_list_type(child_type.ptr), + } + } + } + + /// Creates a array type from its child type. + pub fn array(child_type: &LogicalTypeHandle, size: i32) -> Self { + unsafe { + Self { + ptr: duckdb_create_array_type(child_type.ptr, size as idx_t), + } + } + } + + /// Creates a decimal type from its `width` and `scale`. + pub fn decimal(width: u8, scale: u8) -> Self { + unsafe { + Self { + ptr: duckdb_create_decimal_type(width, scale), + } + } + } + + /// Make a `LogicalType` for `struct` + pub fn struct_type(fields: &[(&str, LogicalTypeHandle)]) -> Self { + let keys: Vec = fields.iter().map(|f| CString::new(f.0).unwrap()).collect(); + let values: Vec = fields.iter().map(|it| it.1.ptr).collect(); + let name_ptrs = keys.iter().map(|it| it.as_ptr()).collect::>(); + + unsafe { + Self { + ptr: duckdb_create_struct_type( + values.as_slice().as_ptr().cast_mut(), + name_ptrs.as_slice().as_ptr().cast_mut(), + fields.len() as idx_t, + ), + } + } + } + + /// Make a `LogicalType` for `union` + pub fn union_type(fields: &[(&str, LogicalTypeHandle)]) -> Self { + let keys: Vec = fields.iter().map(|f| CString::new(f.0).unwrap()).collect(); + let values: Vec = fields.iter().map(|it| it.1.ptr).collect(); + let name_ptrs = keys.iter().map(|it| it.as_ptr()).collect::>(); + + unsafe { + Self { + ptr: duckdb_create_union_type( + values.as_slice().as_ptr().cast_mut(), + name_ptrs.as_slice().as_ptr().cast_mut(), + fields.len() as idx_t, + ), + } + } + } +} + +impl LogicalTypeHandle { + /// Logical type ID + pub fn id(&self) -> LogicalTypeId { + let duckdb_type_id = unsafe { duckdb_get_type_id(self.ptr) }; + duckdb_type_id.into() + } +} + +#[cfg(test)] +mod test { + use super::{LogicalType, LogicalTypeHandle, LogicalTypeId}; + + #[test] + fn test_struct() { + let fields = &[("hello", LogicalTypeHandle::from(LogicalTypeId::Boolean))]; + let typ = LogicalType::new_struct_type(fields).try_as_struct().unwrap(); + + assert_eq!(typ.child_count(), 1); + assert_eq!(typ.child_name(0), "hello"); + assert_eq!(typ.child_type(0).logical_id(), LogicalTypeId::Boolean); + } + + #[test] + fn test_decimal() { + let typ = LogicalType::new_decimal(10, 2).try_as_decimal().unwrap(); + + assert_eq!(typ.decimal_width(), 10); + assert_eq!(typ.decimal_scale(), 2); + } + + #[test] + fn test_union_type() { + let fields = &[ + ("hello", LogicalTypeHandle::from(LogicalTypeId::Boolean)), + ("world", LogicalTypeHandle::from(LogicalTypeId::Integer)), + ]; + let typ = LogicalType::new_union_type(fields).try_as_union().unwrap(); + + assert_eq!(typ.member_count(), 2); + + assert_eq!(typ.member_name(0), "hello"); + assert_eq!(typ.member_type(0).logical_id(), LogicalTypeId::Boolean); + + assert_eq!(typ.member_name(1), "world"); + assert_eq!(typ.member_type(1).logical_id(), LogicalTypeId::Integer); + } + + #[test] + fn test_map_type() { + let key = LogicalTypeHandle::from(LogicalTypeId::Varchar); + let value = LogicalTypeHandle::from(LogicalTypeId::UTinyint); + let map = LogicalTypeHandle::map(&key, &value); + + assert_eq!(map.id(), LogicalTypeId::Map); + let typ = LogicalType::new_map(&key, &value).try_as_map().unwrap(); + assert_eq!(typ.key_type().logical_id(), LogicalTypeId::Varchar); + assert_eq!(typ.value_type().logical_id(), LogicalTypeId::UTinyint); + } +} diff --git a/crates/duckdb/src/core/mod.rs b/crates/duckdb/src/core/mod.rs new file mode 100644 index 00000000..6d40908e --- /dev/null +++ b/crates/duckdb/src/core/mod.rs @@ -0,0 +1,8 @@ +pub mod column_info; +pub mod data_chunk; +pub mod logical_type; +pub mod vector; + +pub use data_chunk::*; +pub use logical_type::*; +pub use vector::*; diff --git a/crates/duckdb/src/core/vector.rs b/crates/duckdb/src/core/vector.rs new file mode 100644 index 00000000..432a12e2 --- /dev/null +++ b/crates/duckdb/src/core/vector.rs @@ -0,0 +1,459 @@ +use std::ffi::{c_char, CString}; +use std::fmt::Debug; +use std::sync::Arc; +use std::{marker, slice}; + +use arrow::datatypes::SchemaRef; +use derive_more::{Constructor, From}; +use libduckdb_sys::{ + duckdb_array_vector_get_child, duckdb_union_type_member_name, duckdb_vector_assign_string_element_len, idx_t, + DuckDbStr, +}; + +use super::column_info::ColumnInfo; +use super::{DataChunkHandle, LogicalTypeHandle}; +use crate::ffi::{ + duckdb_list_entry, duckdb_list_vector_get_child, duckdb_list_vector_get_size, duckdb_list_vector_reserve, + duckdb_list_vector_set_size, duckdb_struct_type_child_count, duckdb_struct_type_child_name, + duckdb_struct_vector_get_child, duckdb_validity_set_row_invalid, duckdb_vector, + duckdb_vector_assign_string_element, duckdb_vector_ensure_validity_writable, duckdb_vector_get_column_type, + duckdb_vector_get_data, duckdb_vector_get_validity, +}; + +/// There is no way to get the length of the FlatVector directly, +/// As the C Api is not exposed making us do this ugly hack. +type GetVectorSize = dyn Fn() -> usize; + +/// A handle to the vector created by data chunk +#[derive(Debug, Constructor, Clone)] +pub(crate) struct VectorHandle { + ptr: duckdb_vector, + _marker: marker::PhantomData, +} + +impl VectorHandle { + pub unsafe fn new_unchecked(ptr: duckdb_vector) -> Self { + Self { + ptr, + _marker: marker::PhantomData, + } + } + + pub fn cast(&self) -> VectorHandle { + VectorHandle { + ptr: self.ptr, + _marker: marker::PhantomData, + } + } + + pub fn as_flat(&self) -> VectorHandle { + self.cast() + } + + pub fn as_list(&self) -> VectorHandle { + self.cast() + } + + pub fn as_array(&self) -> VectorHandle { + self.cast() + } + + pub fn as_struct(&self) -> VectorHandle { + self.cast() + } + + pub fn as_union(&self) -> VectorHandle { + self.cast() + } +} + +impl VectorHandle { + pub fn get_child(&self) -> VectorHandle { + unsafe { VectorHandle::new_unchecked(duckdb_list_vector_get_child(self.ptr)) } + } + + pub fn get_size(&self) -> usize { + unsafe { duckdb_list_vector_get_size(self.ptr) as usize } + } + + pub fn reserve_child_capacity(&mut self, capacity: usize) { + unsafe { + duckdb_list_vector_reserve(self.ptr, capacity as u64); + } + } + + /// Set the length of the list child vector. + pub fn set_child_len(&mut self, new_len: usize) { + unsafe { + duckdb_list_vector_set_size(self.ptr, new_len as u64); + } + } +} + +impl VectorHandle { + pub fn get_child(&self) -> VectorHandle { + unsafe { VectorHandle::new_unchecked(duckdb_array_vector_get_child(self.ptr)) } + } +} + +impl VectorHandle { + pub fn get_child<'b>(&self, idx: usize) -> VectorHandle { + unsafe { VectorHandle::new_unchecked(duckdb_struct_vector_get_child(self.ptr, idx as idx_t)) } + } +} + +impl VectorHandle { + pub fn get_child<'b>(&self, idx: usize) -> VectorHandle { + unsafe { VectorHandle::new_unchecked(duckdb_struct_vector_get_child(self.ptr, idx as idx_t)) } + } +} + +/// A flat vector +#[derive(Constructor)] +pub struct FlatVector { + handle: VectorHandle, + /// get the size of elements for this vector. + /// there is no good way to get the length just using the flat vector handle, + /// hence we need to retrive it from higher above, either a list vector or directly a data chunk + get_capacity: Arc, +} + +impl FlatVector { + /// Returns the capacity of the vector + pub fn capacity(&self) -> usize { + (self.get_capacity)() + } + + /// Returns an unsafe mutable pointer to the vector’s + pub fn as_mut_ptr(&self) -> *mut T { + unsafe { duckdb_vector_get_data(self.handle.ptr.cast()).cast() } + } + + /// Returns a slice of the vector + pub fn as_slice(&self) -> &[T] { + unsafe { slice::from_raw_parts(self.as_mut_ptr(), self.capacity()) } + } + + /// Returns a mutable slice of the vector + pub fn as_mut_slice(&mut self) -> &mut [T] { + unsafe { slice::from_raw_parts_mut(self.as_mut_ptr(), self.capacity()) } + } + + /// Returns the logical type of the vector + pub fn logical_type(&self) -> LogicalTypeHandle { + LogicalTypeHandle::from(unsafe { duckdb_vector_get_column_type(self.handle.ptr) }) + } + + /// Set row as null + pub fn set_null(&mut self, row: usize) { + unsafe { + duckdb_vector_ensure_validity_writable(self.handle.ptr); + let idx = duckdb_vector_get_validity(self.handle.ptr); + duckdb_validity_set_row_invalid(idx, row as u64); + } + } + + /// Copy data to the vector. + pub fn copy(&mut self, data: &[T]) { + let capacity = self.capacity(); + assert!( + data.len() <= capacity, + "Provided data of length: {} exceeded capacity of {}", + data.len(), + capacity + ); + self.as_mut_slice::()[0..data.len()].copy_from_slice(data); + } +} + +/// A trait for inserting data into a vector. +pub trait Inserter { + /// Insert a value into the vector. + fn insert(&self, index: usize, value: T); +} + +impl Inserter for FlatVector { + fn insert(&self, index: usize, value: CString) { + unsafe { + duckdb_vector_assign_string_element(self.handle.ptr, index as u64, value.as_ptr()); + } + } +} + +impl Inserter<&str> for FlatVector { + fn insert(&self, index: usize, value: &str) { + let bytes = value.as_bytes(); + unsafe { + duckdb_vector_assign_string_element_len( + self.handle.ptr, + index as u64, + bytes.as_ptr() as *const c_char, + bytes.len() as u64, + ); + } + } +} + +impl Inserter<&[u8]> for FlatVector { + fn insert(&self, index: usize, value: &[u8]) { + let value_size = value.len(); + unsafe { + // This function also works for binary data. https://duckdb.org/docs/api/c/api#duckdb_vector_assign_string_element_len + duckdb_vector_assign_string_element_len( + self.handle.ptr, + index as u64, + value.as_ptr() as *const ::std::os::raw::c_char, + value_size as u64, + ); + } + } +} + +/// A list vector. +pub struct ListVector { + /// ListVector does not own the vector pointer. + pub entry_list: FlatVector, + pub child: Box, +} + +impl ListVector { + /// Returns true if the list vector is empty. + pub fn is_empty(&self) -> bool { + self.child_len() == 0 + } + + /// Set offset and length to the entry. + pub fn set_entry(&mut self, idx: usize, offset: usize, length: usize) { + self.entry_list.as_mut_slice::()[idx] = duckdb_list_entry { + offset: offset as u64, + length: length as u64, + } + } + + /// Returns the number of entries in the list child vector. + pub fn child_len(&self) -> usize { + self.entry_list.handle.as_list().get_size() + } + + /// Reserve the capacity for its child node. + pub fn reserve_child_capacity(&mut self, capacity: usize) { + self.entry_list.handle.as_list().reserve_child_capacity(capacity) + } + + /// Set the length of the list child vector. + pub fn set_child_len(&mut self, new_len: usize) { + self.entry_list.handle.as_list().set_child_len(new_len) + } +} + +pub struct ArrayVector { + pub child: Box, + array_size: usize, // number of columns +} + +impl ArrayVector { + pub fn array_size(&self) -> usize { + self.array_size + } +} + +/// A struct vector. +pub struct StructVector { + handle: VectorHandle, + /// ListVector does not own the vector pointer. + pub children: Vec, +} + +impl StructVector { + /// Get the logical type of this struct vector. + pub fn logical_type(&self) -> LogicalTypeHandle { + LogicalTypeHandle::from(unsafe { duckdb_vector_get_column_type(self.handle.ptr) }) + } + + /// Get the name of the child by idx. + pub fn child_name(&self, idx: usize) -> DuckDbStr { + let logical_type = self.logical_type(); + unsafe { DuckDbStr::from_ptr(duckdb_struct_type_child_name(logical_type.ptr, idx as u64)) } + } + + /// Get the number of children. + pub fn num_children(&self) -> usize { + let logical_type = self.logical_type(); + unsafe { duckdb_struct_type_child_count(logical_type.ptr) as usize } + } +} + +/// A union vector +pub struct UnionVector { + /// UnionVector does not own the vector pointer + handle: VectorHandle, + pub tags: FlatVector, + pub members: Vec, +} + +impl UnionVector { + /// Get the logical type of this struct vector. + pub fn logical_type(&self) -> LogicalTypeHandle { + LogicalTypeHandle::from(unsafe { duckdb_vector_get_column_type(self.handle.ptr) }) + } + + /// Retrieves the child type of the given union member at the specified index + /// Get the name of the union member. + pub fn member_name(&self, idx: usize) -> DuckDbStr { + let logical_type = self.logical_type(); + unsafe { DuckDbStr::from_ptr(duckdb_union_type_member_name(logical_type.ptr, idx as u64)) } + } +} + +#[derive(From)] +pub enum Vector { + Flat(FlatVector), + List(ListVector), + Array(ArrayVector), + Struct(StructVector), + Union(UnionVector), +} + +impl Vector { + pub fn try_as_flat_mut(&mut self) -> Result<&mut FlatVector, Box> { + match self { + Vector::Flat(x) => Ok(x), + _ => Err("Vector is not a FlatVector".into()), + } + } + + pub fn try_as_list_mut(&mut self) -> Result<&mut ListVector, Box> { + match self { + Vector::List(x) => Ok(x), + _ => Err("Vector is not a ListVector".into()), + } + } + + pub fn try_as_array_mut(&mut self) -> Result<&mut ArrayVector, Box> { + match self { + Vector::Array(x) => Ok(x), + _ => Err("Vector is not a ArrayVector".into()), + } + } + + pub fn try_as_struct_mut(&mut self) -> Result<&mut StructVector, Box> { + match self { + Vector::Struct(x) => Ok(x), + _ => Err("Vector is not a StructVector".into()), + } + } + + #[inline] + pub fn try_as_union_mut(&mut self) -> Result<&mut UnionVector, Box> { + match self { + Vector::Union(x) => Ok(x), + _ => Err("Vector is not a UnionVector".into()), + } + } + + fn new(column_info: &ColumnInfo, base_handle: VectorHandle, size: Arc) -> Self { + match column_info { + ColumnInfo::Flat => { + let vector = FlatVector::new(base_handle.cast(), size); + Vector::Flat(vector) + } + ColumnInfo::List(list) => { + let list_handle = base_handle.clone().as_list(); + let child_handle = list_handle.get_child(); + let entry_list = FlatVector::new(base_handle.as_flat(), size); + + let size_fn = Arc::new(move || list_handle.get_size()); + + let child = Box::new(Self::new(list.child.as_ref(), child_handle, size_fn)); + + ListVector { entry_list, child }.into() + } + ColumnInfo::Array(info) => { + let array_handle = base_handle.as_array(); + let child_handle = array_handle.get_child(); + let array_size = info.array_size; + let get_array_size = Arc::new(move || array_size * size()); + let child = Box::new(Self::new(info.child.as_ref(), child_handle, get_array_size)); + + let array_size = info.array_size; + + ArrayVector { child, array_size }.into() + } + ColumnInfo::Struct(info) => { + let struct_handle = base_handle.as_struct(); + let children = info + .children + .iter() + .enumerate() + .map(|(idx, child)| { + let child_handle = struct_handle.get_child(idx); + Self::new(child, child_handle, size.clone()) + }) + .collect(); + + StructVector { + handle: struct_handle, + children, + } + .into() + } + ColumnInfo::Union(info) => { + let union_handle = base_handle.as_union(); + // first child = tag, idx is 0 + let tag_vector = union_handle.get_child(0).as_flat(); + let tags = FlatVector::new(tag_vector, size.clone()); + + // members idx begin from 1 + let members = info + .members + .iter() + .enumerate() + .map(|(idx, child)| { + let child_handle = union_handle.get_child(idx + 1); + Self::new(child, child_handle.into(), size.clone()) + }) + .collect(); + + UnionVector { + handle: union_handle, + tags, + members, + } + .into() + } + } + } + + pub fn new_from_column_infos(columns: &[ColumnInfo], data_chunk_handle: Arc) -> Vec { + let get_size = { + let data_chunk_handle = data_chunk_handle.clone(); + Arc::new(move || data_chunk_handle.len()) + }; + (0..columns.len()) + .map(move |i| { + let column_info = &columns[i]; + let base_handle = data_chunk_handle.get_vector(i); + Self::new(column_info, base_handle, get_size.clone()) + }) + .collect() + } + + pub fn new_from_record_batch_schema<'b>(schema: &SchemaRef, data_chunk_handle: Arc) -> Vec { + let get_size = { + let data_chunk_handle = data_chunk_handle.clone(); + Arc::new(move || data_chunk_handle.len()) + }; + + schema + .fields + .iter() + .enumerate() + .map(|(i, field)| { + let data_type = field.data_type(); + let column_info = ColumnInfo::new_from_data_type(data_type); + let base_handle = data_chunk_handle.get_vector(i); + Self::new(&column_info, base_handle, get_size.clone()) + }) + .collect() + } +} diff --git a/crates/duckdb/src/error.rs b/crates/duckdb/src/error.rs index 8d07a133..1190d93a 100644 --- a/crates/duckdb/src/error.rs +++ b/crates/duckdb/src/error.rs @@ -1,11 +1,13 @@ +use std::ffi::CStr; +use std::path::PathBuf; +use std::{error, fmt, str}; + use arrow::datatypes::DataType; +use arrow::error::ArrowError; use super::Result; -use crate::{ - ffi, - types::{FromSqlError, Type}, -}; -use std::{error, ffi::CStr, fmt, path::PathBuf, str}; +use crate::ffi; +use crate::types::{FromSqlError, Type}; /// Enum listing possible errors from duckdb. #[derive(Debug)] @@ -63,6 +65,9 @@ pub enum Error { /// Error when datatype to duckdb type ArrowTypeToDuckdbType(String, DataType), + /// Error when datatype to duckdb type + ArrowError(ArrowError), + /// Error when a query that was expected to insert one row did not insert /// any or insert many. StatementChangedRows(usize), @@ -122,6 +127,13 @@ impl From<::std::ffi::NulError> for Error { } } +impl From for Error { + #[cold] + fn from(err: ArrowError) -> Error { + Error::ArrowError(err) + } +} + const UNKNOWN_COLUMN: usize = std::usize::MAX; /// The conversion isn't precise, but it's convenient to have it @@ -186,6 +198,7 @@ impl fmt::Display for Error { Error::InvalidQuery => write!(f, "Query is not read-only"), Error::MultipleStatement => write!(f, "Multiple statements provided"), Error::AppendError => write!(f, "Append error"), + Error::ArrowError(_) => write!(f, "Arrow Error"), } } } @@ -210,6 +223,7 @@ impl error::Error for Error { | Error::InvalidQuery | Error::AppendError | Error::ArrowTypeToDuckdbType(..) + | Error::ArrowError(_) | Error::MultipleStatement => None, Error::FromSqlConversionFailure(_, _, ref err) | Error::ToSqlConversionFailure(ref err) => Some(&**err), } @@ -225,12 +239,12 @@ fn error_from_duckdb_code(code: ffi::duckdb_state, message: Option) -> R #[cold] #[inline] -pub fn result_from_duckdb_appender(code: ffi::duckdb_state, appender: *mut ffi::duckdb_appender) -> Result<()> { +pub fn result_from_duckdb_appender(code: ffi::duckdb_state, appender: &mut ffi::duckdb_appender) -> Result<()> { if code == ffi::DuckDBSuccess { return Ok(()); } unsafe { - let message = if (*appender).is_null() { + let message = if appender.is_null() { Some("appender is null".to_string()) } else { let c_err = ffi::duckdb_appender_error(*appender); diff --git a/crates/duckdb/src/extension.rs b/crates/duckdb/src/extension.rs index 1a6a9690..8c6ee620 100644 --- a/crates/duckdb/src/extension.rs +++ b/crates/duckdb/src/extension.rs @@ -38,7 +38,11 @@ mod test { let db = Connection::open_in_memory()?; assert_eq!( 300f32, - db.query_row::(r#"SELECT SUM(value) FROM read_parquet('https://github.com/duckdb/duckdb-rs/raw/main/crates/duckdb/examples/int32_decimal.parquet');"#, [], |r| r.get(0))? + db.query_row::( + r#"SELECT SUM(value) FROM read_parquet('https://github.com/duckdb/duckdb-rs/raw/main/crates/duckdb/examples/int32_decimal.parquet');"#, + [], + |r| r.get(0) + )? ); Ok(()) } diff --git a/crates/duckdb/src/lib.rs b/crates/duckdb/src/lib.rs index d18f6daf..ab22f14b 100644 --- a/crates/duckdb/src/lib.rs +++ b/crates/duckdb/src/lib.rs @@ -93,6 +93,7 @@ pub use arrow; #[cfg(feature = "polars")] pub use polars::{self, export::arrow as arrow2}; +mod core; #[macro_use] mod error; mod appender; diff --git a/crates/duckdb/src/statement.rs b/crates/duckdb/src/statement.rs index a30c11c6..94f3ea82 100644 --- a/crates/duckdb/src/statement.rs +++ b/crates/duckdb/src/statement.rs @@ -1,15 +1,18 @@ -use std::{convert, ffi::c_void, fmt, mem, os::raw::c_char, ptr, str}; +use std::ffi::c_void; +use std::os::raw::c_char; +use std::sync::Arc; +use std::{convert, fmt, mem, ptr, str}; -use arrow::{array::StructArray, datatypes::SchemaRef}; +use arrow::array::StructArray; +use arrow::datatypes::{DataType, SchemaRef}; +use arrow_convert::deserialize::{ArrowArray, ArrowDeserialize}; use super::{ffi, AndThenRows, Connection, Error, MappedRows, Params, RawStatement, Result, Row, Rows, ValueRef}; +use crate::arrow_batch::Arrow; +use crate::error::result_from_duckdb_prepare; +use crate::types::{TimeUnit, ToSql, ToSqlOutput}; #[cfg(feature = "polars")] use crate::{arrow2, polars_dataframe::Polars}; -use crate::{ - arrow_batch::Arrow, - error::result_from_duckdb_prepare, - types::{TimeUnit, ToSql, ToSqlOutput}, -}; /// A prepared statement. pub struct Statement<'conn> { @@ -109,6 +112,44 @@ impl Statement<'_> { Ok(Arrow::new(self)) } + /// Execute the prepared statement, returning a deserialized vector of type `T` from the resulting arrow RecordBatch. + /// + /// # Example + /// + /// ```rust,no_run + /// # use duckdb::{Result, Connection}; + /// # use arrow::record_batch::RecordBatch; + /// + /// fn query_and_deserialize(conn: &Connection) -> Result> { + /// let result: Vec = conn + /// .prepare("SELECT num FROM test")? + /// .query_arrow_deserialized([])?; + /// Ok(result) + /// } + /// ``` + pub fn query_arrow_deserialized(&mut self, params: impl Params) -> Result> + where + T: ArrowDeserialize + 'static, + { + let arrow_iter = self.query_arrow(params)?; + let mut result = Vec::new(); + + for rb in arrow_iter { + let fields = rb.schema_ref().fields(); + + let struct_or_union_array = if fields.len() == 1 && matches!(fields[0].data_type(), DataType::Union(_, _)) { + rb.column(0).clone() + } else { + Arc::new(StructArray::from(rb)) + }; + let chunk = T::ArrayType::iter_from_array_ref(&struct_or_union_array) + .map(::arrow_deserialize_internal); + result.extend(chunk); + } + + Ok(result) + } + /// Execute the prepared statement, returning a handle to the resulting /// vector of polars DataFrame. /// @@ -555,7 +596,8 @@ impl Statement<'_> { #[cfg(test)] mod test { - use crate::{params_from_iter, types::ToSql, Connection, Error, Result}; + use crate::types::ToSql; + use crate::{params_from_iter, Connection, Error, Result}; #[test] fn test_execute() -> Result<()> { diff --git a/crates/duckdb/src/vtab/arrow.rs b/crates/duckdb/src/vtab/arrow.rs index 941c6ea9..962d254f 100644 --- a/crates/duckdb/src/vtab/arrow.rs +++ b/crates/duckdb/src/vtab/arrow.rs @@ -1,23 +1,18 @@ -use super::{ - vector::{ArrayVector, FlatVector, ListVector, Vector}, - BindInfo, DataChunk, Free, FunctionInfo, InitInfo, LogicalType, LogicalTypeId, StructVector, VTab, -}; use std::ptr::null_mut; -use crate::vtab::vector::Inserter; use arrow::array::{ as_boolean_array, as_generic_binary_array, as_large_list_array, as_list_array, as_primitive_array, as_string_array, - as_struct_array, Array, ArrayData, AsArray, BinaryArray, BooleanArray, Decimal128Array, FixedSizeListArray, - GenericListArray, OffsetSizeTrait, PrimitiveArray, StringArray, StructArray, -}; - -use arrow::{ - datatypes::*, - ffi::{from_ffi, FFI_ArrowArray, FFI_ArrowSchema}, - record_batch::RecordBatch, + as_struct_array, as_union_array, Array, ArrayData, ArrayRef, AsArray, BinaryArray, BooleanArray, Decimal128Array, + FixedSizeListArray, PrimitiveArray, StringArray, StructArray, }; +use arrow::datatypes::*; +use arrow::ffi::{from_ffi, FFI_ArrowArray, FFI_ArrowSchema}; +use arrow::record_batch::RecordBatch; +use num::cast::AsPrimitive; +use num::ToPrimitive; -use num::{cast::AsPrimitive, ToPrimitive}; +use crate::core::{DataChunk, DataChunkHandle, FlatVector, Inserter, LogicalTypeHandle, LogicalTypeId, Vector}; +use crate::vtab::{BindInfo, Free, FunctionInfo, InitInfo, VTab}; /// A pointer to the Arrow record batch for the table function. #[repr(C)] @@ -74,8 +69,10 @@ impl VTab for ArrowVTab { type BindData = ArrowBindData; type InitData = ArrowInitData; - unsafe fn bind(bind: &BindInfo, data: *mut ArrowBindData) -> Result<(), Box> { - (*data).rb = null_mut(); + fn bind(bind: &BindInfo, data: *mut ArrowBindData) -> Result<(), Box> { + unsafe { + (*data).rb = null_mut(); + } let param_count = bind.get_parameter_count(); if param_count != 2 { return Err(format!("Bad param count: {param_count}, expected 2").into()); @@ -95,14 +92,14 @@ impl VTab for ArrowVTab { Ok(()) } - unsafe fn init(_: &InitInfo, data: *mut ArrowInitData) -> Result<(), Box> { + fn init(_: &InitInfo, data: *mut ArrowInitData) -> Result<(), Box> { unsafe { (*data).done = false; } Ok(()) } - unsafe fn func(func: &FunctionInfo, output: &mut DataChunk) -> Result<(), Box> { + fn func(func: &FunctionInfo, output: DataChunkHandle) -> Result<(), Box> { let init_info = func.get_init_data::(); let bind_info = func.get_bind_data::(); unsafe { @@ -111,7 +108,9 @@ impl VTab for ArrowVTab { } else { let rb = Box::from_raw((*bind_info).rb); (*bind_info).rb = null_mut(); // erase ref in case of failure in record_batch_to_duckdb_data_chunk - record_batch_to_duckdb_data_chunk(&rb, output)?; + let mut chunk = + DataChunk::new_from_record_batch_handle(rb.schema_ref(), output.with_capacity(rb.num_rows())); + record_batch_to_duckdb_data_chunk(&rb, &mut chunk)?; (*bind_info).rb = Box::into_raw(rb); (*init_info).done = true; } @@ -119,10 +118,10 @@ impl VTab for ArrowVTab { Ok(()) } - fn parameters() -> Option> { + fn parameters() -> Option> { Some(vec![ - LogicalType::new(LogicalTypeId::UBigint), // file path - LogicalType::new(LogicalTypeId::UBigint), // sheet name + LogicalTypeHandle::from(LogicalTypeId::UBigint), // file path + LogicalTypeHandle::from(LogicalTypeId::UBigint), // sheet name ]) } } @@ -176,7 +175,7 @@ pub fn to_duckdb_type_id(data_type: &DataType) -> Result Result> { +pub fn to_duckdb_logical_type(data_type: &DataType) -> Result> { match data_type { DataType::Dictionary(_, value_type) => to_duckdb_logical_type(value_type), DataType::Struct(fields) => { @@ -184,23 +183,30 @@ pub fn to_duckdb_logical_type(data_type: &DataType) -> Result { + let mut shape = vec![]; + for (_, field) in fields.iter() { + shape.push((field.name().as_str(), to_duckdb_logical_type(field.data_type())?)); + } + Ok(LogicalTypeHandle::union_type(shape.as_slice())) } DataType::List(child) | DataType::LargeList(child) => { - Ok(LogicalType::list(&to_duckdb_logical_type(child.data_type())?)) + Ok(LogicalTypeHandle::list(&to_duckdb_logical_type(child.data_type())?)) } - DataType::FixedSizeList(child, array_size) => Ok(LogicalType::array( + DataType::FixedSizeList(child, array_size) => Ok(LogicalTypeHandle::array( &to_duckdb_logical_type(child.data_type())?, - *array_size as u64, + *array_size, )), DataType::Decimal128(width, scale) if *scale > 0 => { // DuckDB does not support negative decimal scales - Ok(LogicalType::decimal(*width, (*scale).try_into().unwrap())) + Ok(LogicalTypeHandle::decimal(*width, (*scale).try_into().unwrap())) } DataType::Boolean | DataType::Utf8 | DataType::LargeUtf8 | DataType::Binary | DataType::LargeBinary => { - Ok(LogicalType::new(to_duckdb_type_id(data_type)?)) + Ok(LogicalTypeHandle::from(to_duckdb_type_id(data_type)?)) } - dtype if dtype.is_primitive() => Ok(LogicalType::new(to_duckdb_type_id(data_type)?)), + dtype if dtype.is_primitive() => Ok(LogicalTypeHandle::from(to_duckdb_type_id(data_type)?)), _ => Err(format!( "Unsupported data type: {data_type}, please file an issue https://github.com/wangfenjin/duckdb-rs" ) @@ -208,12 +214,108 @@ pub fn to_duckdb_logical_type(data_type: &DataType) -> Result Result<(), Box> { + match input.data_type() { + dt if dt.is_primitive() || matches!(dt, DataType::Boolean) => { + primitive_array_to_vector(input, output.try_as_flat_mut()?) + } + DataType::Utf8 => { + let array = as_string_array(input.as_ref()); + let out: &mut FlatVector = output.try_as_flat_mut()?; + assert!(array.len() <= out.capacity()); + + string_array_to_vector(array, out); + Ok(()) + } + DataType::Binary => { + binary_array_to_vector(as_generic_binary_array(input.as_ref()), output.try_as_flat_mut()?); + Ok(()) + } + DataType::List(_) => { + let array = as_list_array(input.as_ref()); + let out = output.try_as_list_mut()?; + let value_array = array.values(); + for i in 0..array.len() { + let offset = array.value_offsets()[i]; + let length = array.value_length(i); + out.set_entry(i, offset.as_(), length.as_()); + } + + out.reserve_child_capacity(value_array.len()); + out.set_child_len(value_array.len()); + fill_duckdb_vector_from_arrow_vector(value_array, &mut out.child)?; + Ok(()) + } + DataType::LargeList(_) => { + let array = as_large_list_array(input.as_ref()); + let out = &mut output.try_as_list_mut()?; + let value_array = array.values(); + + for i in 0..array.len() { + let offset = array.value_offsets()[i]; + let length = array.value_length(i); + out.set_entry(i, offset.as_(), length.as_()); + } + + out.reserve_child_capacity(value_array.len()); + out.set_child_len(value_array.len()); + fill_duckdb_vector_from_arrow_vector(value_array, &mut out.child)?; + Ok(()) + } + DataType::FixedSizeList(_, _) => { + let array = as_fixed_size_list_array(input.as_ref()); + let out = &mut output.try_as_array_mut()?; + let value_array = array.values(); + fill_duckdb_vector_from_arrow_vector(value_array, &mut out.child)?; + Ok(()) + } + DataType::Struct(_) => { + let struct_array = as_struct_array(input.as_ref()); + let struct_vector = output.try_as_struct_mut()?; + for i in 0..struct_array.num_columns() { + fill_duckdb_vector_from_arrow_vector(struct_array.column(i), &mut struct_vector.children[i])?; + } + Ok(()) + } + DataType::Union(fields, mode) => { + assert_eq!( + mode, + &UnionMode::Sparse, + "duckdb only supports Sparse array for union types" + ); + + let union_array = as_union_array(input.as_ref()); + let union_vector = output.try_as_union_mut().unwrap(); + + // set the tag array + union_vector.tags.copy(union_array.type_ids()); + + // copy the members + for (i, (type_id, _)) in fields.iter().enumerate() { + let column = union_array.child(type_id); + fill_duckdb_vector_from_arrow_vector(column, &mut union_vector.members[i])?; + } + Ok(()) + } + df => { + unimplemented!( + "column {} is not supported yet, please file an issue https://github.com/wangfenjin/duckdb-rs", + df + ); + } + } +} + /// Converts a `RecordBatch` to a `DataChunk` in the DuckDB format. /// /// # Arguments /// /// * `batch` - A reference to the `RecordBatch` to be converted to a `DataChunk`. /// * `chunk` - A mutable reference to the `DataChunk` to store the converted data. +/// ``` pub fn record_batch_to_duckdb_data_chunk( batch: &RecordBatch, chunk: &mut DataChunk, @@ -222,38 +324,7 @@ pub fn record_batch_to_duckdb_data_chunk( assert_eq!(batch.num_columns(), chunk.num_columns()); for i in 0..batch.num_columns() { let col = batch.column(i); - match col.data_type() { - dt if dt.is_primitive() || matches!(dt, DataType::Boolean) => { - primitive_array_to_vector(col, &mut chunk.flat_vector(i))?; - } - DataType::Utf8 => { - string_array_to_vector(as_string_array(col.as_ref()), &mut chunk.flat_vector(i)); - } - DataType::Binary => { - binary_array_to_vector(as_generic_binary_array(col.as_ref()), &mut chunk.flat_vector(i)); - } - DataType::List(_) => { - list_array_to_vector(as_list_array(col.as_ref()), &mut chunk.list_vector(i))?; - } - DataType::LargeList(_) => { - list_array_to_vector(as_large_list_array(col.as_ref()), &mut chunk.list_vector(i))?; - } - DataType::FixedSizeList(_, _) => { - fixed_size_list_array_to_vector(as_fixed_size_list_array(col.as_ref()), &mut chunk.array_vector(i))?; - } - DataType::Struct(_) => { - let struct_array = as_struct_array(col.as_ref()); - let mut struct_vector = chunk.struct_vector(i); - struct_array_to_vector(struct_array, &mut struct_vector)?; - } - _ => { - return Err(format!( - "column {} is not supported yet, please file an issue https://github.com/wangfenjin/duckdb-rs", - batch.schema().field(i) - ) - .into()); - } - } + fill_duckdb_vector_from_arrow_vector(col, &mut chunk.columns[i])?; } chunk.set_len(batch.num_rows()); Ok(()) @@ -274,11 +345,11 @@ fn primitive_array_to_flat_vector(array: &PrimitiveArray< fn primitive_array_to_flat_vector_cast( data_type: DataType, array: &dyn Array, - out_vector: &mut dyn Vector, + out_vector: &mut FlatVector, ) { let array = arrow::compute::kernels::cast::cast(array, &data_type).unwrap(); - let out_vector: &mut FlatVector = out_vector.as_mut_any().downcast_mut().unwrap(); out_vector.copy::(array.as_primitive::().values()); + if let Some(nulls) = array.nulls() { for (i, null) in nulls.iter().enumerate() { if !null { @@ -288,77 +359,43 @@ fn primitive_array_to_flat_vector_cast( } } -fn primitive_array_to_vector(array: &dyn Array, out: &mut dyn Vector) -> Result<(), Box> { +fn primitive_array_to_vector(array: &dyn Array, out: &mut FlatVector) -> Result<(), Box> { match array.data_type() { DataType::Boolean => { - boolean_array_to_vector(as_boolean_array(array), out.as_mut_any().downcast_mut().unwrap()); + boolean_array_to_vector(as_boolean_array(array), out); } DataType::UInt8 => { - primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ); + primitive_array_to_flat_vector::(as_primitive_array(array), out); } DataType::UInt16 => { - primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ); + primitive_array_to_flat_vector::(as_primitive_array(array), out); } DataType::UInt32 => { - primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ); + primitive_array_to_flat_vector::(as_primitive_array(array), out); } DataType::UInt64 => { - primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ); + primitive_array_to_flat_vector::(as_primitive_array(array), out); } DataType::Int8 => { - primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ); + primitive_array_to_flat_vector::(as_primitive_array(array), out); } DataType::Int16 => { - primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ); + primitive_array_to_flat_vector::(as_primitive_array(array), out); } DataType::Int32 => { - primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ); + primitive_array_to_flat_vector::(as_primitive_array(array), out); } DataType::Int64 => { - primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ); + primitive_array_to_flat_vector::(as_primitive_array(array), out); } DataType::Float32 => { - primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ); + primitive_array_to_flat_vector::(as_primitive_array(array), out); } DataType::Float64 => { - primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ); + primitive_array_to_flat_vector::(as_primitive_array(array), out); } DataType::Decimal128(width, _) => { - decimal_array_to_vector( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - *width, - ); + decimal_array_to_vector(as_primitive_array(array), out, *width); } // DuckDB Only supports timetamp_tz in microsecond precision @@ -368,28 +405,19 @@ fn primitive_array_to_vector(array: &dyn Array, out: &mut dyn Vector) -> Result< out, ), DataType::Timestamp(unit, None) => match unit { - TimeUnit::Second => primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ), - TimeUnit::Millisecond => primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ), - TimeUnit::Microsecond => primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ), - TimeUnit::Nanosecond => primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ), + TimeUnit::Second => primitive_array_to_flat_vector::(as_primitive_array(array), out), + TimeUnit::Millisecond => { + primitive_array_to_flat_vector::(as_primitive_array(array), out) + } + TimeUnit::Microsecond => { + primitive_array_to_flat_vector::(as_primitive_array(array), out) + } + TimeUnit::Nanosecond => { + primitive_array_to_flat_vector::(as_primitive_array(array), out) + } }, DataType::Date32 => { - primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ); + primitive_array_to_flat_vector::(as_primitive_array(array), out); } DataType::Date64 => primitive_array_to_flat_vector_cast::(Date32Type::DATA_TYPE, array, out), DataType::Time32(_) => { @@ -472,106 +500,12 @@ fn binary_array_to_vector(array: &BinaryArray, out: &mut FlatVector) { } } -fn list_array_to_vector>( - array: &GenericListArray, - out: &mut ListVector, -) -> Result<(), Box> { - let value_array = array.values(); - let mut child = out.child(value_array.len()); - match value_array.data_type() { - dt if dt.is_primitive() => { - primitive_array_to_vector(value_array.as_ref(), &mut child)?; - } - DataType::Utf8 => { - string_array_to_vector(as_string_array(value_array.as_ref()), &mut child); - } - DataType::Binary => { - binary_array_to_vector(as_generic_binary_array(value_array.as_ref()), &mut child); - } - _ => { - return Err("Nested list is not supported yet.".into()); - } - } - - for i in 0..array.len() { - let offset = array.value_offsets()[i]; - let length = array.value_length(i); - out.set_entry(i, offset.as_(), length.as_()); - } - Ok(()) -} - -fn fixed_size_list_array_to_vector( - array: &FixedSizeListArray, - out: &mut ArrayVector, -) -> Result<(), Box> { - let value_array = array.values(); - let mut child = out.child(value_array.len()); - match value_array.data_type() { - dt if dt.is_primitive() => { - primitive_array_to_vector(value_array.as_ref(), &mut child)?; - } - DataType::Utf8 => { - string_array_to_vector(as_string_array(value_array.as_ref()), &mut child); - } - DataType::Binary => { - binary_array_to_vector(as_generic_binary_array(value_array.as_ref()), &mut child); - } - _ => { - return Err("Nested array is not supported yet.".into()); - } - } - - Ok(()) -} - /// Force downcast of an [`Array`], such as an [`ArrayRef`], to /// [`FixedSizeListArray`], panic'ing on failure. fn as_fixed_size_list_array(arr: &dyn Array) -> &FixedSizeListArray { arr.as_any().downcast_ref::().unwrap() } -fn struct_array_to_vector(array: &StructArray, out: &mut StructVector) -> Result<(), Box> { - for i in 0..array.num_columns() { - let column = array.column(i); - match column.data_type() { - dt if dt.is_primitive() || matches!(dt, DataType::Boolean) => { - primitive_array_to_vector(column, &mut out.child(i))?; - } - DataType::Utf8 => { - string_array_to_vector(as_string_array(column.as_ref()), &mut out.child(i)); - } - DataType::Binary => { - binary_array_to_vector(as_generic_binary_array(column.as_ref()), &mut out.child(i)); - } - DataType::List(_) => { - list_array_to_vector(as_list_array(column.as_ref()), &mut out.list_vector_child(i))?; - } - DataType::LargeList(_) => { - list_array_to_vector(as_large_list_array(column.as_ref()), &mut out.list_vector_child(i))?; - } - DataType::FixedSizeList(_, _) => { - fixed_size_list_array_to_vector( - as_fixed_size_list_array(column.as_ref()), - &mut out.array_vector_child(i), - )?; - } - DataType::Struct(_) => { - let struct_array = as_struct_array(column.as_ref()); - let mut struct_vector = out.struct_vector_child(i); - struct_array_to_vector(struct_array, &mut struct_vector)?; - } - _ => { - unimplemented!( - "Unsupported data type: {}, please file an issue https://github.com/wangfenjin/duckdb-rs", - column.data_type() - ); - } - } - } - Ok(()) -} - /// Pass RecordBatch to duckdb. /// /// # Safety @@ -607,20 +541,22 @@ pub fn arrow_ffi_to_query_params(array: FFI_ArrowArray, schema: FFI_ArrowSchema) #[cfg(test)] mod test { + use std::error::Error; + use std::sync::Arc; + + use arrow::array::{ + Array, ArrayRef, AsArray, BinaryArray, Date32Array, Date64Array, Decimal128Array, Decimal256Array, + FixedSizeListArray, GenericListArray, Int32Array, ListArray, OffsetSizeTrait, PrimitiveArray, StringArray, + Time32SecondArray, Time64MicrosecondArray, TimestampMicrosecondArray, TimestampMillisecondArray, + TimestampNanosecondArray, TimestampSecondArray, + }; + use arrow::buffer::{OffsetBuffer, ScalarBuffer}; + use arrow::datatypes::{i256, ArrowPrimitiveType, DataType, Field, Schema}; + use arrow::record_batch::RecordBatch; + use arrow_convert::{ArrowDeserialize, ArrowField, ArrowSerialize}; + use super::{arrow_recordbatch_to_query_params, ArrowVTab}; use crate::{Connection, Result}; - use arrow::{ - array::{ - Array, ArrayRef, AsArray, BinaryArray, Date32Array, Date64Array, Decimal128Array, Decimal256Array, - FixedSizeListArray, GenericListArray, Int32Array, ListArray, OffsetSizeTrait, PrimitiveArray, StringArray, - StructArray, Time32SecondArray, Time64MicrosecondArray, TimestampMicrosecondArray, - TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray, - }, - buffer::{OffsetBuffer, ScalarBuffer}, - datatypes::{i256, ArrowPrimitiveType, DataType, Field, Fields, Schema}, - record_batch::RecordBatch, - }; - use std::{error::Error, sync::Arc}; #[test] fn test_vtab_arrow() -> Result<(), Box> { @@ -667,35 +603,31 @@ mod test { fn test_append_struct() -> Result<(), Box> { let db = Connection::open_in_memory()?; db.execute_batch("CREATE TABLE t1 (s STRUCT(v VARCHAR, i INTEGER))")?; + + #[derive(ArrowField, ArrowSerialize)] + struct T1Row { + v: String, + i: i32, + } + { - let struct_array = StructArray::from(vec![ - ( - Arc::new(Field::new("v", DataType::Utf8, true)), - Arc::new(StringArray::from(vec![Some("foo"), Some("bar")])) as ArrayRef, - ), - ( - Arc::new(Field::new("i", DataType::Int32, true)), - Arc::new(Int32Array::from(vec![Some(1), Some(2)])) as ArrayRef, - ), - ]); - - let schema = Schema::new(vec![Field::new( - "s", - DataType::Struct(Fields::from(vec![ - Field::new("v", DataType::Utf8, true), - Field::new("i", DataType::Int32, true), - ])), - true, - )]); - - let record_batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(struct_array)])?; + let rows = vec![ + T1Row { + v: "foo".to_string(), + i: 1, + }, + T1Row { + v: "bar".to_string(), + i: 2, + }, + ]; let mut app = db.appender("t1")?; - app.append_record_batch(record_batch)?; + app.append_rows_arrow(&rows, false)?; } + let mut stmt = db.prepare("SELECT s FROM t1")?; let rbs: Vec = stmt.query_arrow([])?.collect(); assert_eq!(rbs.iter().map(|op| op.num_rows()).sum::(), 2); - Ok(()) } @@ -846,6 +778,90 @@ mod test { Ok(()) } + #[test] + fn test_append_union() -> Result<(), Box> { + let db = Connection::open_in_memory()?; + + db.execute_batch("CREATE TABLE tbl1 (u UNION(num INT, str VARCHAR));")?; + + #[derive(PartialEq, Eq, Debug, ArrowField, ArrowSerialize, ArrowDeserialize)] + #[arrow_field(type = "sparse")] + enum TestUnion { + Num(i32), + Str(String), + } + + { + let rows = vec![TestUnion::Num(1), TestUnion::Str("foo".into()), TestUnion::Num(34)]; + + let mut app = db.appender("tbl1")?; + app.append_rows_arrow(&rows, false)?; + } + let mut stmt = db.prepare("SELECT u FROM tbl1")?; + let records: Vec = stmt.query_arrow_deserialized([])?; + assert_eq!( + records, + vec![TestUnion::Num(1), TestUnion::Str("foo".into()), TestUnion::Num(34)] + ); + Ok(()) + } + + #[test] + fn test_nested_struct_in_fixed_list() -> Result<(), Box> { + let db = Connection::open_in_memory()?; + + db.execute_batch( + r#" + CREATE TYPE POINT AS STRUCT(x INT, y INT); + CREATE TABLE tbl1 (line POINT[2]); + INSERT INTO tbl1 VALUES (ARRAY[{x: 1, y: 2}, {x: 3, y: 4}]); + "#, + )?; + + let mut stmt = db.prepare("SELECT line FROM tbl1")?; + let rbs: Vec = stmt.query_arrow([])?.collect(); + assert_eq!(rbs.iter().map(|op| op.num_rows()).sum::(), 1); + Ok(()) + } + + #[test] + fn test_list_column() -> Result<(), Box> { + let db = Connection::open_in_memory()?; + + db.execute_batch("CREATE TABLE tbl1 (scalar Int[])")?; + + { + let mut appender = db.appender("tbl1")?; + appender + .append_rows_arrow(&vec![vec![4, 5], vec![6, 7], vec![8, 9, 10]], false) + .unwrap(); + } + + let mut stmt = db.prepare("SELECT * FROM tbl1")?; + let rbs: Vec = stmt.query_arrow([])?.collect(); + assert_eq!(rbs.iter().map(|op| op.num_rows()).sum::(), 3); + Ok(()) + } + + #[test] + fn test_array_column() -> Result<(), Box> { + let db = Connection::open_in_memory()?; + + db.execute_batch("CREATE TABLE tbl1 (scalar CHAR[2])")?; + + { + let mut appender = db.appender("tbl1")?; + appender + .append_rows_arrow(&vec![[4, 5], [6, 7], [8, 9]], false) + .unwrap(); + } + + let mut stmt = db.prepare("SELECT * FROM tbl1")?; + let rbs: Vec = stmt.query_arrow([])?.collect(); + assert_eq!(rbs.iter().map(|op| op.num_rows()).sum::(), 3); + Ok(()) + } + #[test] fn test_primitive_roundtrip_contains_nulls() -> Result<(), Box> { let mut builder = arrow::array::PrimitiveBuilder::::new(); diff --git a/crates/duckdb/src/vtab/data_chunk.rs b/crates/duckdb/src/vtab/data_chunk.rs deleted file mode 100644 index 3bc6d874..00000000 --- a/crates/duckdb/src/vtab/data_chunk.rs +++ /dev/null @@ -1,133 +0,0 @@ -use super::{ - logical_type::LogicalType, - vector::{ArrayVector, FlatVector, ListVector, StructVector}, -}; -use crate::ffi::{ - duckdb_create_data_chunk, duckdb_data_chunk, duckdb_data_chunk_get_column_count, duckdb_data_chunk_get_size, - duckdb_data_chunk_get_vector, duckdb_data_chunk_set_size, duckdb_destroy_data_chunk, -}; - -/// DataChunk in DuckDB. -pub struct DataChunk { - /// Pointer to the DataChunk in duckdb C API. - ptr: duckdb_data_chunk, - - /// Whether this [DataChunk] own the [DataChunk::ptr]. - owned: bool, -} - -impl DataChunk { - /// Create a new [DataChunk] with the given [LogicalType]s. - pub fn new(logical_types: &[LogicalType]) -> Self { - let num_columns = logical_types.len(); - let mut c_types = logical_types.iter().map(|t| t.ptr).collect::>(); - let ptr = unsafe { duckdb_create_data_chunk(c_types.as_mut_ptr(), num_columns as u64) }; - DataChunk { ptr, owned: true } - } - - /// Get the vector at the specific column index: `idx`. - pub fn flat_vector(&self, idx: usize) -> FlatVector { - FlatVector::from(unsafe { duckdb_data_chunk_get_vector(self.ptr, idx as u64) }) - } - - /// Get a list vector from the column index. - pub fn list_vector(&self, idx: usize) -> ListVector { - ListVector::from(unsafe { duckdb_data_chunk_get_vector(self.ptr, idx as u64) }) - } - - /// Get a array vector from the column index. - pub fn array_vector(&self, idx: usize) -> ArrayVector { - ArrayVector::from(unsafe { duckdb_data_chunk_get_vector(self.ptr, idx as u64) }) - } - - /// Get struct vector at the column index: `idx`. - pub fn struct_vector(&self, idx: usize) -> StructVector { - StructVector::from(unsafe { duckdb_data_chunk_get_vector(self.ptr, idx as u64) }) - } - - /// Set the size of the data chunk - pub fn set_len(&self, new_len: usize) { - unsafe { duckdb_data_chunk_set_size(self.ptr, new_len as u64) }; - } - - /// Get the length / the number of rows in this [DataChunk]. - pub fn len(&self) -> usize { - unsafe { duckdb_data_chunk_get_size(self.ptr) as usize } - } - - /// Check whether this [DataChunk] is empty. - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Get the number of columns in this [DataChunk]. - pub fn num_columns(&self) -> usize { - unsafe { duckdb_data_chunk_get_column_count(self.ptr) as usize } - } - - /// Get the ptr of duckdb_data_chunk in this [DataChunk]. - pub fn get_ptr(&self) -> duckdb_data_chunk { - self.ptr - } -} - -impl From for DataChunk { - fn from(ptr: duckdb_data_chunk) -> Self { - Self { ptr, owned: false } - } -} - -impl Drop for DataChunk { - fn drop(&mut self) { - if self.owned && !self.ptr.is_null() { - unsafe { duckdb_destroy_data_chunk(&mut self.ptr) } - self.ptr = std::ptr::null_mut(); - } - } -} - -#[cfg(test)] -mod test { - use super::{super::logical_type::LogicalTypeId, *}; - - #[test] - fn test_data_chunk_construction() { - let dc = DataChunk::new(&[LogicalType::new(LogicalTypeId::Integer)]); - - assert_eq!(dc.num_columns(), 1); - - drop(dc); - } - - #[test] - fn test_vector() { - let datachunk = DataChunk::new(&[LogicalType::new(LogicalTypeId::Bigint)]); - let mut vector = datachunk.flat_vector(0); - let data = vector.as_mut_slice::(); - - data[0] = 42; - } - - #[test] - fn test_logi() { - let key = LogicalType::new(LogicalTypeId::Varchar); - - let value = LogicalType::new(LogicalTypeId::UTinyint); - - let map = LogicalType::map(&key, &value); - - assert_eq!(map.id(), LogicalTypeId::Map); - - // let union_ = LogicalType::new_union_type(HashMap::from([ - // ("number", LogicalType::new(LogicalTypeId::Bigint)), - // ("string", LogicalType::new(LogicalTypeId::Varchar)), - // ])); - // assert_eq!(union_.type_id(), LogicalTypeId::Union); - - // let struct_ = LogicalType::new_struct_type(HashMap::from([ - // ("number", LogicalType::new(LogicalTypeId::Bigint)), - // ("string", LogicalType::new(LogicalTypeId::Varchar)), - // ])); - // assert_eq!(struct_.type_id(), LogicalTypeId::Struct); - } -} diff --git a/crates/duckdb/src/vtab/excel.rs b/crates/duckdb/src/vtab/excel.rs index 355d9d30..12b0c8b6 100644 --- a/crates/duckdb/src/vtab/excel.rs +++ b/crates/duckdb/src/vtab/excel.rs @@ -1,4 +1,4 @@ -use super::{BindInfo, DataChunk, Free, FunctionInfo, InitInfo, LogicalType, LogicalTypeId, VTab}; +use super::{BindInfo, DataChunk, Free, FunctionInfo, InitInfo, LogicalTypeHandle, LogicalTypeId, VTab}; use crate::vtab::vector::Inserter; use calamine::{open_workbook_auto, DataType, Range, Reader}; @@ -33,7 +33,7 @@ impl VTab for ExcelVTab { type BindData = ExcelBindData; type InitData = ExcelInitData; - unsafe fn bind(bind: &BindInfo, data: *mut ExcelBindData) -> Result<(), Box> { + fn bind(bind: &BindInfo, data: *mut ExcelBindData) -> Result<(), Box> { let param_count = bind.get_parameter_count(); assert!(param_count == 2); let path = bind.get_parameter(0).to_string(); @@ -74,7 +74,7 @@ impl VTab for ExcelVTab { header[idx] .get_string() .unwrap_or_else(|| panic!("idx {} header empty?", idx)), - LogicalType::new(LogicalTypeId::Varchar), + LogicalTypeHandle::from(LogicalTypeId::Varchar), ); } DataType::Float(_) => { @@ -82,7 +82,7 @@ impl VTab for ExcelVTab { header[idx] .get_string() .unwrap_or_else(|| panic!("idx {} header empty?", idx)), - LogicalType::new(LogicalTypeId::Double), + LogicalTypeHandle::from(LogicalTypeId::Double), ); } DataType::Int(_) => { @@ -90,7 +90,7 @@ impl VTab for ExcelVTab { header[idx] .get_string() .unwrap_or_else(|| panic!("idx {} header empty?", idx)), - LogicalType::new(LogicalTypeId::Bigint), + LogicalTypeHandle::from(LogicalTypeId::Bigint), ); } DataType::Bool(_) => { @@ -98,7 +98,7 @@ impl VTab for ExcelVTab { header[idx] .get_string() .unwrap_or_else(|| panic!("idx {} header empty?", idx)), - LogicalType::new(LogicalTypeId::Boolean), + LogicalTypeHandle::from(LogicalTypeId::Boolean), ); } DataType::DateTime(_) => { @@ -106,7 +106,7 @@ impl VTab for ExcelVTab { header[idx] .get_string() .unwrap_or_else(|| panic!("idx {} header empty?", idx)), - LogicalType::new(LogicalTypeId::Date), + LogicalTypeHandle::from(LogicalTypeId::Date), ); } _ => { @@ -125,14 +125,14 @@ impl VTab for ExcelVTab { Ok(()) } - unsafe fn init(_: &InitInfo, data: *mut ExcelInitData) -> Result<(), Box> { + fn init(_: &InitInfo, data: *mut ExcelInitData) -> Result<(), Box> { unsafe { (*data).start = 1; } Ok(()) } - unsafe fn func(func: &FunctionInfo, output: &mut DataChunk) -> Result<(), Box> { + fn func(func: &FunctionInfo, output: &mut DataChunk) -> Result<(), Box> { let init_info = func.get_init_data::(); let bind_info = func.get_bind_data::(); unsafe { @@ -180,10 +180,10 @@ impl VTab for ExcelVTab { Ok(()) } - fn parameters() -> Option> { + fn parameters() -> Option> { Some(vec![ - LogicalType::new(LogicalTypeId::Varchar), // file path - LogicalType::new(LogicalTypeId::Varchar), // sheet name + LogicalTypeHandle::from(LogicalTypeId::Varchar), // file path + LogicalTypeHandle::from(LogicalTypeId::Varchar), // sheet name ]) } } @@ -203,7 +203,8 @@ mod test { .prepare("select count(*) from excel('./examples/Movies_Social_metadata.xlsx', 'Data')")? .query_row::([], |row| row.get(0))?; assert_eq!(3039, val); - let mut stmt = db.prepare("select genres, sum(movie_facebook_likes) from excel('./examples/Movies_Social_metadata.xlsx', 'Data') group by genres order by genres limit 4")?; + let mut stmt = + db.prepare("select genres, sum(movie_facebook_likes) from excel('./examples/Movies_Social_metadata.xlsx', 'Data') group by genres order by genres limit 4")?; // +-------------+---------------------------+ // | genres | sum(movie_facebook_likes) | // +-------------+---------------------------+ diff --git a/crates/duckdb/src/vtab/function.rs b/crates/duckdb/src/vtab/function.rs index e3131d4c..7e62f9a0 100644 --- a/crates/duckdb/src/vtab/function.rs +++ b/crates/duckdb/src/vtab/function.rs @@ -9,7 +9,7 @@ use super::{ duckdb_table_function_set_init, duckdb_table_function_set_local_init, duckdb_table_function_set_name, duckdb_table_function_supports_projection_pushdown, idx_t, }, - LogicalType, Value, + LogicalTypeHandle, Value, }; use std::{ ffi::{c_void, CString}, @@ -28,7 +28,7 @@ impl BindInfo { /// # Arguments /// * `name`: The name of the column /// * `type`: The logical type of the column - pub fn add_result_column(&self, column_name: &str, column_type: LogicalType) { + pub fn add_result_column(&self, column_name: &str, column_type: LogicalTypeHandle) { let c_str = CString::new(column_name).unwrap(); unsafe { duckdb_bind_add_result_column(self.ptr, c_str.as_ptr() as *const c_char, column_type.ptr); @@ -226,7 +226,7 @@ impl TableFunction { /// /// # Arguments /// * `logical_type`: The type of the parameter to add. - pub fn add_parameter(&self, logical_type: &LogicalType) -> &Self { + pub fn add_parameter(&self, logical_type: &LogicalTypeHandle) -> &Self { unsafe { duckdb_table_function_add_parameter(self.ptr, logical_type.ptr); } @@ -238,7 +238,7 @@ impl TableFunction { /// # Arguments /// * `name`: The name of the parameter to add. /// * `logical_type`: The type of the parameter to add. - pub fn add_named_parameter(&self, name: &str, logical_type: &LogicalType) -> &Self { + pub fn add_named_parameter(&self, name: &str, logical_type: &LogicalTypeHandle) -> &Self { unsafe { let string = CString::new(name).unwrap(); duckdb_table_function_add_named_parameter(self.ptr, string.as_ptr(), logical_type.ptr); diff --git a/crates/duckdb/src/vtab/logical_type.rs b/crates/duckdb/src/vtab/logical_type.rs deleted file mode 100644 index 1ee2543a..00000000 --- a/crates/duckdb/src/vtab/logical_type.rs +++ /dev/null @@ -1,341 +0,0 @@ -use std::{ - ffi::{c_char, CString}, - fmt::Debug, -}; - -use crate::ffi::*; - -/// Logical Type Id -/// -#[repr(u32)] -#[derive(Debug, PartialEq, Eq)] -pub enum LogicalTypeId { - /// Boolean - Boolean = DUCKDB_TYPE_DUCKDB_TYPE_BOOLEAN, - /// Tinyint - Tinyint = DUCKDB_TYPE_DUCKDB_TYPE_TINYINT, - /// Smallint - Smallint = DUCKDB_TYPE_DUCKDB_TYPE_SMALLINT, - /// Integer - Integer = DUCKDB_TYPE_DUCKDB_TYPE_INTEGER, - /// Bigint - Bigint = DUCKDB_TYPE_DUCKDB_TYPE_BIGINT, - /// Unsigned Tinyint - UTinyint = DUCKDB_TYPE_DUCKDB_TYPE_UTINYINT, - /// Unsigned Smallint - USmallint = DUCKDB_TYPE_DUCKDB_TYPE_USMALLINT, - /// Unsigned Integer - UInteger = DUCKDB_TYPE_DUCKDB_TYPE_UINTEGER, - /// Unsigned Bigint - UBigint = DUCKDB_TYPE_DUCKDB_TYPE_UBIGINT, - /// Float - Float = DUCKDB_TYPE_DUCKDB_TYPE_FLOAT, - /// Double - Double = DUCKDB_TYPE_DUCKDB_TYPE_DOUBLE, - /// Timestamp - Timestamp = DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP, - /// Date - Date = DUCKDB_TYPE_DUCKDB_TYPE_DATE, - /// Time - Time = DUCKDB_TYPE_DUCKDB_TYPE_TIME, - /// Interval - Interval = DUCKDB_TYPE_DUCKDB_TYPE_INTERVAL, - /// Hugeint - Hugeint = DUCKDB_TYPE_DUCKDB_TYPE_HUGEINT, - /// Varchar - Varchar = DUCKDB_TYPE_DUCKDB_TYPE_VARCHAR, - /// Blob - Blob = DUCKDB_TYPE_DUCKDB_TYPE_BLOB, - /// Decimal - Decimal = DUCKDB_TYPE_DUCKDB_TYPE_DECIMAL, - /// Timestamp S - TimestampS = DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_S, - /// Timestamp MS - TimestampMs = DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_MS, - /// Timestamp NS - TimestampNs = DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_NS, - /// Enum - Enum = DUCKDB_TYPE_DUCKDB_TYPE_ENUM, - /// List - List = DUCKDB_TYPE_DUCKDB_TYPE_LIST, - /// Struct - Struct = DUCKDB_TYPE_DUCKDB_TYPE_STRUCT, - /// Map - Map = DUCKDB_TYPE_DUCKDB_TYPE_MAP, - /// Uuid - Uuid = DUCKDB_TYPE_DUCKDB_TYPE_UUID, - /// Union - Union = DUCKDB_TYPE_DUCKDB_TYPE_UNION, - /// Timestamp TZ - TimestampTZ = DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_TZ, -} - -impl From for LogicalTypeId { - /// Convert from u32 to LogicalTypeId - fn from(value: u32) -> Self { - match value { - DUCKDB_TYPE_DUCKDB_TYPE_BOOLEAN => Self::Boolean, - DUCKDB_TYPE_DUCKDB_TYPE_TINYINT => Self::Tinyint, - DUCKDB_TYPE_DUCKDB_TYPE_SMALLINT => Self::Smallint, - DUCKDB_TYPE_DUCKDB_TYPE_INTEGER => Self::Integer, - DUCKDB_TYPE_DUCKDB_TYPE_BIGINT => Self::Bigint, - DUCKDB_TYPE_DUCKDB_TYPE_UTINYINT => Self::UTinyint, - DUCKDB_TYPE_DUCKDB_TYPE_USMALLINT => Self::USmallint, - DUCKDB_TYPE_DUCKDB_TYPE_UINTEGER => Self::UInteger, - DUCKDB_TYPE_DUCKDB_TYPE_UBIGINT => Self::UBigint, - DUCKDB_TYPE_DUCKDB_TYPE_FLOAT => Self::Float, - DUCKDB_TYPE_DUCKDB_TYPE_DOUBLE => Self::Double, - DUCKDB_TYPE_DUCKDB_TYPE_VARCHAR => Self::Varchar, - DUCKDB_TYPE_DUCKDB_TYPE_BLOB => Self::Blob, - DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP => Self::Timestamp, - DUCKDB_TYPE_DUCKDB_TYPE_DATE => Self::Date, - DUCKDB_TYPE_DUCKDB_TYPE_TIME => Self::Time, - DUCKDB_TYPE_DUCKDB_TYPE_INTERVAL => Self::Interval, - DUCKDB_TYPE_DUCKDB_TYPE_HUGEINT => Self::Hugeint, - DUCKDB_TYPE_DUCKDB_TYPE_DECIMAL => Self::Decimal, - DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_S => Self::TimestampS, - DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_MS => Self::TimestampMs, - DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_NS => Self::TimestampNs, - DUCKDB_TYPE_DUCKDB_TYPE_ENUM => Self::Enum, - DUCKDB_TYPE_DUCKDB_TYPE_LIST => Self::List, - DUCKDB_TYPE_DUCKDB_TYPE_STRUCT => Self::Struct, - DUCKDB_TYPE_DUCKDB_TYPE_MAP => Self::Map, - DUCKDB_TYPE_DUCKDB_TYPE_UUID => Self::Uuid, - DUCKDB_TYPE_DUCKDB_TYPE_UNION => Self::Union, - DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_TZ => Self::TimestampTZ, - _ => panic!(), - } - } -} - -/// DuckDB Logical Type. -/// -pub struct LogicalType { - pub(crate) ptr: duckdb_logical_type, -} - -impl Debug for LogicalType { - /// Debug implementation for LogicalType - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { - let id = self.id(); - match id { - LogicalTypeId::Struct => { - write!(f, "struct<")?; - for i in 0..self.num_children() { - if i > 0 { - write!(f, ", ")?; - } - write!(f, "{}: {:?}", self.child_name(i), self.child(i))?; - } - write!(f, ">") - } - _ => write!(f, "{:?}", self.id()), - } - } -} - -impl Drop for LogicalType { - /// Drop implementation for LogicalType - fn drop(&mut self) { - if !self.ptr.is_null() { - unsafe { - duckdb_destroy_logical_type(&mut self.ptr); - } - } - - self.ptr = std::ptr::null_mut(); - } -} - -impl From for LogicalType { - /// Wrap a DuckDB logical type from C API - fn from(ptr: duckdb_logical_type) -> Self { - Self { ptr } - } -} - -impl LogicalType { - /// Create a new [LogicalType] from [LogicalTypeId] - pub fn new(id: LogicalTypeId) -> Self { - unsafe { - Self { - ptr: duckdb_create_logical_type(id as u32), - } - } - } - - /// Creates a map type from its child type. - pub fn map(key: &LogicalType, value: &LogicalType) -> Self { - unsafe { - Self { - ptr: duckdb_create_map_type(key.ptr, value.ptr), - } - } - } - - /// Creates a list type from its child type. - pub fn list(child_type: &LogicalType) -> Self { - unsafe { - Self { - ptr: duckdb_create_list_type(child_type.ptr), - } - } - } - - /// Creates an array type from its child type. - pub fn array(child_type: &LogicalType, array_size: u64) -> Self { - unsafe { - Self { - ptr: duckdb_create_array_type(child_type.ptr, array_size), - } - } - } - - /// Creates a decimal type from its `width` and `scale`. - pub fn decimal(width: u8, scale: u8) -> Self { - unsafe { - Self { - ptr: duckdb_create_decimal_type(width, scale), - } - } - } - - /// Retrieves the decimal width - /// Returns 0 if the LogicalType is not a decimal - pub fn decimal_width(&self) -> u8 { - unsafe { duckdb_decimal_width(self.ptr) } - } - - /// Retrieves the decimal scale - /// Returns 0 if the LogicalType is not a decimal - pub fn decimal_scale(&self) -> u8 { - unsafe { duckdb_decimal_scale(self.ptr) } - } - - /// Make a `LogicalType` for `struct` - pub fn struct_type(fields: &[(&str, LogicalType)]) -> Self { - let keys: Vec = fields.iter().map(|f| CString::new(f.0).unwrap()).collect(); - let values: Vec = fields.iter().map(|it| it.1.ptr).collect(); - let name_ptrs = keys.iter().map(|it| it.as_ptr()).collect::>(); - - unsafe { - Self { - ptr: duckdb_create_struct_type( - values.as_slice().as_ptr().cast_mut(), - name_ptrs.as_slice().as_ptr().cast_mut(), - fields.len() as idx_t, - ), - } - } - } - - /// Make a `LogicalType` for `union` - pub fn union_type(fields: &[(&str, LogicalType)]) -> Self { - let keys: Vec = fields.iter().map(|f| CString::new(f.0).unwrap()).collect(); - let values: Vec = fields.iter().map(|it| it.1.ptr).collect(); - let name_ptrs = keys.iter().map(|it| it.as_ptr()).collect::>(); - - unsafe { - Self { - ptr: duckdb_create_union_type( - values.as_slice().as_ptr().cast_mut(), - name_ptrs.as_slice().as_ptr().cast_mut(), - fields.len() as idx_t, - ), - } - } - } - - /// Logical type ID - pub fn id(&self) -> LogicalTypeId { - let duckdb_type_id = unsafe { duckdb_get_type_id(self.ptr) }; - duckdb_type_id.into() - } - - /// Logical type children num - pub fn num_children(&self) -> usize { - match self.id() { - LogicalTypeId::Struct => unsafe { duckdb_struct_type_child_count(self.ptr) as usize }, - LogicalTypeId::Union => unsafe { duckdb_union_type_member_count(self.ptr) as usize }, - LogicalTypeId::List => 1, - _ => 0, - } - } - - /// Logical type child name by idx - /// - /// Panics if the logical type is not a struct or union - pub fn child_name(&self, idx: usize) -> String { - unsafe { - let child_name_ptr = match self.id() { - LogicalTypeId::Struct => duckdb_struct_type_child_name(self.ptr, idx as u64), - LogicalTypeId::Union => duckdb_union_type_member_name(self.ptr, idx as u64), - _ => panic!("not a struct or union"), - }; - let c_str = CString::from_raw(child_name_ptr); - let name = c_str.to_str().unwrap(); - name.to_string() - } - } - - /// Logical type child by idx - pub fn child(&self, idx: usize) -> Self { - let c_logical_type = unsafe { - match self.id() { - LogicalTypeId::Struct => duckdb_struct_type_child_type(self.ptr, idx as u64), - LogicalTypeId::Union => duckdb_union_type_member_type(self.ptr, idx as u64), - _ => panic!("not a struct or union"), - } - }; - Self::from(c_logical_type) - } -} - -#[cfg(test)] -mod test { - use super::{LogicalType, LogicalTypeId}; - - #[test] - fn test_struct() { - let fields = &[("hello", LogicalType::new(crate::vtab::LogicalTypeId::Boolean))]; - let typ = LogicalType::struct_type(fields); - - assert_eq!(typ.num_children(), 1); - assert_eq!(typ.child_name(0), "hello"); - assert_eq!(typ.child(0).id(), crate::vtab::LogicalTypeId::Boolean); - } - - #[test] - fn test_decimal() { - let typ = LogicalType::decimal(10, 2); - - assert_eq!(typ.id(), crate::vtab::LogicalTypeId::Decimal); - assert_eq!(typ.decimal_width(), 10); - assert_eq!(typ.decimal_scale(), 2); - } - - #[test] - fn test_decimal_methods() { - let typ = LogicalType::new(crate::vtab::LogicalTypeId::Varchar); - - assert_eq!(typ.decimal_width(), 0); - assert_eq!(typ.decimal_scale(), 0); - } - - #[test] - fn test_union_type() { - let fields = &[ - ("hello", LogicalType::new(LogicalTypeId::Boolean)), - ("world", LogicalType::new(LogicalTypeId::Integer)), - ]; - let typ = LogicalType::union_type(fields); - - assert_eq!(typ.num_children(), 2); - - assert_eq!(typ.child_name(0), "hello"); - assert_eq!(typ.child(0).id(), LogicalTypeId::Boolean); - - assert_eq!(typ.child_name(1), "world"); - assert_eq!(typ.child(1).id(), LogicalTypeId::Integer); - } -} diff --git a/crates/duckdb/src/vtab/mod.rs b/crates/duckdb/src/vtab/mod.rs index 40717819..9883ae50 100644 --- a/crates/duckdb/src/vtab/mod.rs +++ b/crates/duckdb/src/vtab/mod.rs @@ -1,13 +1,14 @@ -use crate::{error::Error, inner_connection::InnerConnection, Connection, Result}; - -use super::{ffi, ffi::duckdb_free}; use std::ffi::c_void; -mod data_chunk; +use super::ffi; +use super::ffi::duckdb_free; +use crate::core::{DataChunkHandle, LogicalTypeHandle}; +use crate::error::Error; +use crate::inner_connection::InnerConnection; +use crate::{Connection, Result}; + mod function; -mod logical_type; mod value; -mod vector; /// The duckdb Arrow table function interface #[cfg(feature = "vtab-arrow")] @@ -20,16 +21,11 @@ pub use self::arrow::{ #[cfg(feature = "vtab-excel")] mod excel; -pub use data_chunk::DataChunk; +use std::mem::size_of; + +use ffi::{duckdb_bind_info, duckdb_data_chunk, duckdb_function_info, duckdb_init_info, duckdb_malloc}; pub use function::{BindInfo, FunctionInfo, InitInfo, TableFunction}; -pub use logical_type::{LogicalType, LogicalTypeId}; pub use value::Value; -pub use vector::{FlatVector, Inserter, ListVector, StructVector, Vector}; - -use ffi::{duckdb_bind_info, duckdb_data_chunk, duckdb_function_info, duckdb_init_info}; - -use ffi::duckdb_malloc; -use std::mem::size_of; /// duckdb_malloc a struct of type T /// used for the bind_info and init_info @@ -78,7 +74,7 @@ pub trait VTab: Sized { /// `bind` does not take ownership of `data`. /// - Concurrent access to `data` (if applicable) must be properly synchronized. /// - The `bind` object must be valid and correctly initialized. - unsafe fn bind(bind: &BindInfo, data: *mut Self::BindData) -> Result<(), Box>; + fn bind(bind: &BindInfo, data: *mut Self::BindData) -> Result<(), Box>; /// Initialize the table function /// /// # Safety @@ -90,7 +86,7 @@ pub trait VTab: Sized { /// - There is no data race when accessing `data`, meaning if `data` is accessed from multiple threads, /// proper synchronization is required. /// - The lifetime of `data` extends beyond the scope of this call to avoid use-after-free errors. - unsafe fn init(init: &InitInfo, data: *mut Self::InitData) -> Result<(), Box>; + fn init(init: &InitInfo, data: *mut Self::InitData) -> Result<(), Box>; /// The actual function /// /// # Safety @@ -105,7 +101,7 @@ pub trait VTab: Sized { /// - The `init_info` and `bind_info` data pointed to remains valid and is not freed until after this function completes. /// - No other threads are concurrently mutating the data pointed to by `init_info` and `bind_info` without proper synchronization. /// - The `output` parameter is correctly initialized and can safely be written to. - unsafe fn func(func: &FunctionInfo, output: &mut DataChunk) -> Result<(), Box>; + fn func(func: &FunctionInfo, output: DataChunkHandle) -> Result<(), Box>; /// Does the table function support pushdown /// default is false fn supports_pushdown() -> bool { @@ -113,12 +109,12 @@ pub trait VTab: Sized { } /// The parameters of the table function /// default is None - fn parameters() -> Option> { + fn parameters() -> Option> { None } /// The named parameters of the table function /// default is None - fn named_parameters() -> Option> { + fn named_parameters() -> Option> { None } } @@ -128,8 +124,8 @@ where T: VTab, { let info = FunctionInfo::from(info); - let mut output = DataChunk::from(output); - let result = T::func(&info, &mut output); + let data_chunk = DataChunkHandle::new_unowned(output); + let result = T::func(&info, data_chunk); if result.is_err() { info.set_error(&result.err().unwrap().to_string()); } @@ -197,11 +193,11 @@ impl InnerConnection { #[cfg(test)] mod test { + use std::error::Error; + use std::ffi::{c_char, CString}; + use std::sync::Arc; + use super::*; - use std::{ - error::Error, - ffi::{c_char, CString}, - }; #[repr(C)] struct HelloBindData { @@ -232,8 +228,8 @@ mod test { type InitData = HelloInitData; type BindData = HelloBindData; - unsafe fn bind(bind: &BindInfo, data: *mut HelloBindData) -> Result<(), Box> { - bind.add_result_column("column0", LogicalType::new(LogicalTypeId::Varchar)); + fn bind(bind: &BindInfo, data: *mut HelloBindData) -> Result<(), Box> { + bind.add_result_column("column0", LogicalTypeHandle::from(LogicalTypeId::Varchar)); let param = bind.get_parameter(0).to_string(); unsafe { (*data).name = CString::new(param).unwrap().into_raw(); @@ -241,14 +237,14 @@ mod test { Ok(()) } - unsafe fn init(_: &InitInfo, data: *mut HelloInitData) -> Result<(), Box> { + fn init(_: &InitInfo, data: *mut HelloInitData) -> Result<(), Box> { unsafe { (*data).done = false; } Ok(()) } - unsafe fn func(func: &FunctionInfo, output: &mut DataChunk) -> Result<(), Box> { + fn func(func: &FunctionInfo, output: DataChunkHandle) -> Result<(), Box> { let init_info = func.get_init_data::(); let bind_info = func.get_bind_data::(); @@ -257,7 +253,8 @@ mod test { output.set_len(0); } else { (*init_info).done = true; - let vector = output.flat_vector(0); + let vector_size = output.len(); + let vector: FlatVector = FlatVector::new(output.get_vector(0), Arc::new(move || vector_size)); let name = CString::from_raw((*bind_info).name); let result = CString::new(format!("Hello {}", name.to_str()?))?; // Can't consume the CString @@ -269,18 +266,19 @@ mod test { Ok(()) } - fn parameters() -> Option> { - Some(vec![LogicalType::new(LogicalTypeId::Varchar)]) + fn parameters() -> Option> { + Some(vec![LogicalTypeHandle::from(LogicalTypeId::Varchar)]) } } struct HelloWithNamedVTab {} + impl VTab for HelloWithNamedVTab { type InitData = HelloInitData; type BindData = HelloBindData; - unsafe fn bind(bind: &BindInfo, data: *mut HelloBindData) -> Result<(), Box> { - bind.add_result_column("column0", LogicalType::new(LogicalTypeId::Varchar)); + fn bind(bind: &BindInfo, data: *mut HelloBindData) -> Result<(), Box> { + bind.add_result_column("column0", LogicalTypeHandle::from(LogicalTypeId::Varchar)); let param = bind.get_named_parameter("name").unwrap().to_string(); assert!(bind.get_named_parameter("unknown_name").is_none()); unsafe { @@ -289,16 +287,19 @@ mod test { Ok(()) } - unsafe fn init(init_info: &InitInfo, data: *mut HelloInitData) -> Result<(), Box> { + fn init(init_info: &InitInfo, data: *mut HelloInitData) -> Result<(), Box> { HelloVTab::init(init_info, data) } - unsafe fn func(func: &FunctionInfo, output: &mut DataChunk) -> Result<(), Box> { + fn func(func: &FunctionInfo, output: DataChunkHandle) -> Result<(), Box> { HelloVTab::func(func, output) } - fn named_parameters() -> Option> { - Some(vec![("name".to_string(), LogicalType::new(LogicalTypeId::Varchar))]) + fn named_parameters() -> Option> { + Some(vec![( + "name".to_string(), + LogicalTypeHandle::from(LogicalTypeId::Varchar), + )]) } } @@ -329,6 +330,8 @@ mod test { #[cfg(feature = "vtab-loadable")] use duckdb_loadable_macros::duckdb_entrypoint; + use crate::core::{FlatVector, Inserter, LogicalTypeId}; + // this function is never called, but is still type checked // Exposes a extern C function named "libhello_ext_init" in the compiled dynamic library, // the "entrypoint" that duckdb will use to load the extension. diff --git a/crates/duckdb/src/vtab/vector.rs b/crates/duckdb/src/vtab/vector.rs deleted file mode 100644 index 7de18578..00000000 --- a/crates/duckdb/src/vtab/vector.rs +++ /dev/null @@ -1,282 +0,0 @@ -use std::{any::Any, ffi::CString, slice}; - -use libduckdb_sys::{duckdb_array_type_array_size, duckdb_array_vector_get_child}; - -use super::LogicalType; -use crate::ffi::{ - duckdb_list_entry, duckdb_list_vector_get_child, duckdb_list_vector_get_size, duckdb_list_vector_reserve, - duckdb_list_vector_set_size, duckdb_struct_type_child_count, duckdb_struct_type_child_name, - duckdb_struct_vector_get_child, duckdb_validity_set_row_invalid, duckdb_vector, - duckdb_vector_assign_string_element, duckdb_vector_assign_string_element_len, - duckdb_vector_ensure_validity_writable, duckdb_vector_get_column_type, duckdb_vector_get_data, - duckdb_vector_get_validity, duckdb_vector_size, -}; - -/// Vector trait. -pub trait Vector { - /// Returns a reference to the underlying Any type that this trait object - fn as_any(&self) -> &dyn Any; - /// Returns a mutable reference to the underlying Any type that this trait object - fn as_mut_any(&mut self) -> &mut dyn Any; -} - -/// A flat vector -pub struct FlatVector { - ptr: duckdb_vector, - capacity: usize, -} - -impl From for FlatVector { - fn from(ptr: duckdb_vector) -> Self { - Self { - ptr, - capacity: unsafe { duckdb_vector_size() as usize }, - } - } -} - -impl Vector for FlatVector { - fn as_any(&self) -> &dyn Any { - self - } - - fn as_mut_any(&mut self) -> &mut dyn Any { - self - } -} - -impl FlatVector { - fn with_capacity(ptr: duckdb_vector, capacity: usize) -> Self { - Self { ptr, capacity } - } - - /// Returns the capacity of the vector - pub fn capacity(&self) -> usize { - self.capacity - } - - /// Returns an unsafe mutable pointer to the vector’s - pub fn as_mut_ptr(&self) -> *mut T { - unsafe { duckdb_vector_get_data(self.ptr).cast() } - } - - /// Returns a slice of the vector - pub fn as_slice(&self) -> &[T] { - unsafe { slice::from_raw_parts(self.as_mut_ptr(), self.capacity()) } - } - - /// Returns a mutable slice of the vector - pub fn as_mut_slice(&mut self) -> &mut [T] { - unsafe { slice::from_raw_parts_mut(self.as_mut_ptr(), self.capacity()) } - } - - /// Returns the logical type of the vector - pub fn logical_type(&self) -> LogicalType { - LogicalType::from(unsafe { duckdb_vector_get_column_type(self.ptr) }) - } - - /// Set row as null - pub fn set_null(&mut self, row: usize) { - unsafe { - duckdb_vector_ensure_validity_writable(self.ptr); - let idx = duckdb_vector_get_validity(self.ptr); - duckdb_validity_set_row_invalid(idx, row as u64); - } - } - - /// Copy data to the vector. - pub fn copy(&mut self, data: &[T]) { - assert!(data.len() <= self.capacity()); - self.as_mut_slice::()[0..data.len()].copy_from_slice(data); - } -} - -/// A trait for inserting data into a vector. -pub trait Inserter { - /// Insert a value into the vector. - fn insert(&self, index: usize, value: T); -} - -impl Inserter for FlatVector { - fn insert(&self, index: usize, value: CString) { - unsafe { - duckdb_vector_assign_string_element(self.ptr, index as u64, value.as_ptr()); - } - } -} - -impl Inserter<&str> for FlatVector { - fn insert(&self, index: usize, value: &str) { - let cstr = CString::new(value.as_bytes()).unwrap(); - unsafe { - duckdb_vector_assign_string_element(self.ptr, index as u64, cstr.as_ptr()); - } - } -} - -impl Inserter<&[u8]> for FlatVector { - fn insert(&self, index: usize, value: &[u8]) { - let value_size = value.len(); - unsafe { - // This function also works for binary data. https://duckdb.org/docs/api/c/api#duckdb_vector_assign_string_element_len - duckdb_vector_assign_string_element_len( - self.ptr, - index as u64, - value.as_ptr() as *const ::std::os::raw::c_char, - value_size as u64, - ); - } - } -} - -/// A list vector. -pub struct ListVector { - /// ListVector does not own the vector pointer. - entries: FlatVector, -} - -impl From for ListVector { - fn from(ptr: duckdb_vector) -> Self { - Self { - entries: FlatVector::from(ptr), - } - } -} - -impl ListVector { - /// Returns the number of entries in the list vector. - pub fn len(&self) -> usize { - unsafe { duckdb_list_vector_get_size(self.entries.ptr) as usize } - } - - /// Returns true if the list vector is empty. - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Returns the child vector. - // TODO: not ideal interface. Where should we keep capacity. - pub fn child(&self, capacity: usize) -> FlatVector { - self.reserve(capacity); - FlatVector::with_capacity(unsafe { duckdb_list_vector_get_child(self.entries.ptr) }, capacity) - } - - /// Set primitive data to the child node. - pub fn set_child(&self, data: &[T]) { - self.child(data.len()).copy(data); - self.set_len(data.len()); - } - - /// Set offset and length to the entry. - pub fn set_entry(&mut self, idx: usize, offset: usize, length: usize) { - self.entries.as_mut_slice::()[idx].offset = offset as u64; - self.entries.as_mut_slice::()[idx].length = length as u64; - } - - /// Reserve the capacity for its child node. - fn reserve(&self, capacity: usize) { - unsafe { - duckdb_list_vector_reserve(self.entries.ptr, capacity as u64); - } - } - - /// Set the length of the list vector. - pub fn set_len(&self, new_len: usize) { - unsafe { - duckdb_list_vector_set_size(self.entries.ptr, new_len as u64); - } - } -} - -/// A array vector. (fixed-size list) -pub struct ArrayVector { - /// ArrayVector does not own the vector pointer. - ptr: duckdb_vector, -} - -impl From for ArrayVector { - fn from(ptr: duckdb_vector) -> Self { - Self { ptr } - } -} - -impl ArrayVector { - /// Get the logical type of this ArrayVector. - pub fn logical_type(&self) -> LogicalType { - LogicalType::from(unsafe { duckdb_vector_get_column_type(self.ptr) }) - } - - pub fn get_array_size(&self) -> u64 { - let ty = self.logical_type(); - unsafe { duckdb_array_type_array_size(ty.ptr) as u64 } - } - - /// Returns the child vector. - /// capacity should be a multiple of the array size. - // TODO: not ideal interface. Where should we keep count. - pub fn child(&self, capacity: usize) -> FlatVector { - FlatVector::with_capacity(unsafe { duckdb_array_vector_get_child(self.ptr) }, capacity) - } - - /// Set primitive data to the child node. - pub fn set_child(&self, data: &[T]) { - self.child(data.len()).copy(data); - } -} - -/// A struct vector. -pub struct StructVector { - /// ListVector does not own the vector pointer. - ptr: duckdb_vector, -} - -impl From for StructVector { - fn from(ptr: duckdb_vector) -> Self { - Self { ptr } - } -} - -impl StructVector { - /// Returns the child by idx in the list vector. - pub fn child(&self, idx: usize) -> FlatVector { - FlatVector::from(unsafe { duckdb_struct_vector_get_child(self.ptr, idx as u64) }) - } - - /// Take the child as [StructVector]. - pub fn struct_vector_child(&self, idx: usize) -> StructVector { - Self::from(unsafe { duckdb_struct_vector_get_child(self.ptr, idx as u64) }) - } - - /// Take the child as [ListVector]. - pub fn list_vector_child(&self, idx: usize) -> ListVector { - ListVector::from(unsafe { duckdb_struct_vector_get_child(self.ptr, idx as u64) }) - } - - /// Take the child as [ArrayVector]. - pub fn array_vector_child(&self, idx: usize) -> ArrayVector { - ArrayVector::from(unsafe { duckdb_struct_vector_get_child(self.ptr, idx as u64) }) - } - - /// Get the logical type of this struct vector. - pub fn logical_type(&self) -> LogicalType { - LogicalType::from(unsafe { duckdb_vector_get_column_type(self.ptr) }) - } - - /// Get the name of the child by idx. - pub fn child_name(&self, idx: usize) -> String { - let logical_type = self.logical_type(); - unsafe { - let child_name_ptr = duckdb_struct_type_child_name(logical_type.ptr, idx as u64); - let c_str = CString::from_raw(child_name_ptr); - let name = c_str.to_str().unwrap(); - // duckdb_free(child_name_ptr.cast()); - name.to_string() - } - } - - /// Get the number of children. - pub fn num_children(&self) -> usize { - let logical_type = self.logical_type(); - unsafe { duckdb_struct_type_child_count(logical_type.ptr) as usize } - } -} diff --git a/crates/libduckdb-sys/src/lib.rs b/crates/libduckdb-sys/src/lib.rs index ae57cadf..464c941f 100644 --- a/crates/libduckdb-sys/src/lib.rs +++ b/crates/libduckdb-sys/src/lib.rs @@ -11,6 +11,9 @@ mod bindings { #[allow(clippy::all)] pub use bindings::*; +mod types; +pub use types::*; + pub const DuckDBError: duckdb_state = duckdb_state_DuckDBError; pub const DuckDBSuccess: duckdb_state = duckdb_state_DuckDBSuccess; diff --git a/crates/libduckdb-sys/src/types.rs b/crates/libduckdb-sys/src/types.rs new file mode 100644 index 00000000..68fd8ae0 --- /dev/null +++ b/crates/libduckdb-sys/src/types.rs @@ -0,0 +1,43 @@ +use std::ffi::{c_char, CStr}; +use std::ops::Deref; + +use crate::duckdb_free; + +pub struct DuckDbStr { + // Invariant: ptr[0..len+1] is valid C string, i.e. ptr[len] is NUL byte. + ptr: core::ptr::NonNull, + len: usize, +} + +impl DuckDbStr { + pub unsafe fn from_ptr(ptr: *const c_char) -> Self { + let len = unsafe { CStr::from_ptr(ptr) }.to_bytes().len(); + unsafe { Self::from_raw_parts(ptr, len) } + } + + pub unsafe fn from_raw_parts(ptr: *const c_char, len: usize) -> Self { + let ptr = unsafe { core::ptr::NonNull::new_unchecked(ptr as *mut c_char) }; + Self { ptr, len } + } + + fn to_bytes_with_nul(&self) -> &[u8] { + let ptr = self.ptr.as_ptr() as *const u8; + unsafe { core::slice::from_raw_parts(ptr, self.len + 1) } + } +} + +impl Deref for DuckDbStr { + type Target = std::ffi::CStr; + + fn deref(&self) -> &Self::Target { + let bytes = self.to_bytes_with_nul(); + unsafe { CStr::from_bytes_with_nul_unchecked(bytes) } + } +} + +impl Drop for DuckDbStr { + fn drop(&mut self) { + let ptr = self.ptr.as_ptr() as *mut core::ffi::c_void; + unsafe { duckdb_free(ptr) }; + } +}