diff --git a/Cargo.lock b/Cargo.lock index 403350e..3348570 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -355,7 +355,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.5.0", + "indexmap 2.6.0", "lexical-core", "num", "serde", @@ -484,7 +484,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.87", ] [[package]] @@ -495,7 +495,7 @@ checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.87", ] [[package]] @@ -519,6 +519,33 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "aws-lc-rs" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe7c2840b66236045acd2607d5866e274380afd87ef99d6226e961e2cb47df45" +dependencies = [ + "aws-lc-sys", + "mirai-annotations", + "paste", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad3a619a9de81e1d7de1f1186dcba4506ed661a0e483d84410fdef0ee87b2f96" +dependencies = [ + "bindgen", + "cc", + "cmake", + "dunce", + "fs_extra", + "libc", + "paste", +] + [[package]] name = "axum" version = "0.6.20" @@ -591,6 +618,29 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags 2.6.0", + "cexpr", + "clang-sys", + "itertools 0.12.1", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn 2.0.87", + "which", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -749,6 +799,15 @@ dependencies = [ "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -792,6 +851,17 @@ dependencies = [ "phf_codegen", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.5.17" @@ -823,7 +893,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.87", ] [[package]] @@ -832,6 +902,15 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +[[package]] +name = "cmake" +version = "0.1.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a" +dependencies = [ + "cc", +] + [[package]] name = "color-eyre" version = "0.6.3" @@ -971,6 +1050,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.20" @@ -1097,7 +1185,7 @@ dependencies = [ "glob", "half", "hashbrown 0.14.5", - "indexmap 2.5.0", + "indexmap 2.6.0", "itertools 0.12.1", "log", "num-traits", @@ -1310,7 +1398,7 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr", "hashbrown 0.14.5", - "indexmap 2.5.0", + "indexmap 2.6.0", "itertools 0.12.1", "log", "paste", @@ -1339,7 +1427,7 @@ dependencies = [ "half", "hashbrown 0.14.5", "hex", - "indexmap 2.5.0", + "indexmap 2.6.0", "itertools 0.12.1", "log", "paste", @@ -1397,7 +1485,7 @@ dependencies = [ "futures", "half", "hashbrown 0.14.5", - "indexmap 2.5.0", + "indexmap 2.6.0", "itertools 0.12.1", "log", "once_cell", @@ -1471,7 +1559,7 @@ dependencies = [ "delta_kernel_derive", "either", "fix-hidden-lifetime-bug", - "indexmap 2.5.0", + "indexmap 2.6.0", "itertools 0.13.0", "lazy_static", "parquet", @@ -1496,7 +1584,7 @@ checksum = "ec5c4fb5b59b1bd55ed8ebcf941f27a327d600c19a4a4103546846c358be93ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.87", ] [[package]] @@ -1545,7 +1633,7 @@ dependencies = [ "fix-hidden-lifetime-bug", "futures", "hashbrown 0.14.5", - "indexmap 2.5.0", + "indexmap 2.6.0", "itertools 0.13.0", "lazy_static", "libc", @@ -1598,6 +1686,8 @@ dependencies = [ "itertools 0.13.0", "lazy_static", "log", + "metrics", + "metrics-exporter-prometheus", "num_cpus", "object_store", "parking_lot", @@ -1664,6 +1754,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "either" version = "1.13.0" @@ -1792,6 +1888,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1801,6 +1903,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "funty" version = "2.0.0" @@ -1863,7 +1971,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.87", ] [[package]] @@ -1950,7 +2058,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.5.0", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", @@ -1969,7 +2077,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.1.0", - "indexmap 2.5.0", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", @@ -2003,6 +2111,15 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "hashbrown" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" +dependencies = [ + "foldhash", +] + [[package]] name = "heck" version = "0.4.1" @@ -2027,6 +2144,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "http" version = "0.2.12" @@ -2138,6 +2264,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.1", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -2246,12 +2373,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.15.1", ] [[package]] @@ -2274,7 +2401,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b23a0c8dfe501baac4adf6ebbfa6eddf8f0c07f56b058cc1288017e32397846c" dependencies = [ "quote", - "syn 2.0.77", + "syn 2.0.87", ] [[package]] @@ -2369,6 +2496,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "lexical-core" version = "0.8.5" @@ -2463,6 +2596,16 @@ dependencies = [ "rle-decode-fast", ] +[[package]] +name = "libloading" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" +dependencies = [ + "cfg-if", + "windows-targets 0.52.6", +] + [[package]] name = "libm" version = "0.2.8" @@ -2564,12 +2707,63 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "metrics" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae428771d17306715c5091d446327d1cfdedc82185c65ba8423ab404e45bf10" +dependencies = [ + "ahash", + "portable-atomic", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b6f8152da6d7892ff1b7a1c0fa3f435e92b5918ad67035c3bb432111d9a29b" +dependencies = [ + "base64 0.22.1", + "http-body-util", + "hyper 1.4.1", + "hyper-rustls", + "hyper-util", + "indexmap 2.6.0", + "ipnet", + "metrics", + "metrics-util", + "quanta", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "metrics-util" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b482df36c13dd1869d73d14d28cd4855fbd6cfc32294bee109908a9f4a4ed7" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.15.1", + "metrics", + "quanta", + "sketches-ddsketch", +] + [[package]] name = "mime" version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.7.4" @@ -2601,6 +2795,22 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "mirai-annotations" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "normalize-line-endings" version = "0.3.0" @@ -2866,7 +3076,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.5.0", + "indexmap 2.6.0", ] [[package]] @@ -2924,7 +3134,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.87", ] [[package]] @@ -2945,6 +3155,12 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +[[package]] +name = "portable-atomic" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" + [[package]] name = "ppv-lite86" version = "0.2.20" @@ -2984,6 +3200,16 @@ dependencies = [ "termtree", ] +[[package]] +name = "prettyplease" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" +dependencies = [ + "proc-macro2", + "syn 2.0.87", +] + [[package]] name = "proc-macro2" version = "1.0.86" @@ -3013,7 +3239,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.87", ] [[package]] @@ -3031,6 +3257,21 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b76f1009795ca44bb5aaae8fd3f18953e209259c33d9b059b1f53d58ab7511db" +[[package]] +name = "quanta" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5167a477619228a0b284fac2674e3c388cba90631d7b7de620e6f1fcd08da5" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + [[package]] name = "quick-xml" version = "0.36.1" @@ -3051,7 +3292,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", + "rustc-hash 2.0.0", "rustls", "socket2", "thiserror", @@ -3068,7 +3309,7 @@ dependencies = [ "bytes", "rand", "ring", - "rustc-hash", + "rustc-hash 2.0.0", "rustls", "slab", "thiserror", @@ -3155,6 +3396,15 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "raw-cpuid" +version = "11.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ab240315c661615f2ee9f0f2cd32d5a7343a84d5ebcccb99d46e6637565e7b0" +dependencies = [ + "bitflags 2.6.0", +] + [[package]] name = "redox_syscall" version = "0.5.4" @@ -3292,6 +3542,12 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hash" version = "2.0.0" @@ -3326,6 +3582,7 @@ version = "0.23.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" dependencies = [ + "aws-lc-rs", "once_cell", "ring", "rustls-pki-types", @@ -3382,6 +3639,7 @@ version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -3475,7 +3733,7 @@ checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.87", ] [[package]] @@ -3579,6 +3837,12 @@ version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" +[[package]] +name = "sketches-ddsketch" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1e9a774a6c28142ac54bb25d25562e6bcf957493a184f15ad4eebccb23e410a" + [[package]] name = "slab" version = "0.4.9" @@ -3665,7 +3929,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.87", ] [[package]] @@ -3705,7 +3969,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.77", + "syn 2.0.87", ] [[package]] @@ -3718,7 +3982,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.77", + "syn 2.0.87", ] [[package]] @@ -3740,9 +4004,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.77" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -3806,7 +4070,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.87", ] [[package]] @@ -3890,7 +4154,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.87", ] [[package]] @@ -3955,7 +4219,7 @@ version = "0.22.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_spanned", "toml_datetime", @@ -4041,7 +4305,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.87", ] [[package]] @@ -4150,7 +4414,7 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.87", ] [[package]] @@ -4262,7 +4526,7 @@ checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.87", ] [[package]] @@ -4321,7 +4585,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.87", "wasm-bindgen-shared", ] @@ -4355,7 +4619,7 @@ checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.87", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4389,6 +4653,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + [[package]] name = "winapi" version = "0.3.9" @@ -4658,7 +4934,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.87", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index c2c706d..bd2d26e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,8 @@ http-body = {version = "0.4.5" } itertools = "0.13.0" lazy_static = "1.4.0" log = "0.4.22" +metrics = {version = "0.24.0", optional = true } +metrics-exporter-prometheus = {version = "0.16.0", optional = true } num_cpus = "1.16.0" object_store = { version = "0.10.2", features = ["aws"], optional = true } parking_lot = "0.12.3" @@ -63,10 +65,11 @@ url = "2.5.2" default = ["functions-parquet"] deltalake = ["dep:deltalake"] flightsql = ["dep:arrow-flight", "dep:tonic"] -experimental-flightsql-server = ["dep:arrow-flight", "dep:tonic"] +experimental-flightsql-server = ["flightsql"] s3 = ["object_store/aws", "url"] functions-json = ["dep:datafusion-functions-json"] functions-parquet = ["dep:datafusion-functions-parquet"] +metrics = ["dep:metrics", "dep:metrics-exporter-prometheus"] [[bin]] name = "dft" diff --git a/src/args.rs b/src/args.rs index 7d96c7a..b8d93a2 100644 --- a/src/args.rs +++ b/src/args.rs @@ -67,6 +67,7 @@ pub struct DftArgs { #[clap(long, short, help = "Only show how long the query took to run")] pub time: bool, + #[cfg(feature = "experimental-flightsql-server")] #[clap(long, help = "Start a FlightSQL server")] pub serve: bool, diff --git a/src/config.rs b/src/config.rs index 2b68486..adcb3f2 100644 --- a/src/config.rs +++ b/src/config.rs @@ -258,6 +258,9 @@ pub struct FlightSQLConfig { pub connection_url: String, #[serde(default = "default_benchmark_iterations")] pub benchmark_iterations: usize, + #[cfg(feature = "metrics")] + #[serde(default = "default_server_metrics_port")] + pub server_metrics_port: String, } #[cfg(feature = "flightsql")] @@ -266,6 +269,8 @@ impl Default for FlightSQLConfig { Self { connection_url: default_connection_url(), benchmark_iterations: default_benchmark_iterations(), + #[cfg(feature = "metrics")] + server_metrics_port: default_server_metrics_port(), } } } @@ -275,6 +280,11 @@ pub fn default_connection_url() -> String { "http://localhost:50051".to_string() } +#[cfg(all(feature = "experimental-flightsql-server", feature = "metrics"))] +fn default_server_metrics_port() -> String { + "0.0.0.0:9000".to_string() +} + #[derive(Clone, Debug, Default, Deserialize)] pub struct EditorConfig { pub experimental_syntax_highlighting: bool, diff --git a/src/lib.rs b/src/lib.rs index 4b8c925..0c2aecd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,7 +4,7 @@ pub mod config; pub mod execution; pub mod extensions; #[cfg(feature = "experimental-flightsql-server")] -pub mod flightsql_server; +pub mod server; pub mod telemetry; pub mod test_utils; pub mod tui; diff --git a/src/main.rs b/src/main.rs index 0d6bb38..a5f0b12 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,20 +23,17 @@ use dft::cli::CliApp; use dft::execution::flightsql::FlightSQLContext; use dft::execution::{local::ExecutionContext, AppExecution, AppType}; #[cfg(feature = "experimental-flightsql-server")] -use dft::flightsql_server::{FlightSqlApp, FlightSqlServiceImpl}; +use dft::server::FlightSqlApp; use dft::telemetry; use dft::tui::state::AppState; use dft::tui::{state, App}; +#[cfg(feature = "experimental-flightsql-server")] use log::info; #[allow(unused_mut)] fn main() -> Result<()> { let cli = DftArgs::parse(); - if !cli.files.is_empty() || !cli.commands.is_empty() || cli.serve { - env_logger::init(); - } - let state = state::initialize(cli.config_path()); // With Runtimes configured correctly the main Tokio runtime should only be used for network @@ -53,8 +50,31 @@ fn main() -> Result<()> { } async fn app_entry_point(cli: DftArgs, state: AppState<'_>) -> Result<()> { + #[cfg(feature = "experimental-flightsql-server")] + if cli.serve { + env_logger::init(); + const DEFAULT_SERVER_ADDRESS: &str = "127.0.0.1:50051"; + info!("Starting FlightSQL server on {}", DEFAULT_SERVER_ADDRESS); + let state = state::initialize(cli.config_path()); + let execution_ctx = + ExecutionContext::try_new(&state.config.execution, AppType::FlightSQLServer)?; + if cli.run_ddl { + execution_ctx.execute_ddl().await; + } + let app_execution = AppExecution::new(execution_ctx); + let app = FlightSqlApp::try_new( + app_execution, + &cli.flightsql_host + .unwrap_or(DEFAULT_SERVER_ADDRESS.to_string()), + &state.config.flightsql.server_metrics_port, + ) + .await?; + app.run_app().await; + return Ok(()); + } // CLI mode: executing commands from files or CLI arguments if !cli.files.is_empty() || !cli.commands.is_empty() { + env_logger::init(); let execution_ctx = ExecutionContext::try_new(&state.config.execution, AppType::Cli)?; #[allow(unused_mut)] let mut app_execution = AppExecution::new(execution_ctx); @@ -70,37 +90,10 @@ async fn app_entry_point(cli: DftArgs, state: AppState<'_>) -> Result<()> { } let app = CliApp::new(app_execution, cli.clone()); app.execute_files_or_commands().await?; - // FlightSQL Server mode: start a FlightSQL server - } else if cli.serve { - #[cfg(not(feature = "experimental-flightsql-server"))] - { - panic!("FlightSQL feature is not enabled"); - } - #[cfg(feature = "experimental-flightsql-server")] - { - const DEFAULT_SERVER_ADDRESS: &str = "127.0.0.1:50051"; - info!("Starting FlightSQL server on {}", DEFAULT_SERVER_ADDRESS); - let state = state::initialize(cli.config_path()); - let execution_ctx = - ExecutionContext::try_new(&state.config.execution, AppType::FlightSQLServer)?; - if cli.run_ddl { - execution_ctx.execute_ddl().await; - } - let app_execution = AppExecution::new(execution_ctx); - let server = FlightSqlServiceImpl::new(app_execution); - let app = FlightSqlApp::new( - server.service(), - &cli.flightsql_host - .unwrap_or(DEFAULT_SERVER_ADDRESS.to_string()), - ) - .await; - app.run_app().await; - } - } - // TUI mode: running the TUI - else { - // use alternate logging for TUI - telemetry::initialize_logs()?; + // FlightSQL Server mode: start a FlightSQL server + } else { + // TUI mode: running the TUI + telemetry::initialize_logs()?; // use alternate logging for TUI let state = state::initialize(cli.config_path()); let execution_ctx = ExecutionContext::try_new(&state.config.execution, AppType::Tui)?; let app_execution = AppExecution::new(execution_ctx); diff --git a/src/server/mod.rs b/src/server/mod.rs new file mode 100644 index 0000000..adbebb5 --- /dev/null +++ b/src/server/mod.rs @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod services; + +use crate::execution::AppExecution; +use crate::test_utils::trailers_layer::TrailersLayer; +use color_eyre::Result; +use log::info; +use metrics::{describe_counter, describe_histogram}; +use metrics_exporter_prometheus::{Matcher, PrometheusBuilder}; +use std::net::SocketAddr; +use std::time::Duration; +use tokio::net::TcpListener; +use tokio::task::JoinHandle; + +const DEFAULT_TIMEOUT_SECONDS: u64 = 60; + +fn initialize_metrics() { + describe_counter!("requests", "Incoming requests by FlightSQL endpoint"); + + describe_histogram!( + "get_flight_info_latency_ms", + metrics::Unit::Milliseconds, + "Get flight info latency ms" + ); + + describe_histogram!( + "do_get_fallback_latency_ms", + metrics::Unit::Milliseconds, + "Do get fallback latency ms" + ) +} + +/// Creates and manages a running FlightSqlServer with a background task +pub struct FlightSqlApp { + /// channel to send shutdown command + shutdown: Option>, + + /// Address the server is listening on + pub addr: SocketAddr, + + /// handle for the server task + handle: Option>>, +} + +impl FlightSqlApp { + /// create a new app for the flightsql server + #[allow(dead_code)] + pub async fn try_new( + app_execution: AppExecution, + addr: &str, + metrics_addr: &str, + ) -> Result { + let flightsql = services::flightsql::FlightSqlServiceImpl::new(app_execution); + // let OS choose a free port + let listener = TcpListener::bind(addr).await.unwrap(); + let addr = listener.local_addr().unwrap(); + + // prepare the shutdown channel + let (tx, rx) = tokio::sync::oneshot::channel(); + + let server_timeout = Duration::from_secs(DEFAULT_TIMEOUT_SECONDS); + + let shutdown_future = async move { + rx.await.ok(); + }; + + let serve_future = tonic::transport::Server::builder() + .timeout(server_timeout) + .layer(TrailersLayer) + .add_service(flightsql.service()) + .serve_with_incoming_shutdown( + tokio_stream::wrappers::TcpListenerStream::new(listener), + shutdown_future, + ); + + #[cfg(feature = "metrics")] + { + let builder = PrometheusBuilder::new(); + let addr: SocketAddr = metrics_addr.parse()?; + info!("Listening to metrics on {addr}"); + builder + .with_http_listener(addr) + .set_buckets_for_metric( + Matcher::Suffix("latency_ms".to_string()), + &[ + 1.0, 3.0, 5.0, 10.0, 25.0, 50.0, 75.0, 100.0, 250.0, 500.0, 1000.0, 2500.0, + 5000.0, 10000.0, 20000.0, + ], + )? + .install() + .expect("failed to install metrics recorder/exporter"); + + initialize_metrics(); + } + + // Run the server in its own background task + let handle = tokio::task::spawn(serve_future); + + let app = Self { + shutdown: Some(tx), + addr, + handle: Some(handle), + }; + Ok(app) + } + + /// Stops the server and waits for the server to shutdown + pub async fn shutdown_and_wait(mut self) { + if let Some(shutdown) = self.shutdown.take() { + shutdown.send(()).expect("server quit early"); + } + if let Some(handle) = self.handle.take() { + handle + .await + .expect("task join error (panic?)") + .expect("Server Error found at shutdown"); + } + } + + pub async fn run_app(self) { + if let Some(handle) = self.handle { + handle + .await + .expect("Unable to run server task") + .expect("Server Error found at shutdown"); + } else { + panic!("Server task not found"); + } + } +} diff --git a/src/flightsql_server/mod.rs b/src/server/services/flightsql.rs similarity index 74% rename from src/flightsql_server/mod.rs rename to src/server/services/flightsql.rs index 1717451..4034b83 100644 --- a/src/flightsql_server/mod.rs +++ b/src/server/services/flightsql.rs @@ -16,7 +16,6 @@ // under the License. use crate::execution::{local::ExecutionContext, AppExecution}; -use crate::test_utils::trailers_layer::TrailersLayer; use arrow_flight::encode::FlightDataEncoderBuilder; use arrow_flight::error::FlightError; use arrow_flight::flight_service_server::{FlightService, FlightServiceServer}; @@ -27,14 +26,12 @@ use datafusion::logical_expr::LogicalPlan; use datafusion::sql::parser::DFParser; use futures::{StreamExt, TryStreamExt}; use log::{debug, error, info}; +use metrics::{counter, histogram}; use prost::Message; use std::collections::HashMap; -use std::net::SocketAddr; use std::str::FromStr; use std::sync::{Arc, Mutex}; -use std::time::Duration; -use tokio::net::TcpListener; -use tokio::task::JoinHandle; +use std::time::Instant; use tonic::{Request, Response, Status}; use uuid::Uuid; @@ -104,18 +101,18 @@ impl FlightSqlServiceImpl { Ok(Response::new(info)) } else { - error!("Error encoding ticket"); + error!("error encoding ticket"); Err(Status::internal("Error encoding ticket")) } } Err(e) => { - error!("Error planning SQL query: {:?}", e); + error!("error planning SQL query: {:?}", e); Err(Status::internal("Error planning SQL query")) } } } Err(e) => { - error!("Error parsing SQL query: {:?}", e); + error!("error parsing SQL query: {:?}", e); Err(Status::internal("Error parsing SQL query")) } } @@ -177,7 +174,7 @@ impl FlightSqlServiceImpl { } } Err(e) => { - error!("Error decoding ticket: {:?}", e); + error!("error decoding ticket: {:?}", e); Err(Status::internal("Error decoding ticket")) } } @@ -193,7 +190,12 @@ impl FlightSqlService for FlightSqlServiceImpl { query: CommandStatementQuery, request: Request, ) -> Result, Status> { - self.get_flight_info_statement_handler(query, request).await + counter!("requests", "endpoint" => "get_flight_info").increment(1); + let start = Instant::now(); + let res = self.get_flight_info_statement_handler(query, request).await; + let duration = start.elapsed(); + histogram!("get_flight_info_latency_ms").record(duration.as_millis() as f64); + res } async fn do_get_statement( @@ -209,86 +211,13 @@ impl FlightSqlService for FlightSqlServiceImpl { request: Request, message: Any, ) -> Result::DoGetStream>, Status> { - self.do_get_fallback_handler(request, message).await + counter!("requests", "endpoint" => "do_get_fallback").increment(1); + let start = Instant::now(); + let res = self.do_get_fallback_handler(request, message).await; + let duration = start.elapsed(); + histogram!("do_get_fallback_latency_ms").record(duration.as_millis() as f64); + res } async fn register_sql_info(&self, _id: i32, _result: &SqlInfo) {} } - -const DEFAULT_TIMEOUT_SECONDS: u64 = 60; - -/// Creates and manages a running FlightSqlServer with a background task -pub struct FlightSqlApp { - /// channel to send shutdown command - shutdown: Option>, - - /// Address the server is listening on - pub addr: SocketAddr, - - /// handle for the server task - handle: Option>>, -} - -impl FlightSqlApp { - /// create a new app for the flightsql server - #[allow(dead_code)] - pub async fn new( - flightsql_server: FlightServiceServer, - addr: &str, - ) -> Self { - // let OS choose a free port - let listener = TcpListener::bind(addr).await.unwrap(); - let addr = listener.local_addr().unwrap(); - - // prepare the shutdown channel - let (tx, rx) = tokio::sync::oneshot::channel(); - - let server_timeout = Duration::from_secs(DEFAULT_TIMEOUT_SECONDS); - - let shutdown_future = async move { - rx.await.ok(); - }; - - let serve_future = tonic::transport::Server::builder() - .timeout(server_timeout) - .layer(TrailersLayer) - .add_service(flightsql_server) - .serve_with_incoming_shutdown( - tokio_stream::wrappers::TcpListenerStream::new(listener), - shutdown_future, - ); - - // Run the server in its own background task - let handle = tokio::task::spawn(serve_future); - - Self { - shutdown: Some(tx), - addr, - handle: Some(handle), - } - } - - /// Stops the server and waits for the server to shutdown - pub async fn shutdown_and_wait(mut self) { - if let Some(shutdown) = self.shutdown.take() { - shutdown.send(()).expect("server quit early"); - } - if let Some(handle) = self.handle.take() { - handle - .await - .expect("task join error (panic?)") - .expect("Server Error found at shutdown"); - } - } - - pub async fn run_app(self) { - if let Some(handle) = self.handle { - handle - .await - .expect("Unable to run server task") - .expect("Server Error found at shutdown"); - } else { - panic!("Server task not found"); - } - } -} diff --git a/src/server/services/mod.rs b/src/server/services/mod.rs new file mode 100644 index 0000000..39fb84d --- /dev/null +++ b/src/server/services/mod.rs @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod flightsql; diff --git a/tests/config.rs b/tests/config.rs index 096e6ce..ce7b98e 100644 --- a/tests/config.rs +++ b/tests/config.rs @@ -85,6 +85,7 @@ impl TestConfigBuilder { self } + #[cfg(feature = "flightsql")] pub fn with_flightsql_benchmark_iterations(&mut self, iterations: u64) -> &mut Self { self.config_text.push_str(&format!( "[flightsql]\nbenchmark_iterations = {}\n",