From b7201818290c68665213455cc7995abfb2afd997 Mon Sep 17 00:00:00 2001 From: Dzejkop Date: Wed, 6 Dec 2023 09:32:08 +0100 Subject: [PATCH] WIP: Integrate telemetry-batteries --- Cargo.lock | 267 ++++++++++++++++++++++++++++++- Cargo.toml | 4 + config.toml | 1 + src/config.rs | 7 + src/main.rs | 23 ++- src/server/routes/transaction.rs | 39 +++-- tests/common/mod.rs | 1 + 7 files changed, 311 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b9e9345..a51d2d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -782,6 +782,15 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "cadence" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f39286bc075b023101dccdb79456a1334221c768b8faede0c2aff7ed29a9482d" +dependencies = [ + "crossbeam-channel", +] + [[package]] name = "camino" version = "1.1.6" @@ -1045,6 +1054,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.3" @@ -2013,7 +2032,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 2.1.0", "slab", "tokio", "tokio-util", @@ -2308,6 +2327,16 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" version = "2.1.0" @@ -2597,6 +2626,39 @@ dependencies = [ "autocfg", ] +[[package]] +name = "metrics" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5" +dependencies = [ + "ahash 0.8.6", + "metrics-macros", + "portable-atomic", +] + +[[package]] +name = "metrics-exporter-statsd" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34a620eecf9e4321ebbef8f2f8e7cd22e098f11b65f2d987ce66faaa8918418" +dependencies = [ + "cadence", + "metrics", + "thiserror", +] + +[[package]] +name = "metrics-macros" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "mime" version = "0.3.17" @@ -2791,12 +2853,111 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "opentelemetry" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9591d937bc0e6d2feb6f71a559540ab300ea49955229c347a517a28d27784c54" +dependencies = [ + "opentelemetry_api", + "opentelemetry_sdk", +] + +[[package]] +name = "opentelemetry-datadog" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f4ecf595095d3b641dd2761a0c3d1f175d3d6c28f38e65418d8004ea3255dd" +dependencies = [ + "futures-core", + "http", + "indexmap 1.9.3", + "itertools 0.10.5", + "once_cell", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-semantic-conventions", + "reqwest", + "rmp", + "thiserror", + "url", +] + +[[package]] +name = "opentelemetry-http" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7594ec0e11d8e33faf03530a4c49af7064ebba81c1480e01be67d90b356508b" +dependencies = [ + "async-trait", + "bytes", + "http", + "opentelemetry_api", + "reqwest", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73c9f9340ad135068800e7f1b24e9e09ed9e7143f5bf8518ded3d3ec69789269" +dependencies = [ + "opentelemetry", +] + +[[package]] +name = "opentelemetry_api" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a81f725323db1b1206ca3da8bb19874bbd3f57c3bcd59471bfb04525b265b9b" +dependencies = [ + "futures-channel", + "futures-util", + "indexmap 1.9.3", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", + "urlencoding", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8e705a0612d48139799fcbaba0d4a90f06277153e43dd2bdc16c6f0edd8026" +dependencies = [ + "async-trait", + "crossbeam-channel", + "futures-channel", + "futures-executor", + "futures-util", + "once_cell", + "opentelemetry_api", + "ordered-float", + "percent-encoding", + "rand", + "regex", + "thiserror", + "tokio", + "tokio-stream", +] + [[package]] name = "option-ext" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ordered-float" +version = "3.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1e1c390732d15f1d48471625cd92d154e66db2c56645e29a9cd26f4699f72dc" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-multimap" version = "0.4.3" @@ -2995,7 +3156,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap", + "indexmap 2.1.0", ] [[package]] @@ -3118,6 +3279,12 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +[[package]] +name = "portable-atomic" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bccab0e7fd7cc19f820a1c8c91720af652d0c88dc9664dd72aef2614f04af3b" + [[package]] name = "postgres-docker-utils" version = "0.1.0" @@ -3491,6 +3658,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "rmp" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f9860a6cc38ed1da53456442089b4dfa35e7cedaa326df63017af88385e6b20" +dependencies = [ + "byteorder", + "num-traits", + "paste", +] + [[package]] name = "ron" version = "0.7.1" @@ -4020,7 +4198,7 @@ dependencies = [ "futures-util", "hashlink", "hex", - "indexmap", + "indexmap 2.1.0", "log", "memchr", "once_cell", @@ -4332,6 +4510,30 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "telemetry-batteries" +version = "0.1.0" +source = "git+https://github.com/worldcoin/telemetry-batteries#42d074d9cd87889b9244e4b2823c73dc03596007" +dependencies = [ + "chrono", + "dirs", + "http", + "metrics", + "metrics-exporter-statsd", + "opentelemetry", + "opentelemetry-datadog", + "opentelemetry-http", + "serde", + "serde_json", + "thiserror", + "tokio", + "tracing", + "tracing-appender", + "tracing-opentelemetry", + "tracing-serde", + "tracing-subscriber", +] + [[package]] name = "tempfile" version = "3.8.1" @@ -4590,7 +4792,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap", + "indexmap 2.1.0", "toml_datetime", "winnow", ] @@ -4601,7 +4803,7 @@ version = "0.20.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" dependencies = [ - "indexmap", + "indexmap 2.1.0", "toml_datetime", "winnow", ] @@ -4612,7 +4814,7 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" dependencies = [ - "indexmap", + "indexmap 2.1.0", "serde", "serde_spanned", "toml_datetime", @@ -4680,6 +4882,18 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-appender" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3566e8ce28cc0a3fe42519fc80e6b4c943cc4c8cef275620eb8dac2d3d4e06cf" +dependencies = [ + "crossbeam-channel", + "thiserror", + "time", + "tracing-subscriber", +] + [[package]] name = "tracing-attributes" version = "0.1.27" @@ -4711,6 +4925,44 @@ dependencies = [ "tracing", ] +[[package]] +name = "tracing-log" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-opentelemetry" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75327c6b667828ddc28f5e3f169036cb793c3f588d83bf0f262a7f062ffed3c8" +dependencies = [ + "once_cell", + "opentelemetry", + "opentelemetry_sdk", + "smallvec", + "tracing", + "tracing-core", + "tracing-log 0.1.4", + "tracing-subscriber", +] + [[package]] name = "tracing-serde" version = "0.1.3" @@ -4734,9 +4986,11 @@ dependencies = [ "serde", "serde_json", "sharded-slab", + "smallvec", "thread_local", "tracing", "tracing-core", + "tracing-log 0.2.0", "tracing-serde", ] @@ -4806,6 +5060,7 @@ dependencies = [ "spki", "sqlx", "strum", + "telemetry-batteries", "test-case", "thiserror", "tokio", diff --git a/Cargo.toml b/Cargo.toml index be920b0..d5bfd8b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ name = "tx-sitter" version = "0.1.0" edition = "2021" +default-run = "tx-sitter" [workspace] members = ["crates/*"] @@ -71,6 +72,9 @@ async-trait = "0.1.74" itertools = "0.12.0" base64 = "0.21.5" +# Company +telemetry-batteries = { git = "https://github.com/worldcoin/telemetry-batteries" } + # Internal postgres-docker-utils = { path = "crates/postgres-docker-utils" } diff --git a/config.toml b/config.toml index fcb1ac3..d34c83f 100644 --- a/config.toml +++ b/config.toml @@ -1,5 +1,6 @@ [service] escalation_interval = "1m" +datadog_enabled = true [server] host = "127.0.0.1:3000" diff --git a/src/config.rs b/src/config.rs index 8836e9c..5813ba4 100644 --- a/src/config.rs +++ b/src/config.rs @@ -17,6 +17,9 @@ pub struct Config { pub struct TxSitterConfig { #[serde(with = "humantime_serde")] pub escalation_interval: Duration, + + #[serde(default)] + pub datadog_enabled: bool, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -108,6 +111,7 @@ mod tests { const WITH_DB_CONNECTION_STRING: &str = indoc! {r#" [service] escalation_interval = "1h" + datadog_enabled = false [server] host = "127.0.0.1:3000" @@ -123,6 +127,7 @@ mod tests { const WITH_DB_PARTS: &str = indoc! {r#" [service] escalation_interval = "1h" + datadog_enabled = false [server] host = "127.0.0.1:3000" @@ -144,6 +149,7 @@ mod tests { let config = Config { service: TxSitterConfig { escalation_interval: Duration::from_secs(60 * 60), + datadog_enabled: false, }, server: ServerConfig { host: SocketAddr::from(([127, 0, 0, 1], 3000)), @@ -167,6 +173,7 @@ mod tests { let config = Config { service: TxSitterConfig { escalation_interval: Duration::from_secs(60 * 60), + datadog_enabled: false, }, server: ServerConfig { host: SocketAddr::from(([127, 0, 0, 1], 3000)), diff --git a/src/main.rs b/src/main.rs index faf405d..4913e51 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,6 +2,8 @@ use std::path::PathBuf; use clap::Parser; use config::FileFormat; +use telemetry_batteries::tracing::batteries::datadog::DatadogBattery; +use tracing::Level; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::EnvFilter; @@ -33,11 +35,6 @@ async fn main() -> eyre::Result<()> { dotenv::from_path(path)?; } - tracing_subscriber::registry() - .with(tracing_subscriber::fmt::layer().pretty().compact()) - .with(EnvFilter::from_default_env()) - .init(); - let mut settings = config::Config::builder(); for arg in &args.config { @@ -60,6 +57,22 @@ async fn main() -> eyre::Result<()> { let config = settings.try_deserialize::()?; + if config.service.datadog_enabled { + let datadog_battery = DatadogBattery::new( + Some("http://localhost:8126"), + Level::DEBUG, + "tx-sitter", + None, + ); + + datadog_battery.init()?; + } else { + tracing_subscriber::registry() + .with(tracing_subscriber::fmt::layer().pretty().compact()) + .with(EnvFilter::from_default_env()) + .init(); + } + let service = Service::new(config).await?; service.wait().await?; diff --git a/src/server/routes/transaction.rs b/src/server/routes/transaction.rs index 25a5f6f..b51c151 100644 --- a/src/server/routes/transaction.rs +++ b/src/server/routes/transaction.rs @@ -110,26 +110,25 @@ pub async fn get_txs( let txs = app.db.read_txs(&tx_id).await?; - let txs = txs - .into_iter() - .map(|tx| GetTxResponse { - tx_id: tx.tx_id, - to: tx.to.0, - data: if tx.data.is_empty() { - None - } else { - Some(tx.data.into()) - }, - value: tx.value.0, - gas_limit: tx.gas_limit.0, - nonce: tx.nonce, - tx_hash: tx.tx_hash.map(|h| h.0), - status: tx - .status - .map(GetTxResponseStatus::TxStatus) - .unwrap_or(GetTxResponseStatus::Unsent(UnsentStatus::Unsent)), - }) - .collect(); + let txs = + txs.into_iter() + .map(|tx| GetTxResponse { + tx_id: tx.tx_id, + to: tx.to.0, + data: if tx.data.is_empty() { + None + } else { + Some(tx.data.into()) + }, + value: tx.value.0, + gas_limit: tx.gas_limit.0, + nonce: tx.nonce, + tx_hash: tx.tx_hash.map(|h| h.0), + status: tx.status.map(GetTxResponseStatus::TxStatus).unwrap_or( + GetTxResponseStatus::Unsent(UnsentStatus::Unsent), + ), + }) + .collect(); Ok(Json(txs)) } diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 50c0460..6620594 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -141,6 +141,7 @@ pub async fn setup_service( let config = Config { service: TxSitterConfig { escalation_interval, + datadog_enabled: false, }, server: ServerConfig { host: SocketAddr::V4(SocketAddrV4::new(