From 573e6786ded60276cacf7a29e8f542fa086f7fe0 Mon Sep 17 00:00:00 2001 From: vianney Date: Fri, 8 Nov 2024 13:34:00 +0100 Subject: [PATCH] [APMSP-1512] Add metadata headers for stats (#712) * Add metadata headers for stats * Add commit_sha and interpreter_vendor to metadata * Add doc for TraceExporterBuilder * Rename version to app_version APMSP-1512 --- data-pipeline-ffi/src/trace_exporter.rs | 2 +- .../examples/send-traces-with-stats.rs | 2 +- data-pipeline/src/stats_exporter.rs | 76 ++++++------- data-pipeline/src/trace_exporter.rs | 100 ++++++++++-------- 4 files changed, 95 insertions(+), 85 deletions(-) diff --git a/data-pipeline-ffi/src/trace_exporter.rs b/data-pipeline-ffi/src/trace_exporter.rs index b4dbbf81c..b3cdbe365 100644 --- a/data-pipeline-ffi/src/trace_exporter.rs +++ b/data-pipeline-ffi/src/trace_exporter.rs @@ -60,7 +60,7 @@ pub unsafe extern "C" fn ddog_trace_exporter_new( .set_language_interpreter(language_interpreter.to_utf8_lossy().as_ref()) .set_hostname(hostname.to_utf8_lossy().as_ref()) .set_env(env.to_utf8_lossy().as_ref()) - .set_version(version.to_utf8_lossy().as_ref()) + .set_app_version(version.to_utf8_lossy().as_ref()) .set_service(service.to_utf8_lossy().as_ref()) .set_input_format(input_format) .set_output_format(output_format) diff --git a/data-pipeline/examples/send-traces-with-stats.rs b/data-pipeline/examples/send-traces-with-stats.rs index 31962f167..34a05e614 100644 --- a/data-pipeline/examples/send-traces-with-stats.rs +++ b/data-pipeline/examples/send-traces-with-stats.rs @@ -34,7 +34,7 @@ fn main() { .set_url("http://localhost:8126") .set_hostname("test") .set_env("testing") - .set_version(env!("CARGO_PKG_VERSION")) + .set_app_version(env!("CARGO_PKG_VERSION")) .set_service("data-pipeline-test") .set_tracer_version(env!("CARGO_PKG_VERSION")) .set_language("rust") diff --git a/data-pipeline/src/stats_exporter.rs b/data-pipeline/src/stats_exporter.rs index 84d109b79..7a7f74c68 100644 --- a/data-pipeline/src/stats_exporter.rs +++ b/data-pipeline/src/stats_exporter.rs @@ -2,6 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 use std::{ + borrow::Borrow, + collections::HashMap, sync::{ atomic::{AtomicU64, Ordering}, Arc, Mutex, @@ -10,38 +12,22 @@ use std::{ }; use datadog_trace_protobuf::pb; -use ddcommon::{connector, tag::Tag, Endpoint}; +use ddcommon::{connector, Endpoint}; use hyper; use tokio::select; use tokio_util::sync::CancellationToken; -use crate::span_concentrator::SpanConcentrator; +use crate::{span_concentrator::SpanConcentrator, trace_exporter::TracerMetadata}; const STATS_ENDPOINT_PATH: &str = "/v0.6/stats"; -/// Metadata required in a ClientStatsPayload -#[derive(Debug, Default, Clone)] -pub struct LibraryMetadata { - pub hostname: String, - pub env: String, - pub version: String, - pub lang: String, - pub tracer_version: String, - pub runtime_id: String, - pub service: String, - pub container_id: String, - pub git_commit_sha: String, - /// Should be left empty by client, except for some specific environment - pub tags: Vec, -} - /// An exporter that concentrates and sends stats to the agent #[derive(Debug)] pub struct StatsExporter { flush_interval: time::Duration, concentrator: Arc>, endpoint: Endpoint, - meta: LibraryMetadata, + meta: TracerMetadata, sequence_id: AtomicU64, client: ddcommon::HttpClient, cancellation_token: CancellationToken, @@ -52,14 +38,14 @@ impl StatsExporter { /// /// - `flush_interval` the interval on which the concentrator is flushed /// - `concentrator` SpanConcentrator storing the stats to be sent to the agent - /// - `meta` the metadata used when sending the ClientStatsPayload to the agent + /// - `meta` metadata used in ClientStatsPayload and as headers to send stats to the agent /// - `endpoint` the Endpoint used to send stats to the agent /// - `cancellation_token` Token used to safely shutdown the exporter by force flushing the /// concentrator pub fn new( flush_interval: time::Duration, concentrator: Arc>, - meta: LibraryMetadata, + meta: TracerMetadata, endpoint: Endpoint, cancellation_token: CancellationToken, ) -> Self { @@ -95,15 +81,23 @@ impl StatsExporter { return Ok(()); } let body = rmp_serde::encode::to_vec_named(&payload)?; - let req = self + + let headers: HashMap<&'static str, String> = self.meta.borrow().into(); + + let mut req_builder = self .endpoint .into_request_builder(concat!("Libdatadog/", env!("CARGO_PKG_VERSION")))? .header( hyper::header::CONTENT_TYPE, ddcommon::header::APPLICATION_MSGPACK, ) - .method(hyper::Method::POST) - .body(hyper::Body::from(body))?; + .method(hyper::Method::POST); + + for (key, value) in &headers { + req_builder = req_builder.header(*key, value); + } + + let req = req_builder.body(hyper::Body::from(body))?; let resp = self.client.request(req).await?; @@ -128,7 +122,7 @@ impl StatsExporter { fn flush(&self, force_flush: bool) -> pb::ClientStatsPayload { let sequence = self.sequence_id.fetch_add(1, Ordering::Relaxed); encode_stats_payload( - self.meta.clone(), + self.meta.borrow(), sequence, self.concentrator .lock() @@ -158,24 +152,24 @@ impl StatsExporter { } fn encode_stats_payload( - meta: LibraryMetadata, + meta: &TracerMetadata, sequence: u64, buckets: Vec, ) -> pb::ClientStatsPayload { pb::ClientStatsPayload { - hostname: meta.hostname, - env: meta.env, - lang: meta.lang, - version: meta.version, - runtime_id: meta.runtime_id, - tracer_version: meta.tracer_version, - service: meta.service, - container_id: meta.container_id, - git_commit_sha: meta.git_commit_sha, - tags: meta.tags.into_iter().map(|t| t.to_string()).collect(), + hostname: meta.hostname.clone(), + env: meta.env.clone(), + lang: meta.language.clone(), + version: meta.app_version.clone(), + runtime_id: meta.runtime_id.clone(), + tracer_version: meta.tracer_version.clone(), + service: meta.service.clone(), sequence, stats: buckets, - // Agent-only field + git_commit_sha: meta.git_commit_sha.clone(), + // These fields will be set by the Agent + container_id: String::new(), + tags: Vec::new(), agent_aggregation: String::new(), image_tag: String::new(), } @@ -211,12 +205,12 @@ mod tests { let _ = is_sync::; } - fn get_test_metadata() -> LibraryMetadata { - LibraryMetadata { + fn get_test_metadata() -> TracerMetadata { + TracerMetadata { hostname: "libdatadog-test".into(), env: "test".into(), - version: "0.0.0".into(), - lang: "rust".into(), + app_version: "0.0.0".into(), + language: "rust".into(), tracer_version: "0.0.0".into(), runtime_id: "e39d6d12-0752-489f-b488-cf80006c0378".into(), service: "stats_exporter_test".into(), diff --git a/data-pipeline/src/trace_exporter.rs b/data-pipeline/src/trace_exporter.rs index 97a46b0c4..ff535a1b6 100644 --- a/data-pipeline/src/trace_exporter.rs +++ b/data-pipeline/src/trace_exporter.rs @@ -1,7 +1,6 @@ // Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 use crate::agent_info::{AgentInfoArc, AgentInfoFetcher}; -use crate::stats_exporter::LibraryMetadata; use crate::{ health_metrics, health_metrics::HealthMetric, span_concentrator::SpanConcentrator, stats_exporter, @@ -154,19 +153,21 @@ fn drop_chunks(traces: &mut Vec>) -> DroppedP0Counts { } } -#[derive(Clone, Default)] -struct TracerMetadata { - hostname: String, - env: String, - version: String, - runtime_id: String, - service: String, - tracer_version: String, - language: String, - language_version: String, - language_interpreter: String, - client_computed_stats: bool, - client_computed_top_level: bool, +#[derive(Clone, Default, Debug)] +pub struct TracerMetadata { + pub hostname: String, + pub env: String, + pub app_version: String, + pub runtime_id: String, + pub service: String, + pub tracer_version: String, + pub language: String, + pub language_version: String, + pub language_interpreter: String, + pub language_interpreter_vendor: String, + pub git_commit_sha: String, + pub client_computed_stats: bool, + pub client_computed_top_level: bool, } impl<'a> From<&'a TracerMetadata> for TracerHeaderTags<'a> { @@ -176,6 +177,7 @@ impl<'a> From<&'a TracerMetadata> for TracerHeaderTags<'a> { lang_version: &tags.language_version, tracer_version: &tags.tracer_version, lang_interpreter: &tags.language_interpreter, + lang_vendor: &tags.language_interpreter_vendor, client_computed_stats: tags.client_computed_stats, client_computed_top_level: tags.client_computed_top_level, ..Default::default() @@ -189,23 +191,6 @@ impl<'a> From<&'a TracerMetadata> for HashMap<&'static str, String> { } } -impl From<&TracerMetadata> for LibraryMetadata { - fn from(tags: &TracerMetadata) -> Self { - LibraryMetadata { - hostname: tags.hostname.clone(), - lang: tags.language.clone(), - env: tags.env.clone(), - version: tags.version.clone(), - tracer_version: tags.tracer_version.clone(), - runtime_id: tags.runtime_id.clone(), - service: tags.service.clone(), - git_commit_sha: String::new(), - container_id: String::new(), - tags: Vec::new(), - } - } -} - enum StatsComputationStatus { /// Client-side stats has been disabled by the tracer Disabled, @@ -338,7 +323,7 @@ impl TraceExporter { let mut stats_exporter = stats_exporter::StatsExporter::new( bucket_size, stats_concentrator.clone(), - self.metadata.borrow().into(), + self.metadata.clone(), Endpoint::from_url(add_path(&self.endpoint.url, STATS_ENDPOINT)), cancellation_token.clone(), ); @@ -685,14 +670,16 @@ const DEFAULT_AGENT_URL: &str = "http://127.0.0.1:8126"; #[derive(Default)] pub struct TraceExporterBuilder { url: Option, - tracer_version: String, hostname: String, env: String, - version: String, + app_version: String, service: String, + tracer_version: String, language: String, language_version: String, language_interpreter: String, + language_interpreter_vendor: String, + git_commit_sha: String, input_format: TraceExporterInputFormat, output_format: TraceExporterOutputFormat, response_callback: Option>, @@ -709,7 +696,7 @@ pub struct TraceExporterBuilder { } impl TraceExporterBuilder { - #[allow(missing_docs)] + /// Set url of the agent pub fn set_url(mut self, url: &str) -> Self { self.url = Some(url.to_owned()); self @@ -721,50 +708,71 @@ impl TraceExporterBuilder { self } + /// Set the hostname used for stats payload + /// Only used when client-side stats is enabled pub fn set_hostname(mut self, hostname: &str) -> Self { hostname.clone_into(&mut self.hostname); self } + /// Set the env used for stats payloads + /// Only used when client-side stats is enabled pub fn set_env(mut self, env: &str) -> Self { env.clone_into(&mut self.env); self } - pub fn set_version(mut self, version: &str) -> Self { - version.clone_into(&mut self.version); + /// Set the app version which corresponds to the `version` meta tag + /// Only used when client-side stats is enabled + pub fn set_app_version(mut self, app_version: &str) -> Self { + app_version.clone_into(&mut self.app_version); self } + /// Set the service name used for stats payloads. + /// Only used when client-side stats is enabled pub fn set_service(mut self, service: &str) -> Self { service.clone_into(&mut self.service); self } - #[allow(missing_docs)] + /// Set the `git_commit_sha` corresponding to the `_dd.git.commit.sha` meta tag + /// Only used when client-side stats is enabled + pub fn set_git_commit_sha(mut self, git_commit_sha: &str) -> Self { + git_commit_sha.clone_into(&mut self.git_commit_sha); + self + } + + /// Set the `Datadog-Meta-Tracer-Version` header pub fn set_tracer_version(mut self, tracer_version: &str) -> Self { tracer_version.clone_into(&mut self.tracer_version); self } - #[allow(missing_docs)] + /// Set the `Datadog-Meta-Lang` header pub fn set_language(mut self, lang: &str) -> Self { lang.clone_into(&mut self.language); self } - #[allow(missing_docs)] + /// Set the `Datadog-Meta-Lang-Version` header pub fn set_language_version(mut self, lang_version: &str) -> Self { lang_version.clone_into(&mut self.language_version); self } - #[allow(missing_docs)] + /// Set the `Datadog-Meta-Lang-Interpreter` header pub fn set_language_interpreter(mut self, lang_interpreter: &str) -> Self { lang_interpreter.clone_into(&mut self.language_interpreter); self } + /// Set the `Datadog-Meta-Lang-Interpreter-Vendor` header + pub fn set_language_interpreter_vendor(mut self, lang_interpreter_vendor: &str) -> Self { + lang_interpreter_vendor.clone_into(&mut self.language_interpreter_vendor); + self + } + #[allow(missing_docs)] pub fn set_input_format(mut self, input_format: TraceExporterInputFormat) -> Self { self.input_format = input_format; @@ -858,12 +866,14 @@ impl TraceExporterBuilder { tracer_version: self.tracer_version, language_version: self.language_version, language_interpreter: self.language_interpreter, + language_interpreter_vendor: self.language_interpreter_vendor, language: self.language, + git_commit_sha: self.git_commit_sha, client_computed_stats: self.client_computed_stats, client_computed_top_level: self.client_computed_top_level, hostname: self.hostname, env: self.env, - version: self.version, + app_version: self.app_version, runtime_id: uuid::Uuid::new_v4().to_string(), service: self.service, }, @@ -906,6 +916,8 @@ mod tests { .set_language("nodejs") .set_language_version("1.0") .set_language_interpreter("v8") + .set_language_interpreter_vendor("node") + .set_git_commit_sha("797e9ea") .set_input_format(TraceExporterInputFormat::Proxy) .set_output_format(TraceExporterOutputFormat::V07) .build() @@ -923,6 +935,8 @@ mod tests { assert_eq!(exporter.metadata.language, "nodejs"); assert_eq!(exporter.metadata.language_version, "1.0"); assert_eq!(exporter.metadata.language_interpreter, "v8"); + assert_eq!(exporter.metadata.language_interpreter_vendor, "node"); + assert_eq!(exporter.metadata.git_commit_sha, "797e9ea"); assert!(!exporter.metadata.client_computed_stats); } @@ -960,6 +974,7 @@ mod tests { language: "rust".to_string(), language_version: "1.52.1".to_string(), language_interpreter: "rustc".to_string(), + language_interpreter_vendor: "rust-lang".to_string(), client_computed_stats: true, client_computed_top_level: true, ..Default::default() @@ -971,6 +986,7 @@ mod tests { assert_eq!(tracer_header_tags.lang, "rust"); assert_eq!(tracer_header_tags.lang_version, "1.52.1"); assert_eq!(tracer_header_tags.lang_interpreter, "rustc"); + assert_eq!(tracer_header_tags.lang_vendor, "rust-lang"); assert!(tracer_header_tags.client_computed_stats); assert!(tracer_header_tags.client_computed_top_level); }