From 816b674c9fa77a7611a0e0b8fce23defb8a50411 Mon Sep 17 00:00:00 2001 From: Bruce Guenter Date: Thu, 12 Sep 2024 15:04:27 -0600 Subject: [PATCH 1/5] Break up the test functions --- src/sinks/new_relic/tests.rs | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/sinks/new_relic/tests.rs b/src/sinks/new_relic/tests.rs index dc4b4afee041b..a0a06cb07313b 100644 --- a/src/sinks/new_relic/tests.rs +++ b/src/sinks/new_relic/tests.rs @@ -69,8 +69,7 @@ async fn component_spec_compliance_data_volume() { } #[test] -fn generate_event_api_model() { - // Without message field +fn generates_event_api_model_without_message_field() { let mut map = HashMap::::new(); map.insert("eventType".into(), Value::from("TestEvent".to_owned())); map.insert("user".into(), Value::from("Joe".to_owned())); @@ -92,8 +91,10 @@ fn generate_event_api_model() { ); assert!(model.0[0].contains_key("user_id")); assert_eq!(model.0[0].get("user_id").unwrap(), &Value::Integer(123456)); +} - // With message field +#[test] +fn generates_event_api_model_with_message_field() { let mut map = HashMap::::new(); map.insert("eventType".into(), Value::from("TestEvent".to_owned())); map.insert("user".into(), Value::from("Joe".to_owned())); @@ -124,8 +125,10 @@ fn generate_event_api_model() { model.0[0].get("message").unwrap().to_string_lossy(), "This is a message".to_owned() ); +} - // With a JSON encoded inside the message field +#[test] +fn generates_event_api_model_with_json_inside_message_field() { let mut map = HashMap::::new(); map.insert("eventType".into(), Value::from("TestEvent".to_owned())); map.insert("user".into(), Value::from("Joe".to_owned())); @@ -159,8 +162,7 @@ fn generate_event_api_model() { } #[test] -fn generate_log_api_model() { - // Without message field +fn generates_log_api_model_without_message_field() { let mut map = HashMap::::new(); map.insert("tag_key".into(), Value::from("tag_value".to_owned())); let event = Event::Log(LogEvent::from(map)); @@ -174,8 +176,10 @@ fn generate_log_api_model() { "tag_value".to_owned() ); assert!(logs[0].contains_key("message")); +} - // With message field +#[test] +fn generates_log_api_model_with_message_field() { let mut map = HashMap::::new(); map.insert("tag_key".into(), Value::from("tag_value".to_owned())); map.insert( @@ -200,8 +204,7 @@ fn generate_log_api_model() { } #[test] -fn generate_metric_api_model() { - // Without timestamp +fn generates_metric_api_model_without_timestamp() { let event = Event::Metric(Metric::new( "my_metric", MetricKind::Absolute, @@ -222,8 +225,10 @@ fn generate_metric_api_model() { assert!(metrics[0].contains_key("value")); assert_eq!(metrics[0].get("value").unwrap(), &Value::from(100.0)); assert!(metrics[0].contains_key("timestamp")); +} - // With timestamp +#[test] +fn generates_metric_api_model_with_timestamp() { let m = Metric::new( "my_metric", MetricKind::Absolute, @@ -246,8 +251,10 @@ fn generate_metric_api_model() { assert!(metrics[0].contains_key("value")); assert_eq!(metrics[0].get("value").unwrap(), &Value::from(100.0)); assert!(metrics[0].contains_key("timestamp")); +} - // Incremental counter +#[test] +fn generates_metric_api_model_incremental_counter() { let m = Metric::new( "my_metric", MetricKind::Incremental, From eec498f7e28fb2cdbdb55e3702afe181dad13e43 Mon Sep 17 00:00:00 2001 From: Bruce Guenter Date: Thu, 12 Sep 2024 16:06:51 -0600 Subject: [PATCH 2/5] Rework test comparison --- src/sinks/new_relic/tests.rs | 146 +++++++++++++++-------------------- 1 file changed, 64 insertions(+), 82 deletions(-) diff --git a/src/sinks/new_relic/tests.rs b/src/sinks/new_relic/tests.rs index a0a06cb07313b..61b74bba7eed6 100644 --- a/src/sinks/new_relic/tests.rs +++ b/src/sinks/new_relic/tests.rs @@ -68,6 +68,14 @@ async fn component_spec_compliance_data_volume() { .await; } +macro_rules! model_map { + ( $( $key:literal : $value:expr , )* ) => { + [ $( ( KeyString::from($key), Value::from($value) ), )* ] + .into_iter() + .collect::>() + } +} + #[test] fn generates_event_api_model_without_message_field() { let mut map = HashMap::::new(); @@ -78,19 +86,14 @@ fn generates_event_api_model_without_message_field() { let model = EventsApiModel::try_from(vec![event]).expect("Failed mapping events into API model"); - assert_eq!(model.0.len(), 1); - assert!(model.0[0].contains_key("eventType")); - assert_eq!( - model.0[0].get("eventType").unwrap().to_string_lossy(), - "TestEvent".to_owned() - ); - assert!(model.0[0].contains_key("user")); assert_eq!( - model.0[0].get("user").unwrap().to_string_lossy(), - "Joe".to_owned() + &model.0[..], + &[model_map! { + "eventType": "TestEvent", + "user": "Joe", + "user_id": 123456, + }] ); - assert!(model.0[0].contains_key("user_id")); - assert_eq!(model.0[0].get("user_id").unwrap(), &Value::Integer(123456)); } #[test] @@ -107,23 +110,14 @@ fn generates_event_api_model_with_message_field() { let model = EventsApiModel::try_from(vec![event]).expect("Failed mapping events into API model"); - assert_eq!(model.0.len(), 1); - assert!(model.0[0].contains_key("eventType")); - assert_eq!( - model.0[0].get("eventType").unwrap().to_string_lossy(), - "TestEvent".to_owned() - ); - assert!(model.0[0].contains_key("user")); - assert_eq!( - model.0[0].get("user").unwrap().to_string_lossy(), - "Joe".to_owned() - ); - assert!(model.0[0].contains_key("user_id")); - assert_eq!(model.0[0].get("user_id").unwrap(), &Value::Integer(123456)); - assert!(model.0[0].contains_key("message")); assert_eq!( - model.0[0].get("message").unwrap().to_string_lossy(), - "This is a message".to_owned() + &model.0[..], + &[model_map! { + "eventType": "TestEvent", + "user": "Joe", + "user_id": 123456, + "message": "This is a message", + }] ); } @@ -141,23 +135,14 @@ fn generates_event_api_model_with_json_inside_message_field() { let model = EventsApiModel::try_from(vec![event]).expect("Failed mapping events into API model"); - assert_eq!(model.0.len(), 1); - assert!(model.0[0].contains_key("eventType")); - assert_eq!( - model.0[0].get("eventType").unwrap().to_string_lossy(), - "TestEvent".to_owned() - ); - assert!(model.0[0].contains_key("user")); assert_eq!( - model.0[0].get("user").unwrap().to_string_lossy(), - "Joe".to_owned() - ); - assert!(model.0[0].contains_key("user_id")); - assert_eq!(model.0[0].get("user_id").unwrap(), &Value::Integer(123456)); - assert!(model.0[0].contains_key("my_key")); - assert_eq!( - model.0[0].get("my_key").unwrap().to_string_lossy(), - "my_value".to_owned() + &model.0[..], + &[model_map! { + "eventType": "TestEvent", + "user": "Joe", + "user_id": 123456, + "my_key": "my_value", + }] ); } @@ -169,13 +154,13 @@ fn generates_log_api_model_without_message_field() { let model = LogsApiModel::try_from(vec![event]).expect("Failed mapping logs into API model"); let logs = model.0[0].get("logs").expect("Logs data store not present"); - assert_eq!(logs.len(), 1); - assert!(logs[0].contains_key("tag_key")); assert_eq!( - logs[0].get("tag_key").unwrap().to_string_lossy(), - "tag_value".to_owned() + &logs[..], + &[model_map! { + "tag_key": "tag_value", + "message": "log from vector", + }] ); - assert!(logs[0].contains_key("message")); } #[test] @@ -190,16 +175,12 @@ fn generates_log_api_model_with_message_field() { let model = LogsApiModel::try_from(vec![event]).expect("Failed mapping logs into API model"); let logs = model.0[0].get("logs").expect("Logs data store not present"); - assert_eq!(logs.len(), 1); - assert!(logs[0].contains_key("tag_key")); - assert_eq!( - logs[0].get("tag_key").unwrap().to_string_lossy(), - "tag_value".to_owned() - ); - assert!(logs[0].contains_key("message")); assert_eq!( - logs[0].get("message").unwrap().to_string_lossy(), - "This is a message".to_owned() + &logs[..], + &[model_map! { + "tag_key": "tag_value", + "message": "This is a message", + }] ); } @@ -216,25 +197,26 @@ fn generates_metric_api_model_without_timestamp() { .get("metrics") .expect("Metric data store not present"); - assert_eq!(metrics.len(), 1); - assert!(metrics[0].contains_key("name")); assert_eq!( - metrics[0].get("name").unwrap().to_string_lossy(), - "my_metric".to_owned() + &metrics[..], + &[model_map! { + "name": "my_metric", + "value": 100.0, + "timestamp": metrics[0].get("timestamp").unwrap().clone(), + "type": "gauge", + }] ); - assert!(metrics[0].contains_key("value")); - assert_eq!(metrics[0].get("value").unwrap(), &Value::from(100.0)); - assert!(metrics[0].contains_key("timestamp")); } #[test] fn generates_metric_api_model_with_timestamp() { + let stamp = DateTime::::from(SystemTime::now()); let m = Metric::new( "my_metric", MetricKind::Absolute, MetricValue::Counter { value: 100.0 }, ) - .with_timestamp(Some(DateTime::::from(SystemTime::now()))); + .with_timestamp(Some(stamp)); let event = Event::Metric(m); let model = MetricsApiModel::try_from(vec![event]).expect("Failed mapping metrics into API model"); @@ -242,25 +224,26 @@ fn generates_metric_api_model_with_timestamp() { .get("metrics") .expect("Metric data store not present"); - assert_eq!(metrics.len(), 1); - assert!(metrics[0].contains_key("name")); assert_eq!( - metrics[0].get("name").unwrap().to_string_lossy(), - "my_metric".to_owned() + &metrics[..], + &[model_map! { + "name": "my_metric", + "value": 100.0, + "timestamp": stamp.timestamp(), + "type": "gauge", + }] ); - assert!(metrics[0].contains_key("value")); - assert_eq!(metrics[0].get("value").unwrap(), &Value::from(100.0)); - assert!(metrics[0].contains_key("timestamp")); } #[test] fn generates_metric_api_model_incremental_counter() { + let stamp = DateTime::::from(SystemTime::now()); let m = Metric::new( "my_metric", MetricKind::Incremental, MetricValue::Counter { value: 100.0 }, ) - .with_timestamp(Some(DateTime::::from(SystemTime::now()))) + .with_timestamp(Some(stamp)) .with_interval_ms(NonZeroU32::new(1000)); let event = Event::Metric(m); let model = @@ -269,15 +252,14 @@ fn generates_metric_api_model_incremental_counter() { .get("metrics") .expect("Metric data store not present"); - assert_eq!(metrics.len(), 1); - assert!(metrics[0].contains_key("name")); assert_eq!( - metrics[0].get("name").unwrap().to_string_lossy(), - "my_metric".to_owned() + &metrics[..], + &[model_map! { + "name": "my_metric", + "value": 100.0, + "interval.ms": 1000, + "timestamp": stamp.timestamp(), + "type": "count", + }] ); - assert!(metrics[0].contains_key("value")); - assert_eq!(metrics[0].get("value").unwrap(), &Value::from(100.0)); - assert!(metrics[0].contains_key("timestamp")); - assert!(metrics[0].contains_key("interval.ms")); - assert_eq!(metrics[0].get("interval.ms").unwrap(), &Value::from(1000)); } From e1046f5f7ffafacd2e7f424dd11aa0fc9c6248a2 Mon Sep 17 00:00:00 2001 From: Bruce Guenter Date: Fri, 13 Sep 2024 12:14:50 -0600 Subject: [PATCH 3/5] Use `vrl::value!` to create log event values --- src/sinks/new_relic/tests.rs | 56 +++++++++++++++--------------------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/src/sinks/new_relic/tests.rs b/src/sinks/new_relic/tests.rs index 61b74bba7eed6..b64629334daf0 100644 --- a/src/sinks/new_relic/tests.rs +++ b/src/sinks/new_relic/tests.rs @@ -4,6 +4,7 @@ use chrono::{DateTime, Utc}; use futures::{future::ready, stream}; use serde::Deserialize; use vector_lib::config::{init_telemetry, Tags, Telemetry}; +use vrl::value; use super::*; use crate::{ @@ -78,11 +79,11 @@ macro_rules! model_map { #[test] fn generates_event_api_model_without_message_field() { - let mut map = HashMap::::new(); - map.insert("eventType".into(), Value::from("TestEvent".to_owned())); - map.insert("user".into(), Value::from("Joe".to_owned())); - map.insert("user_id".into(), Value::from(123456)); - let event = Event::Log(LogEvent::from(map)); + let event = Event::Log(LogEvent::from(value!({ + "eventType": "TestEvent", + "user": "Joe", + "user_id": 123456, + }))); let model = EventsApiModel::try_from(vec![event]).expect("Failed mapping events into API model"); @@ -98,15 +99,12 @@ fn generates_event_api_model_without_message_field() { #[test] fn generates_event_api_model_with_message_field() { - let mut map = HashMap::::new(); - map.insert("eventType".into(), Value::from("TestEvent".to_owned())); - map.insert("user".into(), Value::from("Joe".to_owned())); - map.insert("user_id".into(), Value::from(123456)); - map.insert( - "message".into(), - Value::from("This is a message".to_owned()), - ); - let event = Event::Log(LogEvent::from(map)); + let event = Event::Log(LogEvent::from(value!({ + "eventType": "TestEvent", + "user": "Joe", + "user_id": 123456, + "message": "This is a message", + }))); let model = EventsApiModel::try_from(vec![event]).expect("Failed mapping events into API model"); @@ -123,15 +121,12 @@ fn generates_event_api_model_with_message_field() { #[test] fn generates_event_api_model_with_json_inside_message_field() { - let mut map = HashMap::::new(); - map.insert("eventType".into(), Value::from("TestEvent".to_owned())); - map.insert("user".into(), Value::from("Joe".to_owned())); - map.insert("user_id".into(), Value::from(123456)); - map.insert( - "message".into(), - Value::from("{\"my_key\" : \"my_value\"}".to_owned()), - ); - let event = Event::Log(LogEvent::from(map)); + let event = Event::Log(LogEvent::from(value!({ + "eventType": "TestEvent", + "user": "Joe", + "user_id": 123456, + "message": "{\"my_key\" : \"my_value\"}", + }))); let model = EventsApiModel::try_from(vec![event]).expect("Failed mapping events into API model"); @@ -148,9 +143,7 @@ fn generates_event_api_model_with_json_inside_message_field() { #[test] fn generates_log_api_model_without_message_field() { - let mut map = HashMap::::new(); - map.insert("tag_key".into(), Value::from("tag_value".to_owned())); - let event = Event::Log(LogEvent::from(map)); + let event = Event::Log(LogEvent::from(value!({"tag_key": "tag_value"}))); let model = LogsApiModel::try_from(vec![event]).expect("Failed mapping logs into API model"); let logs = model.0[0].get("logs").expect("Logs data store not present"); @@ -165,13 +158,10 @@ fn generates_log_api_model_without_message_field() { #[test] fn generates_log_api_model_with_message_field() { - let mut map = HashMap::::new(); - map.insert("tag_key".into(), Value::from("tag_value".to_owned())); - map.insert( - "message".into(), - Value::from("This is a message".to_owned()), - ); - let event = Event::Log(LogEvent::from(map)); + let event = Event::Log(LogEvent::from(value!({ + "tag_key": "tag_value", + "message": "This is a message", + }))); let model = LogsApiModel::try_from(vec![event]).expect("Failed mapping logs into API model"); let logs = model.0[0].get("logs").expect("Logs data store not present"); From cd921b98ecc007d88f75d3b6b11a80307f158271 Mon Sep 17 00:00:00 2001 From: Bruce Guenter Date: Tue, 17 Sep 2024 12:36:24 -0600 Subject: [PATCH 4/5] Fix encoding of dotted field names through allowing nesting This converts log event into a logs API model simply by transmuting the type wrapper and dropping all arrays, which are not supported by the API. We could flatten out the keys, as this is what New Relic does internally, and we used to do that, but the flattening iterator accessed through `LogEvent::convert_to_fields` adds quotes to dotted fields names, which produces broken attributes in New Relic, and nesting objects is actually a (slightly) more efficient representation of the key names. --- ...newrelic-log-attribute-dotted-names.fix.md | 1 + src/sinks/new_relic/model.rs | 61 +++++++++++++------ src/sinks/new_relic/tests.rs | 24 +++++++- 3 files changed, 66 insertions(+), 20 deletions(-) create mode 100644 changelog.d/21304-newrelic-log-attribute-dotted-names.fix.md diff --git a/changelog.d/21304-newrelic-log-attribute-dotted-names.fix.md b/changelog.d/21304-newrelic-log-attribute-dotted-names.fix.md new file mode 100644 index 0000000000000..81b2495e33ac2 --- /dev/null +++ b/changelog.d/21304-newrelic-log-attribute-dotted-names.fix.md @@ -0,0 +1 @@ +Fixed a bug in the `new_relic` sinks that caused dotted attribute names to be encoded incorrectly with quotes. diff --git a/src/sinks/new_relic/model.rs b/src/sinks/new_relic/model.rs index da6f402061f5e..376ecdfff0ea5 100644 --- a/src/sinks/new_relic/model.rs +++ b/src/sinks/new_relic/model.rs @@ -8,6 +8,7 @@ use std::{ use chrono::{DateTime, Utc}; use ordered_float::NotNan; use serde::{Deserialize, Serialize}; +use vector_lib::event::ObjectMap; use vector_lib::internal_event::{ComponentEventsDropped, INTENTIONAL, UNINTENTIONAL}; use vrl::event_path; @@ -21,14 +22,13 @@ pub enum NewRelicApiModel { Logs(LogsApiModel), } -type KeyValData = HashMap; -type DataStore = HashMap>; +type DataStore = HashMap>; #[derive(Serialize, Deserialize, Debug)] pub struct MetricsApiModel(pub Vec); impl MetricsApiModel { - pub fn new(metric_array: Vec) -> Self { + pub fn new(metric_array: Vec) -> Self { let mut metric_store = DataStore::new(); metric_store.insert("metrics".into(), metric_array); Self(vec![metric_store]) @@ -55,7 +55,7 @@ impl TryFrom> for MetricsApiModel { // Generate Value::Object() from BTreeMap let (series, data, _) = metric.into_parts(); - let mut metric_data = KeyValData::new(); + let mut metric_data = ObjectMap::new(); // We only handle gauge and counter metrics // Extract value & type and set type-related attributes @@ -145,10 +145,10 @@ impl TryFrom> for MetricsApiModel { } #[derive(Serialize, Deserialize, Debug)] -pub struct EventsApiModel(pub Vec); +pub struct EventsApiModel(pub Vec); impl EventsApiModel { - pub fn new(events_array: Vec) -> Self { + pub fn new(events_array: Vec) -> Self { Self(events_array) } } @@ -160,7 +160,7 @@ impl TryFrom> for EventsApiModel { let mut num_non_log_events = 0; let mut num_nan_value = 0; - let events_array: Vec> = buf_events + let events_array: Vec = buf_events .into_iter() .filter_map(|event| { let Some(log) = event.try_into_log() else { @@ -168,7 +168,7 @@ impl TryFrom> for EventsApiModel { return None; }; - let mut event_model = KeyValData::new(); + let mut event_model = ObjectMap::new(); for (k, v) in log.convert_to_fields() { event_model.insert(k, v.clone()); } @@ -242,7 +242,7 @@ impl TryFrom> for EventsApiModel { pub struct LogsApiModel(pub Vec); impl LogsApiModel { - pub fn new(logs_array: Vec) -> Self { + pub fn new(logs_array: Vec) -> Self { let mut logs_store = DataStore::new(); logs_store.insert("logs".into(), logs_array); Self(vec![logs_store]) @@ -254,8 +254,9 @@ impl TryFrom> for LogsApiModel { fn try_from(buf_events: Vec) -> Result { let mut num_non_log_events = 0; + let mut num_non_object_events = 0; - let logs_array: Vec> = buf_events + let logs_array: Vec = buf_events .into_iter() .filter_map(|event| { let Some(log) = event.try_into_log() else { @@ -263,22 +264,37 @@ impl TryFrom> for LogsApiModel { return None; }; - let mut log_model = KeyValData::new(); - for (k, v) in log.convert_to_fields() { - log_model.insert(k, v.clone()); - } - if log.get(event_path!("message")).is_none() { - log_model.insert("message".into(), Value::from("log from vector".to_owned())); + // We convert the log event into a logs API model simply by transmuting the type + // wrapper and dropping all arrays, which are not supported by the API. We could + // flatten out the keys, as this is what New Relic does internally, and we used to + // do that, but the flattening iterator accessed through + // `LogEvent::convert_to_fields` adds quotes to dotted fields names, which produces + // broken attributes in New Relic, and nesting objects is actually a (slightly) more + // efficient representation of the key names. + let (value, _metadata) = log.into_parts(); + let Some(mut obj) = value.into_object() else { + num_non_object_events += 1; + return None; + }; + strip_arrays(&mut obj); + if !obj.contains_key("message") { + obj.insert("message".into(), Value::from("log from vector".to_owned())); } - Some(log_model) + Some(obj) }) .collect(); if num_non_log_events > 0 { emit!(ComponentEventsDropped:: { count: num_non_log_events, - reason: "non-log event" + reason: "non-log event", + }); + } + if num_non_object_events > 0 { + emit!(ComponentEventsDropped:: { + count: num_non_object_events, + reason: "non-object event", }); } @@ -289,3 +305,12 @@ impl TryFrom> for LogsApiModel { } } } + +fn strip_arrays(obj: &mut ObjectMap) { + obj.retain(|_key, value| !value.is_array()); + obj.iter_mut().for_each(|(_key, value)| { + if let Some(obj) = value.as_object_mut() { + strip_arrays(obj); + } + }); +} diff --git a/src/sinks/new_relic/tests.rs b/src/sinks/new_relic/tests.rs index b64629334daf0..78f3e55db68df 100644 --- a/src/sinks/new_relic/tests.rs +++ b/src/sinks/new_relic/tests.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, convert::TryFrom, num::NonZeroU32, time::SystemTime}; +use std::{collections::BTreeMap, convert::TryFrom, num::NonZeroU32, time::SystemTime}; use chrono::{DateTime, Utc}; use futures::{future::ready, stream}; @@ -73,7 +73,7 @@ macro_rules! model_map { ( $( $key:literal : $value:expr , )* ) => { [ $( ( KeyString::from($key), Value::from($value) ), )* ] .into_iter() - .collect::>() + .collect::>() } } @@ -174,6 +174,26 @@ fn generates_log_api_model_with_message_field() { ); } +#[test] +fn generates_log_api_model_with_dotted_fields() { + let sub = value!({"four": 2}); + let event = Event::Log(LogEvent::from(value!({ + "one.two": 1, + "three": sub, + }))); + let model = LogsApiModel::try_from(vec![event]).expect("Failed mapping logs into API model"); + let logs = model.0[0].get("logs").expect("Logs data store not present"); + + assert_eq!( + &logs[..], + &[model_map! { + "one.two": 1, + "three": model_map! {"four": 2,}, + "message": "log from vector", + }] + ); +} + #[test] fn generates_metric_api_model_without_timestamp() { let event = Event::Metric(Metric::new( From ab41ee378619820206baa655a7fb982967a2a123 Mon Sep 17 00:00:00 2001 From: Bruce Guenter Date: Tue, 17 Sep 2024 12:53:47 -0600 Subject: [PATCH 5/5] Drop redundant `model_map!` test macro --- src/sinks/new_relic/tests.rs | 94 +++++++++++++++++------------------- 1 file changed, 43 insertions(+), 51 deletions(-) diff --git a/src/sinks/new_relic/tests.rs b/src/sinks/new_relic/tests.rs index 78f3e55db68df..66be6888ec984 100644 --- a/src/sinks/new_relic/tests.rs +++ b/src/sinks/new_relic/tests.rs @@ -1,15 +1,15 @@ -use std::{collections::BTreeMap, convert::TryFrom, num::NonZeroU32, time::SystemTime}; +use std::{convert::TryFrom, num::NonZeroU32, time::SystemTime}; use chrono::{DateTime, Utc}; use futures::{future::ready, stream}; use serde::Deserialize; use vector_lib::config::{init_telemetry, Tags, Telemetry}; -use vrl::value; +use vrl::{btreemap, value}; use super::*; use crate::{ config::{GenerateConfig, SinkConfig, SinkContext}, - event::{Event, KeyString, LogEvent, Metric, MetricKind, MetricValue, Value}, + event::{Event, LogEvent, Metric, MetricKind, MetricValue}, test_util::{ components::{ run_and_assert_data_volume_sink_compliance, run_and_assert_sink_compliance, @@ -69,14 +69,6 @@ async fn component_spec_compliance_data_volume() { .await; } -macro_rules! model_map { - ( $( $key:literal : $value:expr , )* ) => { - [ $( ( KeyString::from($key), Value::from($value) ), )* ] - .into_iter() - .collect::>() - } -} - #[test] fn generates_event_api_model_without_message_field() { let event = Event::Log(LogEvent::from(value!({ @@ -89,10 +81,10 @@ fn generates_event_api_model_without_message_field() { assert_eq!( &model.0[..], - &[model_map! { - "eventType": "TestEvent", - "user": "Joe", - "user_id": 123456, + &[btreemap! { + "eventType" => "TestEvent", + "user" => "Joe", + "user_id" => 123456, }] ); } @@ -110,11 +102,11 @@ fn generates_event_api_model_with_message_field() { assert_eq!( &model.0[..], - &[model_map! { - "eventType": "TestEvent", - "user": "Joe", - "user_id": 123456, - "message": "This is a message", + &[btreemap! { + "eventType" =>"TestEvent", + "user" =>"Joe", + "user_id" =>123456, + "message" =>"This is a message", }] ); } @@ -132,11 +124,11 @@ fn generates_event_api_model_with_json_inside_message_field() { assert_eq!( &model.0[..], - &[model_map! { - "eventType": "TestEvent", - "user": "Joe", - "user_id": 123456, - "my_key": "my_value", + &[btreemap! { + "eventType" =>"TestEvent", + "user" =>"Joe", + "user_id" =>123456, + "my_key" =>"my_value", }] ); } @@ -149,9 +141,9 @@ fn generates_log_api_model_without_message_field() { assert_eq!( &logs[..], - &[model_map! { - "tag_key": "tag_value", - "message": "log from vector", + &[btreemap! { + "tag_key" =>"tag_value", + "message" =>"log from vector", }] ); } @@ -167,9 +159,9 @@ fn generates_log_api_model_with_message_field() { assert_eq!( &logs[..], - &[model_map! { - "tag_key": "tag_value", - "message": "This is a message", + &[btreemap! { + "tag_key" =>"tag_value", + "message" =>"This is a message", }] ); } @@ -186,10 +178,10 @@ fn generates_log_api_model_with_dotted_fields() { assert_eq!( &logs[..], - &[model_map! { - "one.two": 1, - "three": model_map! {"four": 2,}, - "message": "log from vector", + &[btreemap! { + "one.two" =>1, + "three" =>btreemap! {"four" =>2,}, + "message" =>"log from vector", }] ); } @@ -209,11 +201,11 @@ fn generates_metric_api_model_without_timestamp() { assert_eq!( &metrics[..], - &[model_map! { - "name": "my_metric", - "value": 100.0, - "timestamp": metrics[0].get("timestamp").unwrap().clone(), - "type": "gauge", + &[btreemap! { + "name" =>"my_metric", + "value" =>100.0, + "timestamp" =>metrics[0].get("timestamp").unwrap().clone(), + "type" =>"gauge", }] ); } @@ -236,11 +228,11 @@ fn generates_metric_api_model_with_timestamp() { assert_eq!( &metrics[..], - &[model_map! { - "name": "my_metric", - "value": 100.0, - "timestamp": stamp.timestamp(), - "type": "gauge", + &[btreemap! { + "name" =>"my_metric", + "value" =>100.0, + "timestamp" =>stamp.timestamp(), + "type" =>"gauge", }] ); } @@ -264,12 +256,12 @@ fn generates_metric_api_model_incremental_counter() { assert_eq!( &metrics[..], - &[model_map! { - "name": "my_metric", - "value": 100.0, - "interval.ms": 1000, - "timestamp": stamp.timestamp(), - "type": "count", + &[btreemap! { + "name" =>"my_metric", + "value" =>100.0, + "interval.ms" =>1000, + "timestamp" =>stamp.timestamp(), + "type" =>"count", }] ); }