feat(events): add incoming webhook payload to api events logger (#2852)

Co-authored-by: Sampras lopes <[email protected]>

connector-template/mod.rs

@@ -485,7 +485,7 @@ impl api::IncomingWebhook for {{project-name | downcase | pascal_case}} {
     fn get_webhook_resource_object(
         &self,
         _request: &api::IncomingWebhookRequestDetails<'_>,
-    ) -> CustomResult<serde_json::Value, errors::ConnectorError> {
+    ) -> CustomResult<Box<dyn erased_serde::Serialize>, errors::ConnectorError> {
         Err(errors::ConnectorError::WebhooksNotImplemented).into_report()
     }
 }

crates/common_utils/src/ext_traits.rs

@@ -223,6 +223,7 @@ pub trait ByteSliceExt {
 }
 
 impl ByteSliceExt for [u8] {
+    #[track_caller]
     fn parse_struct<'de, T>(
         &'de self,
         type_name: &'static str,

crates/masking/Cargo.toml

@@ -10,6 +10,7 @@ license.workspace = true
 [features]
 default = ["alloc", "serde", "diesel"]
 alloc = ["zeroize/alloc"]
+serde = ["dep:serde", "dep:serde_json"]
 
 [package.metadata.docs.rs]
 all-features = true
@@ -19,7 +20,7 @@ rustdoc-args = ["--cfg", "docsrs"]
 bytes = { version = "1", optional = true }
 diesel = { version = "2.1.0", features = ["postgres", "serde_json", "time"], optional = true }
 serde = { version = "1", features = ["derive"], optional = true }
-serde_json = "1.0.96"
+serde_json = { version = "1.0.96", optional = true }
 subtle = "=2.4.1"
 zeroize = { version = "1.6", default-features = false }
 

crates/masking/src/lib.rs

@@ -42,7 +42,9 @@ mod vec;
 #[cfg(feature = "serde")]
 mod serde;
 #[cfg(feature = "serde")]
-pub use crate::serde::{masked_serialize, Deserialize, SerializableSecret, Serialize};
+pub use crate::serde::{
+    masked_serialize, Deserialize, ErasedMaskSerialize, SerializableSecret, Serialize,
+};
 
 /// This module should be included with asterisk.
 ///

crates/masking/src/serde.rs

@@ -91,6 +91,31 @@ pub fn masked_serialize<T: Serialize>(value: &T) -> Result<Value, serde_json::Er
     })
 }
 
+///
+/// Masked serialization.
+///
+/// Trait object for supporting serialization to Value while accounting for masking
+/// The usual Serde Serialize trait cannot be used as trait objects
+/// like &dyn Serialize or boxed trait objects like Box<dyn Serialize> because of Rust's "object safety" rules.
+/// In particular, the trait contains generic methods which cannot be made into a trait object.
+/// In this case we remove the generic for assuming the serialization to be of 2 types only raw json or masked json
+pub trait ErasedMaskSerialize {
+    /// Masked serialization.
+    fn masked_serialize(&self) -> Result<Value, serde_json::Error>;
+    /// Normal serialization.
+    fn raw_serialize(&self) -> Result<Value, serde_json::Error>;
+}
+
+impl<T: Serialize> ErasedMaskSerialize for T {
+    fn masked_serialize(&self) -> Result<Value, serde_json::Error> {
+        masked_serialize(self)
+    }
+
+    fn raw_serialize(&self) -> Result<Value, serde_json::Error> {
+        serde_json::to_value(self)
+    }
+}
+
 use pii_serializer::PIISerializer;
 
 mod pii_serializer {

crates/router/Cargo.toml

@@ -115,6 +115,7 @@ router_derive = { version = "0.1.0", path = "../router_derive" }
 router_env = { version = "0.1.0", path = "../router_env", features = ["log_extra_implicit_fields", "log_custom_entries_to_extra"] }
 scheduler = { version = "0.1.0", path = "../scheduler", default-features = false }
 storage_impl = { version = "0.1.0", path = "../storage_impl", default-features = false }
+erased-serde = "0.3.31"
 
 [build-dependencies]
 router_env = { version = "0.1.0", path = "../router_env", default-features = false }

crates/router/src/connector/aci.rs

@@ -572,7 +572,7 @@ impl api::IncomingWebhook for Aci {
     fn get_webhook_resource_object(
         &self,
         _request: &api::IncomingWebhookRequestDetails<'_>,
-    ) -> CustomResult<serde_json::Value, errors::ConnectorError> {
+    ) -> CustomResult<Box<dyn masking::ErasedMaskSerialize>, errors::ConnectorError> {
         Err(errors::ConnectorError::WebhooksNotImplemented).into_report()
     }
 }

crates/router/src/connector/adyen.rs

@@ -1600,17 +1600,13 @@ impl api::IncomingWebhook for Adyen {
     fn get_webhook_resource_object(
         &self,
         request: &api::IncomingWebhookRequestDetails<'_>,
-    ) -> CustomResult<serde_json::Value, errors::ConnectorError> {
+    ) -> CustomResult<Box<dyn masking::ErasedMaskSerialize>, errors::ConnectorError> {
         let notif = get_webhook_object_from_body(request.body)
             .change_context(errors::ConnectorError::WebhookEventTypeNotFound)?;
 
         let response: adyen::Response = notif.into();
 
-        let res_json = serde_json::to_value(response)
-            .into_report()
-            .change_context(errors::ConnectorError::WebhookResourceObjectNotFound)?;
-
-        Ok(res_json)
+        Ok(Box::new(response))
     }
 
     fn get_webhook_api_response(

crates/router/src/connector/airwallex.rs

@@ -1081,13 +1081,13 @@ impl api::IncomingWebhook for Airwallex {
     fn get_webhook_resource_object(
         &self,
         request: &api::IncomingWebhookRequestDetails<'_>,
-    ) -> CustomResult<serde_json::Value, errors::ConnectorError> {
+    ) -> CustomResult<Box<dyn masking::ErasedMaskSerialize>, errors::ConnectorError> {
         let details: airwallex::AirwallexWebhookObjectResource = request
             .body
             .parse_struct("AirwallexWebhookObjectResource")
             .change_context(errors::ConnectorError::WebhookResourceObjectNotFound)?;
 
-        Ok(details.data.object)
+        Ok(Box::new(details.data.object))
     }
 
     fn get_dispute_details(

crates/router/src/connector/airwallex/transformers.rs

@@ -824,7 +824,8 @@ pub enum AirwallexDisputeStage {
 
 #[derive(Debug, Deserialize)]
 pub struct AirwallexWebhookDataResource {
-    pub object: serde_json::Value,
+    // Should this be a secret by default since it represents webhook payload
+    pub object: Secret<serde_json::Value>,
 }
 
 #[derive(Debug, Deserialize)]

crates/router/src/connector/authorizedotnet.rs

@@ -875,17 +875,15 @@ impl api::IncomingWebhook for Authorizedotnet {
     fn get_webhook_resource_object(
         &self,
         request: &api::IncomingWebhookRequestDetails<'_>,
-    ) -> CustomResult<serde_json::Value, errors::ConnectorError> {
+    ) -> CustomResult<Box<dyn masking::ErasedMaskSerialize>, errors::ConnectorError> {
         let payload: authorizedotnet::AuthorizedotnetWebhookObjectId = request
             .body
             .parse_struct("AuthorizedotnetWebhookObjectId")
             .change_context(errors::ConnectorError::WebhookResourceObjectNotFound)?;
-        let sync_payload = serde_json::to_value(
+
+        Ok(Box::new(
             authorizedotnet::AuthorizedotnetSyncResponse::try_from(payload)?,
-        )
-        .into_report()
-        .change_context(errors::ConnectorError::ResponseHandlingFailed)?;
-        Ok(sync_payload)
+        ))
     }
 }
 

crates/router/src/connector/bambora.rs

@@ -685,7 +685,7 @@ impl api::IncomingWebhook for Bambora {
     fn get_webhook_resource_object(
         &self,
         _request: &api::IncomingWebhookRequestDetails<'_>,
-    ) -> CustomResult<serde_json::Value, errors::ConnectorError> {
+    ) -> CustomResult<Box<dyn masking::ErasedMaskSerialize>, errors::ConnectorError> {
         Err(errors::ConnectorError::WebhooksNotImplemented).into_report()
     }
 }

crates/router/src/connector/bankofamerica.rs

@@ -812,7 +812,7 @@ impl api::IncomingWebhook for Bankofamerica {
     fn get_webhook_resource_object(
         &self,
         _request: &api::IncomingWebhookRequestDetails<'_>,
-    ) -> CustomResult<serde_json::Value, errors::ConnectorError> {
+    ) -> CustomResult<Box<dyn masking::ErasedMaskSerialize>, errors::ConnectorError> {
         Err(errors::ConnectorError::WebhooksNotImplemented).into_report()
     }
 }

crates/router/src/connector/bitpay.rs

@@ -23,7 +23,7 @@ use crate::{
         api::{self, ConnectorCommon, ConnectorCommonExt},
         ErrorResponse, Response,
     },
-    utils::{self, BytesExt, Encode},
+    utils::{self, BytesExt},
 };
 
 #[derive(Debug, Clone)]
@@ -393,12 +393,11 @@ impl api::IncomingWebhook for Bitpay {
     fn get_webhook_resource_object(
         &self,
         request: &api::IncomingWebhookRequestDetails<'_>,
-    ) -> CustomResult<serde_json::Value, errors::ConnectorError> {
+    ) -> CustomResult<Box<dyn masking::ErasedMaskSerialize>, errors::ConnectorError> {
         let notif: BitpayWebhookDetails = request
             .body
             .parse_struct("BitpayWebhookDetails")
             .change_context(errors::ConnectorError::WebhookEventTypeNotFound)?;
-        Encode::<BitpayWebhookDetails>::encode_to_value(&notif)
-            .change_context(errors::ConnectorError::WebhookBodyDecodingFailed)
+        Ok(Box::new(notif))
     }
 }

crates/router/src/connector/bluesnap.rs

@@ -1119,15 +1119,13 @@ impl api::IncomingWebhook for Bluesnap {
     fn get_webhook_resource_object(
         &self,
         request: &api::IncomingWebhookRequestDetails<'_>,
-    ) -> CustomResult<serde_json::Value, errors::ConnectorError> {
+    ) -> CustomResult<Box<dyn masking::ErasedMaskSerialize>, errors::ConnectorError> {
         let resource: bluesnap::BluesnapWebhookObjectResource =
             serde_urlencoded::from_bytes(request.body)
                 .into_report()
                 .change_context(errors::ConnectorError::WebhookResourceObjectNotFound)?;
 
-        let res_json = serde_json::Value::try_from(resource)?;
-
-        Ok(res_json)
+        Ok(Box::new(resource))
     }
 }
 

crates/router/src/connector/boku.rs

@@ -627,7 +627,7 @@ impl api::IncomingWebhook for Boku {
     fn get_webhook_resource_object(
         &self,
         _request: &api::IncomingWebhookRequestDetails<'_>,
-    ) -> CustomResult<serde_json::Value, errors::ConnectorError> {
+    ) -> CustomResult<Box<dyn masking::ErasedMaskSerialize>, errors::ConnectorError> {
         Err(errors::ConnectorError::WebhooksNotImplemented).into_report()
     }
 }

crates/router/src/connector/braintree.rs

@@ -1418,17 +1418,13 @@ impl api::IncomingWebhook for Braintree {
     fn get_webhook_resource_object(
         &self,
         request: &api::IncomingWebhookRequestDetails<'_>,
-    ) -> CustomResult<serde_json::Value, errors::ConnectorError> {
+    ) -> CustomResult<Box<dyn masking::ErasedMaskSerialize>, errors::ConnectorError> {
         let notif = get_webhook_object_from_body(request.body)
             .change_context(errors::ConnectorError::WebhookBodyDecodingFailed)?;
 
         let response = decode_webhook_payload(notif.bt_payload.replace('\n', "").as_bytes())?;
 
-        let res_json = serde_json::to_value(response)
-            .into_report()
-            .change_context(errors::ConnectorError::WebhookResourceObjectNotFound)?;
-
-        Ok(res_json)
+        Ok(Box::new(response))
     }
 
     fn get_webhook_api_response(

crates/router/src/connector/cashtocode.rs

@@ -391,16 +391,13 @@ impl api::IncomingWebhook for Cashtocode {
     fn get_webhook_resource_object(
         &self,
         request: &api::IncomingWebhookRequestDetails<'_>,
-    ) -> CustomResult<serde_json::Value, errors::ConnectorError> {
+    ) -> CustomResult<Box<dyn masking::ErasedMaskSerialize>, errors::ConnectorError> {
         let webhook: transformers::CashtocodeIncomingWebhook = request
             .body
             .parse_struct("CashtocodeIncomingWebhook")
             .change_context(errors::ConnectorError::WebhookBodyDecodingFailed)?;
-        let res_json =
-            utils::Encode::<transformers::CashtocodeIncomingWebhook>::encode_to_value(&webhook)
-                .change_context(errors::ConnectorError::WebhookBodyDecodingFailed)?;
 
-        Ok(res_json)
+        Ok(Box::new(webhook))
     }
 
     fn get_webhook_api_response(

crates/router/src/connector/checkout.rs

@@ -1261,7 +1261,7 @@ impl api::IncomingWebhook for Checkout {
     fn get_webhook_resource_object(
         &self,
         request: &api::IncomingWebhookRequestDetails<'_>,
-    ) -> CustomResult<serde_json::Value, errors::ConnectorError> {
+    ) -> CustomResult<Box<dyn masking::ErasedMaskSerialize>, errors::ConnectorError> {
         let event_type_data: checkout::CheckoutWebhookEventTypeBody = request
             .body
             .parse_struct("CheckoutWebhookBody")
@@ -1281,7 +1281,10 @@ impl api::IncomingWebhook for Checkout {
             utils::Encode::<checkout::PaymentsResponse>::encode_to_value(&payment_response)
                 .change_context(errors::ConnectorError::WebhookResourceObjectNotFound)?
         };
-        Ok(resource_object)
+        // Ideally this should be a strict type that has type information
+        // PII information is likely being logged here when this response will be logged.
+
+        Ok(Box::new(resource_object))
     }
 
     fn get_dispute_details(

crates/router/src/connector/coinbase.rs

@@ -426,12 +426,12 @@ impl api::IncomingWebhook for Coinbase {
     fn get_webhook_resource_object(
         &self,
         request: &api::IncomingWebhookRequestDetails<'_>,
-    ) -> CustomResult<serde_json::Value, errors::ConnectorError> {
+    ) -> CustomResult<Box<dyn masking::ErasedMaskSerialize>, errors::ConnectorError> {
         let notif: CoinbaseWebhookDetails = request
             .body
             .parse_struct("CoinbaseWebhookDetails")
             .change_context(errors::ConnectorError::WebhookBodyDecodingFailed)?;
-        Encode::<CoinbaseWebhookDetails>::encode_to_value(&notif.event)
-            .change_context(errors::ConnectorError::WebhookBodyDecodingFailed)
+
+        Ok(Box::new(notif.event))
     }
 }

crates/router/src/connector/cryptopay.rs

@@ -455,13 +455,13 @@ impl api::IncomingWebhook for Cryptopay {
     fn get_webhook_resource_object(
         &self,
         request: &api::IncomingWebhookRequestDetails<'_>,
-    ) -> CustomResult<serde_json::Value, errors::ConnectorError> {
+    ) -> CustomResult<Box<dyn masking::ErasedMaskSerialize>, errors::ConnectorError> {
         let notif: CryptopayWebhookDetails =
             request
                 .body
                 .parse_struct("CryptopayWebhookDetails")
                 .change_context(errors::ConnectorError::WebhookBodyDecodingFailed)?;
-        Encode::<CryptopayWebhookDetails>::encode_to_value(&notif)
-            .change_context(errors::ConnectorError::WebhookBodyDecodingFailed)
+
+        Ok(Box::new(notif))
     }
 }

crates/router/src/connector/cybersource.rs

@@ -805,7 +805,7 @@ impl api::IncomingWebhook for Cybersource {
     fn get_webhook_resource_object(
         &self,
         _request: &api::IncomingWebhookRequestDetails<'_>,
-    ) -> CustomResult<serde_json::Value, errors::ConnectorError> {
+    ) -> CustomResult<Box<dyn masking::ErasedMaskSerialize>, errors::ConnectorError> {
         Err(errors::ConnectorError::WebhooksNotImplemented).into_report()
     }
 }

crates/router/src/connector/dlocal.rs

@@ -674,7 +674,7 @@ impl api::IncomingWebhook for Dlocal {
     fn get_webhook_resource_object(
         &self,
         _request: &api::IncomingWebhookRequestDetails<'_>,
-    ) -> CustomResult<serde_json::Value, errors::ConnectorError> {
+    ) -> CustomResult<Box<dyn masking::ErasedMaskSerialize>, errors::ConnectorError> {
         Err(errors::ConnectorError::WebhooksNotImplemented).into_report()
     }
 }

crates/router/src/connector/dummyconnector.rs

@@ -579,7 +579,7 @@ impl<const T: u8> api::IncomingWebhook for DummyConnector<T> {
     fn get_webhook_resource_object(
         &self,
         _request: &api::IncomingWebhookRequestDetails<'_>,
-    ) -> CustomResult<serde_json::Value, errors::ConnectorError> {
+    ) -> CustomResult<Box<dyn masking::ErasedMaskSerialize>, errors::ConnectorError> {
         Err(errors::ConnectorError::WebhooksNotImplemented).into_report()
     }
 }

crates/router/src/connector/fiserv.rs

@@ -787,7 +787,7 @@ impl api::IncomingWebhook for Fiserv {
     fn get_webhook_resource_object(
         &self,
         _request: &api::IncomingWebhookRequestDetails<'_>,
-    ) -> CustomResult<serde_json::Value, errors::ConnectorError> {
+    ) -> CustomResult<Box<dyn masking::ErasedMaskSerialize>, errors::ConnectorError> {
         Err(errors::ConnectorError::WebhooksNotImplemented).into_report()
     }
 }

crates/router/src/connector/forte.rs

@@ -669,7 +669,7 @@ impl api::IncomingWebhook for Forte {
     fn get_webhook_resource_object(
         &self,
         _request: &api::IncomingWebhookRequestDetails<'_>,
-    ) -> CustomResult<serde_json::Value, errors::ConnectorError> {
+    ) -> CustomResult<Box<dyn masking::ErasedMaskSerialize>, errors::ConnectorError> {
         Err(errors::ConnectorError::WebhooksNotImplemented).into_report()
     }
 }

crates/router/src/connector/globalpay.rs

@@ -932,14 +932,15 @@ impl api::IncomingWebhook for Globalpay {
     fn get_webhook_resource_object(
         &self,
         request: &api::IncomingWebhookRequestDetails<'_>,
-    ) -> CustomResult<Value, errors::ConnectorError> {
+    ) -> CustomResult<Box<dyn masking::ErasedMaskSerialize>, errors::ConnectorError> {
         let details = std::str::from_utf8(request.body)
             .into_report()
             .change_context(errors::ConnectorError::WebhookBodyDecodingFailed)?;
-        let res_json = serde_json::from_str(details)
-            .into_report()
-            .change_context(errors::ConnectorError::WebhookResourceObjectNotFound)?;
-        Ok(res_json)
+        Ok(Box::new(
+            serde_json::from_str(details)
+                .into_report()
+                .change_context(errors::ConnectorError::WebhookResourceObjectNotFound)?,
+        ))
     }
 }