From 511fdd449d8a162e8e0ff7f09536b0aba9f49ea9 Mon Sep 17 00:00:00 2001 From: "pixincreate@work" <69745008+pixincreate@users.noreply.github.com> Date: Fri, 22 Dec 2023 13:38:23 +0530 Subject: [PATCH 1/5] Add TASKS_ADDED_COUNT and TASKS_RESET_COUNT metrics --- crates/router/src/core/payments/helpers.rs | 4 ++-- crates/router/src/routes/metrics.rs | 3 +++ crates/scheduler/src/metrics.rs | 2 -- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/router/src/core/payments/helpers.rs b/crates/router/src/core/payments/helpers.rs index 2cd62fbd4914..9aabb4dc6185 100644 --- a/crates/router/src/core/payments/helpers.rs +++ b/crates/router/src/core/payments/helpers.rs @@ -987,14 +987,14 @@ where match schedule_time { Some(stime) => { if !requeue { - // scheduler_metrics::TASKS_ADDED_COUNT.add(&metrics::CONTEXT, 1, &[]); // Metrics + metrics::TASKS_ADDED_COUNT.add(&metrics::CONTEXT, 1, &[]); // Metrics super::add_process_sync_task(&*state.store, payment_attempt, stime) .await .into_report() .change_context(errors::ApiErrorResponse::InternalServerError) .attach_printable("Failed while adding task to process tracker") } else { - // scheduler_metrics::TASKS_RESET_COUNT.add(&metrics::CONTEXT, 1, &[]); // Metrics + metrics::TASKS_RESET_COUNT.add(&metrics::CONTEXT, 1, &[]); // Metrics super::reset_process_sync_task(&*state.store, payment_attempt, stime) .await .into_report() diff --git a/crates/router/src/routes/metrics.rs b/crates/router/src/routes/metrics.rs index b3629ab7d52b..6c3293dba9d0 100644 --- a/crates/router/src/routes/metrics.rs +++ b/crates/router/src/routes/metrics.rs @@ -113,5 +113,8 @@ counter_metric!(AUTO_RETRY_GSM_MATCH_COUNT, GLOBAL_METER); counter_metric!(AUTO_RETRY_EXHAUSTED_COUNT, GLOBAL_METER); counter_metric!(AUTO_RETRY_PAYMENT_COUNT, GLOBAL_METER); +counter_metric!(TASKS_ADDED_COUNT, GLOBAL_METER); // Tasks added to process tracker +counter_metric!(TASKS_RESET_COUNT, GLOBAL_METER); // Tasks reset in process tracker for requeue flow + pub mod request; pub mod utils; diff --git a/crates/scheduler/src/metrics.rs b/crates/scheduler/src/metrics.rs index 134f5599b31d..ca4fb9ec2424 100644 --- a/crates/scheduler/src/metrics.rs +++ b/crates/scheduler/src/metrics.rs @@ -6,8 +6,6 @@ global_meter!(PT_METER, "PROCESS_TRACKER"); histogram_metric!(CONSUMER_STATS, PT_METER, "CONSUMER_OPS"); counter_metric!(PAYMENT_COUNT, PT_METER); // No. of payments created -counter_metric!(TASKS_ADDED_COUNT, PT_METER); // Tasks added to process tracker -counter_metric!(TASKS_RESET_COUNT, PT_METER); // Tasks reset in process tracker for requeue flow counter_metric!(TASKS_PICKED_COUNT, PT_METER); // Tasks picked by counter_metric!(BATCHES_CREATED, PT_METER); // Batches added to stream counter_metric!(BATCHES_CONSUMED, PT_METER); // Batches consumed by consumer From 9eb8bb38cb17b4e51db28c34096626d308010939 Mon Sep 17 00:00:00 2001 From: "pixincreate@work" <69745008+pixincreate@users.noreply.github.com> Date: Sat, 23 Dec 2023 18:06:23 +0530 Subject: [PATCH 2/5] Add proper comments --- crates/router/src/core/payments/helpers.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/router/src/core/payments/helpers.rs b/crates/router/src/core/payments/helpers.rs index 9aabb4dc6185..e3664dba6954 100644 --- a/crates/router/src/core/payments/helpers.rs +++ b/crates/router/src/core/payments/helpers.rs @@ -987,14 +987,16 @@ where match schedule_time { Some(stime) => { if !requeue { - metrics::TASKS_ADDED_COUNT.add(&metrics::CONTEXT, 1, &[]); // Metrics + // Increment the count of added tasks every time a payment has been confirmed or PSync has been called + metrics::TASKS_ADDED_COUNT.add(&metrics::CONTEXT, 1, &[]); super::add_process_sync_task(&*state.store, payment_attempt, stime) .await .into_report() .change_context(errors::ApiErrorResponse::InternalServerError) .attach_printable("Failed while adding task to process tracker") } else { - metrics::TASKS_RESET_COUNT.add(&metrics::CONTEXT, 1, &[]); // Metrics + // When the requeue is true, we reset the tasks count as we reset the task every time it is requeued + metrics::TASKS_RESET_COUNT.add(&metrics::CONTEXT, 1, &[]); super::reset_process_sync_task(&*state.store, payment_attempt, stime) .await .into_report() From 04d850d2a0a510730d9a0b010cc17251df27fa95 Mon Sep 17 00:00:00 2001 From: "PiX@work" <69745008+pixincreate@users.noreply.github.com> Date: Fri, 12 Jan 2024 18:03:48 +0530 Subject: [PATCH 3/5] Add attributes and metrics to refunds and delete_tokenized_data --- crates/router/src/core/api_keys.rs | 8 +++++++ .../router/src/core/payment_methods/vault.rs | 21 ++++++++++++++++++- crates/router/src/core/payments/helpers.rs | 18 ++++++++++++++-- crates/router/src/core/refunds.rs | 16 +++++++++++++- crates/router/src/workflows/api_key_expiry.rs | 9 ++++++++ 5 files changed, 68 insertions(+), 4 deletions(-) diff --git a/crates/router/src/core/api_keys.rs b/crates/router/src/core/api_keys.rs index c1ddc43cd65d..3ca5a68bf3c2 100644 --- a/crates/router/src/core/api_keys.rs +++ b/crates/router/src/core/api_keys.rs @@ -270,6 +270,14 @@ pub async fn add_api_key_expiry_task( api_key_expiry_tracker.key_id ) })?; + metrics::TASKS_ADDED_COUNT.add( + &metrics::CONTEXT, + 1, + &[metrics::request::add_attributes( + "api_key_expiry", + format!("{:#?}", expiry_reminder_days.to_owned()), + )], + ); Ok(()) } diff --git a/crates/router/src/core/payment_methods/vault.rs b/crates/router/src/core/payment_methods/vault.rs index 070bca234c8e..bf2b9807e1fe 100644 --- a/crates/router/src/core/payment_methods/vault.rs +++ b/crates/router/src/core/payment_methods/vault.rs @@ -984,6 +984,14 @@ pub async fn start_tokenize_data_workflow( .clone() .finish_with_status(db.as_scheduler(), format!("COMPLETED_BY_PT_{id}")) .await?; + metrics::TASKS_ADDED_COUNT.add( + &metrics::CONTEXT, + 1, + &[metrics::request::add_attributes( + "delete_tokenize_data", + "DELETE_TOKENIZE_DATA", + )], + ); } Err(err) => { logger::error!("Err: Deleting Card From Locker : {:?}", err); @@ -1026,7 +1034,18 @@ pub async fn retry_delete_tokenize( let schedule_time = get_delete_tokenize_schedule_time(db, pm, pt.retry_count).await; match schedule_time { - Some(s_time) => pt.retry(db.as_scheduler(), s_time).await, + Some(s_time) => { + let retry_schedule = pt.retry(db.as_scheduler(), s_time).await; + metrics::TASKS_RESET_COUNT.add( + &metrics::CONTEXT, + 1, + &[metrics::request::add_attributes( + "delete_tokenize_data", + "DELETE_TOKENIZE_DATA", + )], + ); + retry_schedule + } None => { pt.finish_with_status(db.as_scheduler(), "RETRIES_EXCEEDED".to_string()) .await diff --git a/crates/router/src/core/payments/helpers.rs b/crates/router/src/core/payments/helpers.rs index b7cff6fb4a8c..fd359fd63ed4 100644 --- a/crates/router/src/core/payments/helpers.rs +++ b/crates/router/src/core/payments/helpers.rs @@ -987,7 +987,14 @@ where Some(stime) => { if !requeue { // Increment the count of added tasks every time a payment has been confirmed or PSync has been called - metrics::TASKS_ADDED_COUNT.add(&metrics::CONTEXT, 1, &[]); + metrics::TASKS_ADDED_COUNT.add( + &metrics::CONTEXT, + 1, + &[metrics::request::add_attributes( + "operation", + format!("{:#?}", operation), + )], + ); super::add_process_sync_task(&*state.store, payment_attempt, stime) .await .into_report() @@ -995,7 +1002,14 @@ where .attach_printable("Failed while adding task to process tracker") } else { // When the requeue is true, we reset the tasks count as we reset the task every time it is requeued - metrics::TASKS_RESET_COUNT.add(&metrics::CONTEXT, 1, &[]); + metrics::TASKS_RESET_COUNT.add( + &metrics::CONTEXT, + 1, + &[metrics::request::add_attributes( + "operation", + format!("{:#?}", operation), + )], + ); super::reset_process_sync_task(&*state.store, payment_attempt, stime) .await .into_report() diff --git a/crates/router/src/core/refunds.rs b/crates/router/src/core/refunds.rs index e60c341dedcf..c035510c9971 100644 --- a/crates/router/src/core/refunds.rs +++ b/crates/router/src/core/refunds.rs @@ -1088,6 +1088,12 @@ pub async fn add_refund_sync_task( refund.refund_id ) })?; + metrics::TASKS_ADDED_COUNT.add( + &metrics::CONTEXT, + 1, + &[metrics::request::add_attributes("refund", "REFUND")], + ); + Ok(response) } @@ -1170,7 +1176,15 @@ pub async fn retry_refund_sync_task( get_refund_sync_process_schedule_time(db, &connector, &merchant_id, pt.retry_count).await?; match schedule_time { - Some(s_time) => pt.retry(db.as_scheduler(), s_time).await, + Some(s_time) => { + let retry_schedule = pt.retry(db.as_scheduler(), s_time).await; + metrics::TASKS_RESET_COUNT.add( + &metrics::CONTEXT, + 1, + &[metrics::request::add_attributes("refund", "REFUND")], + ); + retry_schedule + } None => { pt.finish_with_status(db.as_scheduler(), "RETRIES_EXCEEDED".to_string()) .await diff --git a/crates/router/src/workflows/api_key_expiry.rs b/crates/router/src/workflows/api_key_expiry.rs index 62d8a54c4024..e4cd733acf1e 100644 --- a/crates/router/src/workflows/api_key_expiry.rs +++ b/crates/router/src/workflows/api_key_expiry.rs @@ -115,6 +115,15 @@ Team Hyperswitch"), let task_ids = vec![task_id]; db.process_tracker_update_process_status_by_ids(task_ids, updated_process_tracker_data) .await?; + // Remaining tasks are re-scheduled, so will be resetting the added count + metrics::TASKS_RESET_COUNT.add( + &metrics::CONTEXT, + 1, + &[metrics::request::add_attributes( + "api_key_expiry", + format!("API KEY EXPIRY"), + )], + ); } Ok(()) From 8807bde4468c9ccf49651dc2f92864cf48e34e8e Mon Sep 17 00:00:00 2001 From: "PiX@work" <69745008+pixincreate@users.noreply.github.com> Date: Sun, 14 Jan 2024 23:53:24 +0530 Subject: [PATCH 4/5] Now it makes sense --- crates/router/src/core/api_keys.rs | 5 +---- crates/router/src/core/payment_methods/vault.rs | 8 ++++---- crates/router/src/core/payments/helpers.rs | 4 ++-- crates/router/src/core/refunds.rs | 4 ++-- crates/router/src/workflows/api_key_expiry.rs | 5 +---- 5 files changed, 10 insertions(+), 16 deletions(-) diff --git a/crates/router/src/core/api_keys.rs b/crates/router/src/core/api_keys.rs index 3ca5a68bf3c2..78d4e801e8f2 100644 --- a/crates/router/src/core/api_keys.rs +++ b/crates/router/src/core/api_keys.rs @@ -273,10 +273,7 @@ pub async fn add_api_key_expiry_task( metrics::TASKS_ADDED_COUNT.add( &metrics::CONTEXT, 1, - &[metrics::request::add_attributes( - "api_key_expiry", - format!("{:#?}", expiry_reminder_days.to_owned()), - )], + &[metrics::request::add_attributes("flow", "ApiKeyExpiry")], ); Ok(()) diff --git a/crates/router/src/core/payment_methods/vault.rs b/crates/router/src/core/payment_methods/vault.rs index bf2b9807e1fe..003d13ab4837 100644 --- a/crates/router/src/core/payment_methods/vault.rs +++ b/crates/router/src/core/payment_methods/vault.rs @@ -988,8 +988,8 @@ pub async fn start_tokenize_data_workflow( &metrics::CONTEXT, 1, &[metrics::request::add_attributes( - "delete_tokenize_data", - "DELETE_TOKENIZE_DATA", + "flow", + "DeleteTokenizeData", )], ); } @@ -1040,8 +1040,8 @@ pub async fn retry_delete_tokenize( &metrics::CONTEXT, 1, &[metrics::request::add_attributes( - "delete_tokenize_data", - "DELETE_TOKENIZE_DATA", + "flow", + "DeleteTokenizeData", )], ); retry_schedule diff --git a/crates/router/src/core/payments/helpers.rs b/crates/router/src/core/payments/helpers.rs index d02d70f81775..e15fed0327ff 100644 --- a/crates/router/src/core/payments/helpers.rs +++ b/crates/router/src/core/payments/helpers.rs @@ -991,7 +991,7 @@ where &metrics::CONTEXT, 1, &[metrics::request::add_attributes( - "operation", + "flow", format!("{:#?}", operation), )], ); @@ -1006,7 +1006,7 @@ where &metrics::CONTEXT, 1, &[metrics::request::add_attributes( - "operation", + "flow", format!("{:#?}", operation), )], ); diff --git a/crates/router/src/core/refunds.rs b/crates/router/src/core/refunds.rs index c035510c9971..4b1c33296e65 100644 --- a/crates/router/src/core/refunds.rs +++ b/crates/router/src/core/refunds.rs @@ -1091,7 +1091,7 @@ pub async fn add_refund_sync_task( metrics::TASKS_ADDED_COUNT.add( &metrics::CONTEXT, 1, - &[metrics::request::add_attributes("refund", "REFUND")], + &[metrics::request::add_attributes("flow", "Refund")], ); Ok(response) @@ -1181,7 +1181,7 @@ pub async fn retry_refund_sync_task( metrics::TASKS_RESET_COUNT.add( &metrics::CONTEXT, 1, - &[metrics::request::add_attributes("refund", "REFUND")], + &[metrics::request::add_attributes("flow", "Refund")], ); retry_schedule } diff --git a/crates/router/src/workflows/api_key_expiry.rs b/crates/router/src/workflows/api_key_expiry.rs index e4cd733acf1e..eb3c1d9c1ce9 100644 --- a/crates/router/src/workflows/api_key_expiry.rs +++ b/crates/router/src/workflows/api_key_expiry.rs @@ -119,10 +119,7 @@ Team Hyperswitch"), metrics::TASKS_RESET_COUNT.add( &metrics::CONTEXT, 1, - &[metrics::request::add_attributes( - "api_key_expiry", - format!("API KEY EXPIRY"), - )], + &[metrics::request::add_attributes("flow", "ApiKeyExpiry")], ); } From ec84e39cc3565ecf8b42e8cc6861fe06933dfc70 Mon Sep 17 00:00:00 2001 From: "PiX@work" <69745008+pixincreate@users.noreply.github.com> Date: Thu, 18 Jan 2024 12:40:03 +0530 Subject: [PATCH 5/5] It should be in place where the task gets added not when it starts --- crates/router/src/core/payment_methods/cards.rs | 8 ++++++++ crates/router/src/core/payment_methods/vault.rs | 8 -------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/router/src/core/payment_methods/cards.rs b/crates/router/src/core/payment_methods/cards.rs index 39bc54fa1578..f005920c24fc 100644 --- a/crates/router/src/core/payment_methods/cards.rs +++ b/crates/router/src/core/payment_methods/cards.rs @@ -2855,6 +2855,14 @@ impl TempLockerCardSupport { ) .await?; metrics::TOKENIZED_DATA_COUNT.add(&metrics::CONTEXT, 1, &[]); + metrics::TASKS_ADDED_COUNT.add( + &metrics::CONTEXT, + 1, + &[metrics::request::add_attributes( + "flow", + "DeleteTokenizeData", + )], + ); Ok(card) } } diff --git a/crates/router/src/core/payment_methods/vault.rs b/crates/router/src/core/payment_methods/vault.rs index 003d13ab4837..59be4087eaa1 100644 --- a/crates/router/src/core/payment_methods/vault.rs +++ b/crates/router/src/core/payment_methods/vault.rs @@ -984,14 +984,6 @@ pub async fn start_tokenize_data_workflow( .clone() .finish_with_status(db.as_scheduler(), format!("COMPLETED_BY_PT_{id}")) .await?; - metrics::TASKS_ADDED_COUNT.add( - &metrics::CONTEXT, - 1, - &[metrics::request::add_attributes( - "flow", - "DeleteTokenizeData", - )], - ); } Err(err) => { logger::error!("Err: Deleting Card From Locker : {:?}", err);