Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(metrics): Add TASKS_ADDED_COUNT and TASKS_RESET_COUNT metrics in router scheduler flow #3189

Merged
merged 9 commits into from
Jan 18, 2024
8 changes: 8 additions & 0 deletions crates/router/src/core/api_keys.rs
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,14 @@ pub async fn add_api_key_expiry_task(
api_key_expiry_tracker.key_id
)
})?;
metrics::TASKS_ADDED_COUNT.add(
&metrics::CONTEXT,
1,
&[metrics::request::add_attributes(
"api_key_expiry",
pixincreate marked this conversation as resolved.
Show resolved Hide resolved
format!("{:#?}", expiry_reminder_days.to_owned()),
)],
);

Ok(())
}
Expand Down
21 changes: 20 additions & 1 deletion crates/router/src/core/payment_methods/vault.rs
Original file line number Diff line number Diff line change
Expand Up @@ -984,6 +984,14 @@ pub async fn start_tokenize_data_workflow(
.clone()
.finish_with_status(db.as_scheduler(), format!("COMPLETED_BY_PT_{id}"))
.await?;
metrics::TASKS_ADDED_COUNT.add(
&metrics::CONTEXT,
1,
&[metrics::request::add_attributes(
"delete_tokenize_data",
"DELETE_TOKENIZE_DATA",
)],
);
}
Err(err) => {
logger::error!("Err: Deleting Card From Locker : {:?}", err);
Expand Down Expand Up @@ -1026,7 +1034,18 @@ pub async fn retry_delete_tokenize(
let schedule_time = get_delete_tokenize_schedule_time(db, pm, pt.retry_count).await;

match schedule_time {
Some(s_time) => pt.retry(db.as_scheduler(), s_time).await,
Some(s_time) => {
let retry_schedule = pt.retry(db.as_scheduler(), s_time).await;
metrics::TASKS_RESET_COUNT.add(
&metrics::CONTEXT,
1,
&[metrics::request::add_attributes(
"delete_tokenize_data",
"DELETE_TOKENIZE_DATA",
)],
);
retry_schedule
}
None => {
pt.finish_with_status(db.as_scheduler(), "RETRIES_EXCEEDED".to_string())
.await
Expand Down
20 changes: 18 additions & 2 deletions crates/router/src/core/payments/helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -986,14 +986,30 @@ where
match schedule_time {
Some(stime) => {
if !requeue {
// scheduler_metrics::TASKS_ADDED_COUNT.add(&metrics::CONTEXT, 1, &[]); // Metrics
// Here, increment the count of added tasks every time a payment has been confirmed or PSync has been called
metrics::TASKS_ADDED_COUNT.add(
&metrics::CONTEXT,
1,
&[metrics::request::add_attributes(
"operation",
format!("{:#?}", operation),
)],
);
super::add_process_sync_task(&*state.store, payment_attempt, stime)
.await
.into_report()
.change_context(errors::ApiErrorResponse::InternalServerError)
.attach_printable("Failed while adding task to process tracker")
} else {
// scheduler_metrics::TASKS_RESET_COUNT.add(&metrics::CONTEXT, 1, &[]); // Metrics
// When the requeue is true, we reset the tasks count as we reset the task every time it is requeued
metrics::TASKS_RESET_COUNT.add(
&metrics::CONTEXT,
1,
&[metrics::request::add_attributes(
"operation",
format!("{:#?}", operation),
)],
);
super::reset_process_sync_task(&*state.store, payment_attempt, stime)
.await
.into_report()
Expand Down
16 changes: 15 additions & 1 deletion crates/router/src/core/refunds.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1088,6 +1088,12 @@ pub async fn add_refund_sync_task(
refund.refund_id
)
})?;
metrics::TASKS_ADDED_COUNT.add(
&metrics::CONTEXT,
1,
&[metrics::request::add_attributes("refund", "REFUND")],
);

Ok(response)
}

Expand Down Expand Up @@ -1170,7 +1176,15 @@ pub async fn retry_refund_sync_task(
get_refund_sync_process_schedule_time(db, &connector, &merchant_id, pt.retry_count).await?;

match schedule_time {
Some(s_time) => pt.retry(db.as_scheduler(), s_time).await,
Some(s_time) => {
let retry_schedule = pt.retry(db.as_scheduler(), s_time).await;
metrics::TASKS_RESET_COUNT.add(
&metrics::CONTEXT,
1,
&[metrics::request::add_attributes("refund", "REFUND")],
);
retry_schedule
}
None => {
pt.finish_with_status(db.as_scheduler(), "RETRIES_EXCEEDED".to_string())
.await
Expand Down
3 changes: 3 additions & 0 deletions crates/router/src/routes/metrics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -113,5 +113,8 @@ counter_metric!(AUTO_RETRY_GSM_MATCH_COUNT, GLOBAL_METER);
counter_metric!(AUTO_RETRY_EXHAUSTED_COUNT, GLOBAL_METER);
counter_metric!(AUTO_RETRY_PAYMENT_COUNT, GLOBAL_METER);

counter_metric!(TASKS_ADDED_COUNT, GLOBAL_METER); // Tasks added to process tracker
counter_metric!(TASKS_RESET_COUNT, GLOBAL_METER); // Tasks reset in process tracker for requeue flow

pub mod request;
pub mod utils;
9 changes: 9 additions & 0 deletions crates/router/src/workflows/api_key_expiry.rs
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,15 @@ Team Hyperswitch"),
let task_ids = vec![task_id];
db.process_tracker_update_process_status_by_ids(task_ids, updated_process_tracker_data)
.await?;
// Remaining tasks are re-scheduled, so will be resetting the added count
metrics::TASKS_RESET_COUNT.add(
&metrics::CONTEXT,
1,
&[metrics::request::add_attributes(
"api_key_expiry",
format!("API KEY EXPIRY"),
)],
);
}

Ok(())
Expand Down
2 changes: 0 additions & 2 deletions crates/scheduler/src/metrics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@ global_meter!(PT_METER, "PROCESS_TRACKER");
histogram_metric!(CONSUMER_STATS, PT_METER, "CONSUMER_OPS");

counter_metric!(PAYMENT_COUNT, PT_METER); // No. of payments created
counter_metric!(TASKS_ADDED_COUNT, PT_METER); // Tasks added to process tracker
counter_metric!(TASKS_RESET_COUNT, PT_METER); // Tasks reset in process tracker for requeue flow
counter_metric!(TASKS_PICKED_COUNT, PT_METER); // Tasks picked by
counter_metric!(BATCHES_CREATED, PT_METER); // Batches added to stream
counter_metric!(BATCHES_CONSUMED, PT_METER); // Batches consumed by consumer
Expand Down
Loading