From e4b703cd229e9f1cc68cc17087a792e78593d0d3 Mon Sep 17 00:00:00 2001 From: Derek Dyer Date: Tue, 8 Oct 2024 12:22:51 -0400 Subject: [PATCH] 91367 Debt Resolution Exhausted Metrics (#18472) * 91367 Debt Resolution Exhausted Metrics * Add back old metrics for backwards compatability * Add notes * Remove file * Remove file * Add updates * Add updates * Add lib update * Add updates * Remove uneeded block * Add modules * Add updates * Add updates * Add udpates * Remove specs * Add updates * Add linter fixes * Reorganize changes to not exceed PR size limit Revert "Remove file" This reverts commit b9a68afe9f50afb04851f92362f4540dbeff5d9d. Revert "Remove file" This reverts commit 3e2c74075392d24906c174da9bf8b1739876d0ac. Revert "Remove specs" This reverts commit 3e1827782c46c2f650cea39b651fb17234306aa3. * Fix linter * Fix unit test * Feedback * Missed comment --------- Co-authored-by: Jun Tao Luo --- .../new_statement_notification_job.rb | 3 +- .../parse_new_statements_job.rb | 12 +++- .../v0/form5655/vba_submission_job.rb | 10 +++- .../v0/form5655/vha_submission_job.rb | 8 +++ .../form5655/vha/sharepoint_submission_job.rb | 3 +- .../v0/form5655/vha/vbs_submission_job.rb | 3 +- .../debt_api/v0/vba_submission_job_spec.rb | 56 +++++++++++++++++++ .../v0/vha/sharepoint_submission_job_spec.rb | 36 ++++++++++-- .../v0/vha/vbs_submission_job_spec.rb | 18 ++++++ .../new_statement_notification_job_spec.rb | 7 ++- .../parse_new_statements_job_spec.rb | 32 +++++++++++ 11 files changed, 176 insertions(+), 12 deletions(-) diff --git a/app/sidekiq/copay_notifications/new_statement_notification_job.rb b/app/sidekiq/copay_notifications/new_statement_notification_job.rb index 9f16aba27a7..550ae630d36 100644 --- a/app/sidekiq/copay_notifications/new_statement_notification_job.rb +++ b/app/sidekiq/copay_notifications/new_statement_notification_job.rb @@ -29,7 +29,8 @@ class NewStatementNotificationJob end sidekiq_retries_exhausted do |_msg, ex| - StatsD.increment("#{STATSD_KEY_PREFIX}.failure") + StatsD.increment("#{STATSD_KEY_PREFIX}.failure") # remove when we get more data into the retries_exhausted below + StatsD.increment("#{STATSD_KEY_PREFIX}.retries_exhausted") Rails.logger.error <<~LOG NewStatementNotificationJob retries exhausted: Exception: #{ex.class} - #{ex.message} diff --git a/app/sidekiq/copay_notifications/parse_new_statements_job.rb b/app/sidekiq/copay_notifications/parse_new_statements_job.rb index 3f2bf9c348c..8472eac849f 100644 --- a/app/sidekiq/copay_notifications/parse_new_statements_job.rb +++ b/app/sidekiq/copay_notifications/parse_new_statements_job.rb @@ -10,9 +10,19 @@ class ParseNewStatementsJob JOB_INTERVAL = Settings.mcp.notifications.job_interval # number of jobs to perform at next interval BATCH_SIZE = Settings.mcp.notifications.batch_size + STATSD_KEY_PREFIX = 'api.copay_notifications.json_file' + + sidekiq_retries_exhausted do |_msg, ex| + StatsD.increment("#{STATSD_KEY_PREFIX}.retries_exhausted") + Rails.logger.error <<~LOG + CopayNotifications::ParseNewStatementsJob retries exhausted: + Exception: #{ex.class} - #{ex.message} + Backtrace: #{ex.backtrace.join("\n")} + LOG + end def perform(statements_json_byte) - StatsD.increment('api.copay_notifications.json_file.total') + StatsD.increment("#{STATSD_KEY_PREFIX}.total") # Decode and parse large json file (~60-90k objects) statements_json = Oj.load(Base64.decode64(statements_json_byte)) unique_statements = statements_json.uniq { |statement| statement['veteranIdentifier'] } diff --git a/modules/debts_api/app/sidekiq/debts_api/v0/form5655/vba_submission_job.rb b/modules/debts_api/app/sidekiq/debts_api/v0/form5655/vba_submission_job.rb index 115e042f481..50e939a174c 100644 --- a/modules/debts_api/app/sidekiq/debts_api/v0/form5655/vba_submission_job.rb +++ b/modules/debts_api/app/sidekiq/debts_api/v0/form5655/vba_submission_job.rb @@ -12,8 +12,16 @@ class V0::Form5655::VBASubmissionJob class MissingUserAttributesError < StandardError; end - sidekiq_retries_exhausted do |job, _ex| + sidekiq_retries_exhausted do |job, ex| + StatsD.increment("#{STATS_KEY}.retries_exhausted") + submission_id = job['args'][0] user_uuid = job['args'][1] + Rails.logger.error <<~LOG + V0::Form5655::VBASubmissionJob retries exhausted: + submission_id: #{submission_id} | user_id: #{user_uuid} + Exception: #{ex.class} - #{ex.message} + Backtrace: #{ex.backtrace.join("\n")} + LOG UserProfileAttributes.find(user_uuid)&.destroy end diff --git a/modules/debts_api/app/sidekiq/debts_api/v0/form5655/vha_submission_job.rb b/modules/debts_api/app/sidekiq/debts_api/v0/form5655/vha_submission_job.rb index c4b8f65fbb4..2b2e8b89847 100644 --- a/modules/debts_api/app/sidekiq/debts_api/v0/form5655/vha_submission_job.rb +++ b/modules/debts_api/app/sidekiq/debts_api/v0/form5655/vha_submission_job.rb @@ -13,7 +13,15 @@ class V0::Form5655::VHASubmissionJob class MissingUserAttributesError < StandardError; end sidekiq_retries_exhausted do |job, _ex| + StatsD.increment("#{STATS_KEY}.retries_exhausted") + submission_id = job['args'][0] user_uuid = job['args'][1] + Rails.logger.error <<~LOG + V0::Form5655::VHASubmissionJob retries exhausted: + Exception: #{ex.class} - #{ex.message} + Backtrace: #{ex.backtrace.join("\n")} + submission_id: #{submission_id} | user_id: #{user_uuid} + LOG UserProfileAttributes.find(user_uuid)&.destroy end diff --git a/modules/debts_api/app/workers/debts_api/v0/form5655/vha/sharepoint_submission_job.rb b/modules/debts_api/app/workers/debts_api/v0/form5655/vha/sharepoint_submission_job.rb index bb36971fa5e..dfbaebe2e9b 100644 --- a/modules/debts_api/app/workers/debts_api/v0/form5655/vha/sharepoint_submission_job.rb +++ b/modules/debts_api/app/workers/debts_api/v0/form5655/vha/sharepoint_submission_job.rb @@ -11,7 +11,8 @@ class V0::Form5655::VHA::SharepointSubmissionJob sidekiq_options retry: 4 sidekiq_retries_exhausted do |job, _ex| - StatsD.increment("#{STATS_KEY}.failure") + StatsD.increment("#{STATS_KEY}.failure") # Deprecate this in favor of exhausted naming convention below + StatsD.increment("#{STATS_KEY}.retries_exhausted") submission_id = job['args'][0] submission = DebtsApi::V0::Form5655Submission.find(submission_id) submission.register_failure("SharePoint Submission Failed: #{job['error_message']}.") diff --git a/modules/debts_api/app/workers/debts_api/v0/form5655/vha/vbs_submission_job.rb b/modules/debts_api/app/workers/debts_api/v0/form5655/vha/vbs_submission_job.rb index 20387a2ca50..c93dba7c128 100644 --- a/modules/debts_api/app/workers/debts_api/v0/form5655/vha/vbs_submission_job.rb +++ b/modules/debts_api/app/workers/debts_api/v0/form5655/vha/vbs_submission_job.rb @@ -13,7 +13,8 @@ class V0::Form5655::VHA::VBSSubmissionJob class MissingUserAttributesError < StandardError; end sidekiq_retries_exhausted do |job, _ex| - StatsD.increment("#{STATS_KEY}.failure") + StatsD.increment("#{STATS_KEY}.failure") # Deprecate this in favor of exhausted naming convention below + StatsD.increment("#{STATS_KEY}.retries_exhausted") submission_id = job['args'][0] user_uuid = job['args'][1] UserProfileAttributes.find(user_uuid)&.destroy diff --git a/modules/debts_api/spec/sidekiq/debt_api/v0/vba_submission_job_spec.rb b/modules/debts_api/spec/sidekiq/debt_api/v0/vba_submission_job_spec.rb index 9e271180e53..656bb14b187 100644 --- a/modules/debts_api/spec/sidekiq/debt_api/v0/vba_submission_job_spec.rb +++ b/modules/debts_api/spec/sidekiq/debt_api/v0/vba_submission_job_spec.rb @@ -38,5 +38,61 @@ expect(form_submission.error_message).to eq('uhoh') end end + + context 'with retries exhausted' do + let(:config) { described_class } + let(:missing_attributes_exception) do + e = DebtsApi::V0::Form5655::VBASubmissionJob::MissingUserAttributesError.new('abc-123') + allow(e).to receive(:backtrace).and_return(%w[backtrace1 backtrace2]) + e + end + + let(:standard_exception) do + e = StandardError.new('abc-123') + allow(e).to receive(:backtrace).and_return(%w[backtrace1 backtrace2]) + e + end + + let(:msg) do + { + 'class' => 'YourJobClassName', + 'args' => %w[123 123-abc], + 'jid' => '12345abcde', + 'retry_count' => 5 + } + end + + it 'handles MissingUserAttributesError' do + expected_log_message = <<~LOG + V0::Form5655::VBASubmissionJob retries exhausted: + submission_id: 123 | user_id: 123-abc + Exception: #{missing_attributes_exception.class} - #{missing_attributes_exception.message} + Backtrace: #{missing_attributes_exception.backtrace.join("\n")} + LOG + + expect(StatsD).to receive(:increment).with( + "#{DebtsApi::V0::Form5655::VBASubmissionJob::STATS_KEY}.retries_exhausted" + ) + + expect(Rails.logger).to receive(:error).with(expected_log_message) + config.sidekiq_retries_exhausted_block.call(msg, missing_attributes_exception) + end + + it 'handles unexpected errors' do + expected_log_message = <<~LOG + V0::Form5655::VBASubmissionJob retries exhausted: + submission_id: 123 | user_id: 123-abc + Exception: #{standard_exception.class} - #{standard_exception.message} + Backtrace: #{standard_exception.backtrace.join("\n")} + LOG + + expect(StatsD).to receive(:increment).with( + "#{DebtsApi::V0::Form5655::VBASubmissionJob::STATS_KEY}.retries_exhausted" + ) + + expect(Rails.logger).to receive(:error).with(expected_log_message) + config.sidekiq_retries_exhausted_block.call(msg, standard_exception) + end + end end end diff --git a/modules/debts_api/spec/workers/debt_api/v0/vha/sharepoint_submission_job_spec.rb b/modules/debts_api/spec/workers/debt_api/v0/vha/sharepoint_submission_job_spec.rb index b2c404a61a5..88b3f17192c 100644 --- a/modules/debts_api/spec/workers/debt_api/v0/vha/sharepoint_submission_job_spec.rb +++ b/modules/debts_api/spec/workers/debt_api/v0/vha/sharepoint_submission_job_spec.rb @@ -8,16 +8,44 @@ describe '#perform' do let(:form_submission) { build(:debts_api_form5655_submission) } - context 'when all retries are exhausted' do - before do - allow(DebtsApi::V0::Form5655Submission).to receive(:find).and_return(form_submission) - end + before do + allow(DebtsApi::V0::Form5655Submission).to receive(:find).and_return(form_submission) + end + context 'when all retries are exhausted' do it 'sets submission to failure' do described_class.within_sidekiq_retries_exhausted_block({ 'jid' => 123 }) do expect(form_submission).to receive(:register_failure) end end end + + context 'with retries exhausted' do + let(:config) { described_class } + let(:msg) do + { + 'class' => 'YourJobClassName', + 'args' => %w[123], + 'jid' => '12345abcde', + 'retry_count' => 5 + } + end + + let(:standard_exception) do + e = StandardError.new('abc-123') + allow(e).to receive(:backtrace).and_return(%w[backtrace1 backtrace2]) + e + end + + it 'increments the retries exhausted counter' do + statsd_key = DebtsApi::V0::Form5655::VHA::SharepointSubmissionJob::STATS_KEY + + ["#{statsd_key}.failure", "#{statsd_key}.retries_exhausted", 'api.fsr_submission.failure'].each do |key| + expect(StatsD).to receive(:increment).with(key) + end + + config.sidekiq_retries_exhausted_block.call(msg, standard_exception) + end + end end end diff --git a/modules/debts_api/spec/workers/debt_api/v0/vha/vbs_submission_job_spec.rb b/modules/debts_api/spec/workers/debt_api/v0/vha/vbs_submission_job_spec.rb index 64b77094e45..40a9c8b47b9 100644 --- a/modules/debts_api/spec/workers/debt_api/v0/vha/vbs_submission_job_spec.rb +++ b/modules/debts_api/spec/workers/debt_api/v0/vha/vbs_submission_job_spec.rb @@ -9,6 +9,14 @@ let(:form_submission) { build(:debts_api_form5655_submission) } let(:user) { build(:user, :loa3) } let(:user_data) { build(:user_profile_attributes) } + let(:msg) do + { + 'class' => 'YourJobClassName', + 'args' => %w[123 123-abc], + 'jid' => '12345abcde', + 'retry_count' => 5 + } + end context 'when all retries are exhausted' do before do @@ -20,6 +28,16 @@ expect(form_submission).to receive(:register_failure) end end + + it 'increments the retries exhausted counter' do + statsd_key = DebtsApi::V0::Form5655::VHA::VBSSubmissionJob::STATS_KEY + + ["#{statsd_key}.failure", "#{statsd_key}.retries_exhausted", 'api.fsr_submission.failure'].each do |key| + expect(StatsD).to receive(:increment).with(key) + end + + described_class.sidekiq_retries_exhausted_block.call(msg, StandardError.new('abc-123')) + end end end end diff --git a/spec/sidekiq/copay_notifications/new_statement_notification_job_spec.rb b/spec/sidekiq/copay_notifications/new_statement_notification_job_spec.rb index 5f704d3dbbc..5d5d24a119b 100644 --- a/spec/sidekiq/copay_notifications/new_statement_notification_job_spec.rb +++ b/spec/sidekiq/copay_notifications/new_statement_notification_job_spec.rb @@ -123,9 +123,10 @@ Backtrace: #{exception.backtrace.join("\n")} LOG - expect(StatsD).to receive(:increment).with( - "#{CopayNotifications::NewStatementNotificationJob::STATSD_KEY_PREFIX}.failure" - ) + statsd_key = CopayNotifications::NewStatementNotificationJob::STATSD_KEY_PREFIX + ["#{statsd_key}.failure", "#{statsd_key}.retries_exhausted"].each do |key| + expect(StatsD).to receive(:increment).with(key) + end expect(Rails.logger).to receive(:error).with(expected_log_message) config.sidekiq_retries_exhausted_block.call(msg, exception) end diff --git a/spec/sidekiq/copay_notifications/parse_new_statements_job_spec.rb b/spec/sidekiq/copay_notifications/parse_new_statements_job_spec.rb index 861006f3c26..01ab6ed52e3 100644 --- a/spec/sidekiq/copay_notifications/parse_new_statements_job_spec.rb +++ b/spec/sidekiq/copay_notifications/parse_new_statements_job_spec.rb @@ -54,5 +54,37 @@ job.perform(statements_json_byte) end end + + context 'with retries exhausted' do + let(:config) { described_class } + let(:exception) do + e = StandardError.new('Test error') + allow(e).to receive(:backtrace).and_return(%w[backtrace1 backtrace2]) + e + end + let(:msg) do + { + 'class' => 'YourJobClassName', + 'args' => [statement], + 'jid' => '12345abcde', + 'retry_count' => 5 + } + end + + it 'logs the error' do + expected_log_message = <<~LOG + CopayNotifications::ParseNewStatementsJob retries exhausted: + Exception: #{exception.class} - #{exception.message} + Backtrace: #{exception.backtrace.join("\n")} + LOG + + expect(StatsD).to receive(:increment).with( + "#{CopayNotifications::ParseNewStatementsJob::STATSD_KEY_PREFIX}.retries_exhausted" + ) + + expect(Rails.logger).to receive(:error).with(expected_log_message) + config.sidekiq_retries_exhausted_block.call(statements_json_byte, exception) + end + end end end