From febc3b0a37163c35b6a0171d7b22d8e7260f1488 Mon Sep 17 00:00:00 2001 From: Richard Lynch Date: Fri, 6 Dec 2024 16:21:25 +0000 Subject: [PATCH 1/5] Add fe approved failing pv task report Adds the model to handle generating the reporting code for further education claims that have been approved but with a failing provider verification. --- ...aims_with_failing_provider_verification.rb | 105 +++++++++ .../further_education_payments/eligibility.rb | 10 + ...with_failing_provider_verification_spec.rb | 207 ++++++++++++++++++ 3 files changed, 322 insertions(+) create mode 100644 app/models/admin/reports/fe_approved_claims_with_failing_provider_verification.rb create mode 100644 spec/models/admin/reports/fe_approved_claims_with_failing_provider_verification_spec.rb diff --git a/app/models/admin/reports/fe_approved_claims_with_failing_provider_verification.rb b/app/models/admin/reports/fe_approved_claims_with_failing_provider_verification.rb new file mode 100644 index 0000000000..b6ffaf8c09 --- /dev/null +++ b/app/models/admin/reports/fe_approved_claims_with_failing_provider_verification.rb @@ -0,0 +1,105 @@ +module Admin + module Reports + class FeApprovedClaimsWithFailingProviderVerification + HEADERS = [ + "Claim reference", + "Full name", + "Claim amount", + "Claim status", + "Decision date", + "Decision agent", + "Contract of employment", + "Teaching responsibilities", + "First 5 years of teaching", + "One full term", + "Timetabled teaching hours", + "Age range taught", + "Subject", + "Course", + "2.5 hours weekly teaching", + "Performance", + "Disciplinary" + ] + + def filename + "fe_approved_claims_with_failing_provider_verification.csv" + end + + def to_csv + CSV.generate( + row_sep: "\r\n", + write_headers: true, + headers: HEADERS + ) do |csv| + rows.each { |row| csv << row } + end + end + + private + + def rows + scope.map(&ClaimPresenter.method(:new)).map(&:to_a) + end + + def scope + Claim + .by_policy(Policies::FurtherEducationPayments) + .approved + .joins(:tasks) + .merge(Task.where(name: "provider_verification", passed: false)) + .includes(:eligibility, decisions: :created_by) + end + + class ClaimPresenter + include Admin::ClaimsHelper + include ActionView::Helpers::NumberHelper + + def initialize(claim) + @claim = claim + end + + def to_a + [ + claim.reference, + claim.full_name, + number_to_currency(claim.award_amount, precision: 0), + status(claim), + approval_date, + approval.created_by.full_name, + present_assertion("contract_type"), + present_assertion("teaching_responsibilities"), + present_assertion("further_education_teaching_start_year"), + present_assertion("taught_at_least_one_term"), + present_assertion("teaching_hours_per_week"), + present_assertion("half_teaching_hours"), + present_assertion("subjects_taught"), + "??", # FIXME RL: not sure what courses should be + present_assertion("teaching_hours_per_week_next_term"), + present_assertion("subject_to_formal_performance_action"), + present_assertion("subject_to_disciplinary_action") + ] + end + + private + + attr_reader :claim + + def approval_date + I18n.l(approval.created_at.to_date, format: :day_month_year) + end + + def approval + @approval ||= claim.decisions.reject(&:undone).last + end + + def present_assertion(name) + case claim.eligibility.verification_assertion(name) + when true then "Yes" + when false then "No" + else "N/A" # fixed and variable contracts have different assertions + end + end + end + end + end +end diff --git a/app/models/policies/further_education_payments/eligibility.rb b/app/models/policies/further_education_payments/eligibility.rb index 33375244af..e501b46d69 100644 --- a/app/models/policies/further_education_payments/eligibility.rb +++ b/app/models/policies/further_education_payments/eligibility.rb @@ -94,6 +94,10 @@ def eligible_itt_subject nil end + def verification_assertion(name) + assertion_hash[name] + end + private def provider_and_claimant_names_match? @@ -116,6 +120,12 @@ def provider_first_name def provider_last_name verification.dig("verifier", "last_name") end + + def assertion_hash + @assertion_hash ||= verification.fetch("assertions").map do |assertion| + [assertion["name"], assertion["outcome"]] + end.to_h + end end end end diff --git a/spec/models/admin/reports/fe_approved_claims_with_failing_provider_verification_spec.rb b/spec/models/admin/reports/fe_approved_claims_with_failing_provider_verification_spec.rb new file mode 100644 index 0000000000..d4b8e2a705 --- /dev/null +++ b/spec/models/admin/reports/fe_approved_claims_with_failing_provider_verification_spec.rb @@ -0,0 +1,207 @@ +require "rails_helper" + +RSpec.describe Admin::Reports::FeApprovedClaimsWithFailingProviderVerification do + around do |example| + travel_to Date.new(2024, 11, 1) do + example.run + end + end + + describe "#to_csv" do + it "returns a csv of approved fe claims with failing provider verification" do + fe_claim_with_passing_provider_check = create( + :claim, + :approved, + policy: Policies::FurtherEducationPayments + ) + + create( + :task, + :passed, + name: "provider_verification", + claim: fe_claim_with_passing_provider_check + ) + + fe_fixed_claim_with_failing_provider_check = create( + :claim, + :approved, + policy: Policies::FurtherEducationPayments, + first_name: "Elizabeth", + surname: "Hoover", + qa_required: true, + eligibility_attributes: { + award_amount: 2_000, + contract_type: "permanent", + verification: { + assertions: [ + { + name: "contract_type", + outcome: true + }, + { + name: "teaching_responsibilities", + outcome: true + }, + { + name: "further_education_teaching_start_year", + outcome: true + }, + { + name: "teaching_hours_per_week", + outcome: true + }, + { + name: "half_teaching_hours", + outcome: false + }, + { + name: "subjects_taught", + outcome: false + }, + { + name: "subject_to_formal_performance_action", + outcome: true + }, + { + name: "subject_to_disciplinary_action", + outcome: true + } + ] + } + } + ) + + create( + :task, + :failed, + name: "provider_verification", + claim: fe_fixed_claim_with_failing_provider_check + ) + + fe_variable_claim_with_failing_provider_check = create( + :claim, + :approved, + policy: Policies::FurtherEducationPayments, + first_name: "Edna", + surname: "Krabappel", + eligibility_attributes: { + award_amount: 3_000, + contract_type: "variable_hours", + verification: { + assertions: [ + { + name: "contract_type", + outcome: true + }, + { + name: "teaching_responsibilities", + outcome: true + }, + { + name: "further_education_teaching_start_year", + outcome: true + }, + { + name: "taught_at_least_one_term", + outcome: true + }, + { + name: "teaching_hours_per_week", + outcome: true + }, + { + name: "half_teaching_hours", + outcome: true + }, + { + name: "subjects_taught", + outcome: true + }, + { + name: "teaching_hours_per_week_next_term", + outcome: true + }, + { + name: "subject_to_formal_performance_action", + outcome: true + }, + { + name: "subject_to_disciplinary_action", + outcome: false + } + ] + } + } + ) + + create( + :task, + :failed, + name: "provider_verification", + claim: fe_variable_claim_with_failing_provider_check + ) + + csv = CSV.parse(described_class.new.to_csv, headers: true) + + expect(csv.to_a).to match_array([ + [ + "Claim reference", + "Full name", + "Claim amount", + "Claim status", + "Decision date", + "Decision agent", + "Contract of employment", + "Teaching responsibilities", + "First 5 years of teaching", + "One full term", + "Timetabled teaching hours", + "Age range taught", + "Subject", + "Course", + "2.5 hours weekly teaching", + "Performance", + "Disciplinary" + ], + [ + fe_fixed_claim_with_failing_provider_check.reference, + "Elizabeth Hoover", + "£2,000", + "Approved awaiting QA", + "01/11/2024", + "Aaron Admin", + "Yes", # contract of employment + "Yes", # teaching responsibilities + "Yes", # first 5 years of teaching + "N/A", # one full term - not a question for fixed term contracts + "Yes", # timetabled teaching hours + "No", # age range taught + "No", # subject + "??", # course + "N/A", # 2.5 hours weekly teaching + "Yes", # performance + "Yes" # disciplinary + ], + [ + fe_variable_claim_with_failing_provider_check.reference, + "Edna Krabappel", + "£3,000", + "Approved awaiting payroll", + "01/11/2024", + "Aaron Admin", + "Yes", # contract of employment + "Yes", # teaching responsibilities + "Yes", # first 5 years of teaching + "Yes", # one full term + "Yes", # timetabled teaching hours + "Yes", # age range taught + "Yes", # subject + "??", # course + "Yes", # 2.5 hours weekly teaching + "Yes", # performance + "No" # disciplinary + ] + ]) + end + end +end From 8ad7d7db91bef149b47f96c4c1a3b787c2f15410 Mon Sep 17 00:00:00 2001 From: Richard Lynch Date: Mon, 9 Dec 2024 14:41:32 +0000 Subject: [PATCH 2/5] Add reports controller Adds the controller and views for downloading the ops reports. --- app/controllers/admin/reports_controller.rb | 27 ++++++ app/views/admin/claims/index.html.erb | 1 + app/views/admin/reports/index.html.erb | 17 ++++ config/routes.rb | 1 + spec/features/admin/reports_spec.rb | 102 ++++++++++++++++++++ 5 files changed, 148 insertions(+) create mode 100644 app/controllers/admin/reports_controller.rb create mode 100644 app/views/admin/reports/index.html.erb create mode 100644 spec/features/admin/reports_spec.rb diff --git a/app/controllers/admin/reports_controller.rb b/app/controllers/admin/reports_controller.rb new file mode 100644 index 0000000000..5081b148e4 --- /dev/null +++ b/app/controllers/admin/reports_controller.rb @@ -0,0 +1,27 @@ +module Admin + class ReportsController < BaseAdminController + before_action :ensure_service_operator + + def index + end + + def show + respond_to do |format| + format.csv do + send_data(report.to_csv, filename: report.filename) + end + end + end + + private + + def report + @report ||= case params[:name] + when "fe-approved-claims-with-failing-provider-verification" + Reports::FeApprovedClaimsWithFailingProviderVerification.new + else + raise ActiveRecord::RecordNotFound + end + end + end +end diff --git a/app/views/admin/claims/index.html.erb b/app/views/admin/claims/index.html.erb index 64f5c251ff..d872447b07 100644 --- a/app/views/admin/claims/index.html.erb +++ b/app/views/admin/claims/index.html.erb @@ -16,6 +16,7 @@ <%= link_to "Upload TPS data", new_admin_tps_data_upload_path, class: "govuk-button govuk-button--secondary", data: { module: "govuk-button" }, role: :button %> <%= link_to "Upload SLC data", new_admin_student_loans_data_upload_path, class: "govuk-button govuk-button--secondary", data: { module: "govuk-button" }, role: :button %> <%= link_to "Upload fraud prevention data", new_admin_fraud_risk_csv_upload_path, class: "govuk-button govuk-button--secondary", data: { module: "govuk-button" }, role: :button %> + <%= link_to "Reports", admin_reports_path, class: "govuk-button govuk-button--secondary", data: { module: "govuk-button" }, role: :button %> <%= render "allocations_form" %> diff --git a/app/views/admin/reports/index.html.erb b/app/views/admin/reports/index.html.erb new file mode 100644 index 0000000000..8a9d58e73e --- /dev/null +++ b/app/views/admin/reports/index.html.erb @@ -0,0 +1,17 @@ +<% content_for :back_link do %> + <%= govuk_back_link href: admin_claims_path %> +<% end %> + +
+
+

+ Reports +

+ + <%= govuk_button_link_to( + "FE TRI approved claims whereby the provider check status is 'failed'", + admin_report_path("fe-approved-claims-with-failing-provider-verification", format: :csv), + secondary: true + ) %> +
+
diff --git a/config/routes.rb b/config/routes.rb index f2c13854bc..b3aa1dd5cc 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -133,6 +133,7 @@ def matches?(request) resources :tps_data_uploads, only: [:new, :create] resources :fraud_risk_csv_uploads, only: [:new, :create] resource :fraud_risk_csv_download, only: :show + resources :reports, only: [:index, :show], param: :name resources :payroll_runs, only: [:index, :new, :create, :show, :destroy] do resources :payment_confirmation_report_uploads, only: [:new, :create] diff --git a/spec/features/admin/reports_spec.rb b/spec/features/admin/reports_spec.rb new file mode 100644 index 0000000000..93ba372d41 --- /dev/null +++ b/spec/features/admin/reports_spec.rb @@ -0,0 +1,102 @@ +require "rails_helper" + +RSpec.describe "Admin reports" do + around do |example| + travel_to Date.new(2024, 12, 6) do + example.run + end + end + + describe "Approved FE claims with failing provider verification" do + it "returns a CSV report" do + claim = create( + :claim, + :approved, + policy: Policies::FurtherEducationPayments, + first_name: "Elizabeth", + surname: "Hoover", + qa_required: true, + eligibility_attributes: { + award_amount: 2_000, + contract_type: "permanent", + verification: { + assertions: [ + { + name: "contract_type", + outcome: true + }, + { + name: "teaching_responsibilities", + outcome: true + }, + { + name: "further_education_teaching_start_year", + outcome: true + }, + { + name: "teaching_hours_per_week", + outcome: true + }, + { + name: "half_teaching_hours", + outcome: false + }, + { + name: "subjects_taught", + outcome: false + }, + { + name: "subject_to_formal_performance_action", + outcome: true + }, + { + name: "subject_to_disciplinary_action", + outcome: true + } + ] + } + } + ) + + create( + :task, + :failed, + name: "provider_verification", + claim: claim + ) + + sign_in_as_service_operator + + visit admin_claims_path + + click_on "Reports" + + click_on( + "FE TRI approved claims whereby the provider check status is 'failed'" + ) + + csv_data = page.body + + csv = CSV.parse(csv_data, headers: true) + row = csv.first + + expect(row.fetch("Claim reference")).to eq(claim.reference) + expect(row.fetch("Full name")).to eq("Elizabeth Hoover") + expect(row.fetch("Claim amount")).to eq("£2,000") + expect(row.fetch("Claim status")).to eq("Approved awaiting QA") + expect(row.fetch("Decision date")).to eq("06/12/2024") + expect(row.fetch("Decision agent")).to eq("Aaron Admin") + expect(row.fetch("Contract of employment")).to eq("Yes") + expect(row.fetch("Teaching responsibilities")).to eq("Yes") + expect(row.fetch("First 5 years of teaching")).to eq("Yes") + expect(row.fetch("One full term")).to eq("N/A") + expect(row.fetch("Timetabled teaching hours")).to eq("Yes") + expect(row.fetch("Age range taught")).to eq("No") + expect(row.fetch("Subject")).to eq("No") + expect(row.fetch("Course")).to eq("??") + expect(row.fetch("2.5 hours weekly teaching")).to eq("N/A") + expect(row.fetch("Performance")).to eq("Yes") + expect(row.fetch("Disciplinary")).to eq("Yes") + end + end +end From a3d539aeda244e17121236aa06f2af9ad07b32a8 Mon Sep 17 00:00:00 2001 From: Richard Lynch Date: Mon, 9 Dec 2024 16:26:21 +0000 Subject: [PATCH 3/5] Adds approved claims failing qualification tasks One gotcha that we might need to deal with in this report is the `dqt_teacher_status`. When the qualification claim verifier runs is uses the dqt record to set some notes on the claim. If the claim has a populated `dqt_teacher_status` field it uses that to build the dqt teacher record object, if not the status is fetched from the api, however this information from the api is not persisted to the claim. When generating the report we don't want to hit the dqt api for potentially many claims, so if we're missing this dqt status we don't include it in the report. Checking the current academic year's claims there don't seem to be any claims that would be returned from this report that are missing their `dqt_teacher_status`. If missing `dqt_teacher_status` in this report is causing issues for the ops team, we could consider parsing the claim notes to get this information. --- app/controllers/admin/reports_controller.rb | 2 + ...roved_claims_failing_qualification_task.rb | 121 ++++++++ app/views/admin/reports/index.html.erb | 6 + spec/features/admin/reports_spec.rb | 64 +++++ ..._claims_failing_qualification_task_spec.rb | 261 ++++++++++++++++++ 5 files changed, 454 insertions(+) create mode 100644 app/models/admin/reports/approved_claims_failing_qualification_task.rb create mode 100644 spec/models/admin/reports/approved_claims_failing_qualification_task_spec.rb diff --git a/app/controllers/admin/reports_controller.rb b/app/controllers/admin/reports_controller.rb index 5081b148e4..0ec277834d 100644 --- a/app/controllers/admin/reports_controller.rb +++ b/app/controllers/admin/reports_controller.rb @@ -19,6 +19,8 @@ def report @report ||= case params[:name] when "fe-approved-claims-with-failing-provider-verification" Reports::FeApprovedClaimsWithFailingProviderVerification.new + when "approved-claims-failing-qualification-task" + Reports::ApprovedClaimsFailingQualificationTask.new else raise ActiveRecord::RecordNotFound end diff --git a/app/models/admin/reports/approved_claims_failing_qualification_task.rb b/app/models/admin/reports/approved_claims_failing_qualification_task.rb new file mode 100644 index 0000000000..b33363886f --- /dev/null +++ b/app/models/admin/reports/approved_claims_failing_qualification_task.rb @@ -0,0 +1,121 @@ +module Admin + module Reports + class ApprovedClaimsFailingQualificationTask + HEADERS = [ + "Claim reference", + "Teacher reference number", + "Policy", + "Status", + "Decision date", + "Decision agent", + "Qualification", + "ITT start year", + "ITT subject", + "ITT subjects", + "ITT start date", + "QTS award date", + "Qualification name" + ] + + def filename + "approved_claims_failing_qualification_task" + end + + def to_csv + CSV.generate( + row_sep: "\r\n", + write_headers: true, + headers: HEADERS + ) do |csv| + rows.each { |row| csv << row } + end + end + + private + + def rows + scope.map(&ClaimPresenter.method(:new)).map(&:to_a) + end + + def scope + Claim + .approved + .where(academic_year: AcademicYear.current) + .joins(:tasks) + .merge(Task.where(name: "qualifications", passed: false)) + .includes(:eligibility, decisions: :created_by) + end + + class ClaimPresenter + include Admin::ClaimsHelper + + def initialize(claim) + @claim = claim + end + + def to_a + [ + claim.reference, + claim.eligibility.teacher_reference_number, + I18n.t("#{claim.policy.locale_key}.policy_acronym"), + status(claim), + I18n.l(approval.created_at.to_date, format: :day_month_year), + approval.created_by.full_name, + qualification, + itt_academic_year&.to_s, + eligible_itt_subject, + dqt_teacher_record.itt_subjects.join(", "), + I18n.l(dqt_teacher_record.itt_start_date, format: :day_month_year), + I18n.l(dqt_teacher_record.qts_award_date, format: :day_month_year), + dqt_teacher_record.qualification_name + ] + end + + private + + attr_reader :claim + + def approval + @approval ||= claim.decisions.reject(&:undone).last + end + + # StudentLoans doesn't have an eligible_itt_subject + def eligible_itt_subject + claim.eligibility.try(:eligible_itt_subject) + end + + # StudentLoans doesn't have an itt_academic_year + def itt_academic_year + claim.eligibility.try(:itt_academic_year) + end + + # StudentLoans doesn't have a qualification + def qualification + claim.eligibility.try(:qualification) + end + + def itt_subjects + dqt_teacher_record&.itt_subjects + end + + def itt_start_date + dqt_teacher_record&.itt_start_date + end + + def qts_award_date + dqt_teacher_record&.qts_award_date + end + + def qualification_name + dqt_teacher_record&.qualification_name + end + + def dqt_teacher_record + @dqt_teacher_record ||= if claim.has_dqt_record? + Dqt::Teacher.new(claim.dqt_teacher_status) + end + end + end + end + end +end diff --git a/app/views/admin/reports/index.html.erb b/app/views/admin/reports/index.html.erb index 8a9d58e73e..29074aefb4 100644 --- a/app/views/admin/reports/index.html.erb +++ b/app/views/admin/reports/index.html.erb @@ -13,5 +13,11 @@ admin_report_path("fe-approved-claims-with-failing-provider-verification", format: :csv), secondary: true ) %> + + <%= govuk_button_link_to( + "Approved claims failing qualification task", + admin_report_path("approved-claims-failing-qualification-task", format: :csv), + secondary: true + ) %> diff --git a/spec/features/admin/reports_spec.rb b/spec/features/admin/reports_spec.rb index 93ba372d41..aa9e3c2e9d 100644 --- a/spec/features/admin/reports_spec.rb +++ b/spec/features/admin/reports_spec.rb @@ -99,4 +99,68 @@ expect(row.fetch("Disciplinary")).to eq("Yes") end end + + describe "Approved claims failing qualification task" do + it "returns a CSV report" do + claim = create( + :claim, + :with_dqt_teacher_status, + :approved, + policy: Policies::LevellingUpPremiumPayments, + first_name: "Elizabeth", + surname: "Hoover", + eligibility_attributes: { + teacher_reference_number: "1234567", + qualification: :postgraduate_itt, + itt_academic_year: "2023/2024", + eligible_itt_subject: :mathematics + }, + dqt_teacher_status: { + initial_teacher_training: { + programme_start_date: "2022-09-01", + subject1: "mathematics", + subject1_code: "G100", + qualification: "BA (Hons)" + }, + qualified_teacher_status: { + qts_date: "2022-12-01" + } + } + ) + + create( + :task, + :failed, + name: "qualifications", + claim: claim + ) + + sign_in_as_service_operator + + visit admin_claims_path + + click_on "Reports" + + click_on "Approved claims failing qualification task" + + csv_data = page.body + + csv = CSV.parse(csv_data, headers: true) + row = csv.first + + expect(row.fetch("Claim reference")).to eq(claim.reference) + expect(row.fetch("Teacher reference number")).to eq("1234567") + expect(row.fetch("Policy")).to eq("STRI") + expect(row.fetch("Status")).to eq("Approved awaiting payroll") + expect(row.fetch("Decision date")).to eq("06/12/2024") + expect(row.fetch("Decision agent")).to eq("Aaron Admin") + expect(row.fetch("Qualification")).to eq("postgraduate_itt") + expect(row.fetch("ITT start year")).to eq("2023/2024") + expect(row.fetch("ITT subject")).to eq("mathematics") + expect(row.fetch("ITT subjects")).to eq("mathematics") + expect(row.fetch("ITT start date")).to eq("01/09/2022") + expect(row.fetch("QTS award date")).to eq("01/12/2022") + expect(row.fetch("Qualification name")).to eq("BA (Hons)") + end + end end diff --git a/spec/models/admin/reports/approved_claims_failing_qualification_task_spec.rb b/spec/models/admin/reports/approved_claims_failing_qualification_task_spec.rb new file mode 100644 index 0000000000..1c95a7d5bd --- /dev/null +++ b/spec/models/admin/reports/approved_claims_failing_qualification_task_spec.rb @@ -0,0 +1,261 @@ +require "rails_helper" + +RSpec.describe Admin::Reports::ApprovedClaimsFailingQualificationTask do + around do |example| + travel_to Date.new(2024, 11, 1) do + example.run + end + end + + describe "to_csv" do + it "returns a csv of the claims" do + # excluded, claim not approved + ecp_claim_unapporved_failed_qualification_task = create( + :claim, + policy: Policies::EarlyCareerPayments, + academic_year: AcademicYear.new(2024) + ) + + create( + :task, + :failed, + name: "qualifications", + claim: ecp_claim_unapporved_failed_qualification_task + ) + + # excluded, task passed + lup_claim_approved_passed_qualification_task = create( + :claim, + :approved, + policy: Policies::LevellingUpPremiumPayments, + academic_year: AcademicYear.new(2024) + ) + + create( + :task, + :passed, + name: "qualifications", + claim: lup_claim_approved_passed_qualification_task + ) + + # excluded, claim not approved + tslr_claim_rejected = create( + :claim, + :rejected, + policy: Policies::StudentLoans, + academic_year: AcademicYear.new(2024) + ) + + create( + :task, + :failed, + name: "qualifications", + claim: tslr_claim_rejected + ) + + # excluded, wrong policy! + _fe_claim = create( + :claim, + :approved, + policy: Policies::FurtherEducationPayments, + academic_year: AcademicYear.new(2024) + ) + + # excluded, previous academic year + ecp_claim_unapporved_failed_qualification_task = create( + :claim, + :approved, + policy: Policies::EarlyCareerPayments, + academic_year: AcademicYear.new(2023) + ) + + create( + :task, + :failed, + name: "qualifications", + claim: ecp_claim_unapporved_failed_qualification_task + ) + + # included + ecp_claim_approved_failed_qualification_task = create( + :claim, + :approved, + :flagged_for_qa, + policy: Policies::EarlyCareerPayments, + academic_year: AcademicYear.new(2024), + decision_creator: create( + :dfe_signin_user, + given_name: "Some", + family_name: "admin" + ), + eligibility_attributes: { + teacher_reference_number: "1111111", + eligible_itt_subject: :mathematics, + itt_academic_year: AcademicYear.new(2021), + qualification: :postgraduate_itt + }, + dqt_teacher_status: { + qualified_teacher_status: { + qts_date: "2023-09-01", + name: "Qualified teacher (trained)" + }, + initial_teacher_training: { + programme_start_date: "2022-08-01", + subject1: "mathematics", + subject1_code: "100403", + subject2: "physics", + subject3_code: "F300", + qualification: "Graduate Diploma" + } + } + ) + + create( + :task, + :failed, + name: "qualifications", + claim: ecp_claim_approved_failed_qualification_task + ) + + # included + lup_claim_approved_failed_qualification_task = create( + :claim, + :approved, + policy: Policies::LevellingUpPremiumPayments, + academic_year: AcademicYear.new(2024), + decision_creator: create( + :dfe_signin_user, + given_name: "Some", + family_name: "admin" + ), + eligibility_attributes: { + teacher_reference_number: "2222222", + eligible_itt_subject: :physics, + itt_academic_year: AcademicYear.new(2021), + qualification: :postgraduate_itt + }, + dqt_teacher_status: { + qualified_teacher_status: { + qts_date: "2023-10-01", + name: "Qualified teacher (trained)" + }, + initial_teacher_training: { + programme_start_date: "2022-08-02", + subject1: "physics", + subject1_code: "F300", + qualification: "Graduate Diploma" + } + } + ) + + create(:payment, claims: [lup_claim_approved_failed_qualification_task]) + + create( + :task, + :failed, + name: "qualifications", + claim: lup_claim_approved_failed_qualification_task + ) + + # included + tslr_claim_approved_failed_qualification_task = create( + :claim, + :approved, + policy: Policies::StudentLoans, + academic_year: AcademicYear.new(2024), + decision_creator: create( + :dfe_signin_user, + given_name: "Some", + family_name: "admin" + ), + eligibility_attributes: { + teacher_reference_number: "3333333" + }, + dqt_teacher_status: { + qualified_teacher_status: { + qts_date: "2023-10-01", + name: "Qualified teacher (trained)" + }, + initial_teacher_training: { + programme_start_date: "2022-08-02", + subject1: "physics", + subject1_code: "F300", + qualification: "Graduate Diploma" + } + } + ) + + create( + :task, + :failed, + name: "qualifications", + claim: tslr_claim_approved_failed_qualification_task + ) + + csv = CSV.parse(described_class.new.to_csv, headers: true) + + expect(csv.to_a).to match_array([ + [ + "Claim reference", + "Teacher reference number", + "Policy", + "Status", + "Decision date", + "Decision agent", + "Qualification", + "ITT start year", + "ITT subject", + "ITT subjects", + "ITT start date", + "QTS award date", + "Qualification name" + ], + [ + ecp_claim_approved_failed_qualification_task.reference, + "1111111", + "ECP", + "Approved awaiting QA", + "01/11/2024", + "Some admin", + "postgraduate_itt", + "2021/2022", + "mathematics", + "mathematics, physics", + "01/08/2022", + "01/09/2023", + "Graduate Diploma" + ], + [ + lup_claim_approved_failed_qualification_task.reference, + "2222222", + "STRI", + "Payrolled", + "01/11/2024", + "Some admin", + "postgraduate_itt", + "2021/2022", + "physics", + "physics", + "02/08/2022", + "01/10/2023", + "Graduate Diploma" + ], + [ + tslr_claim_approved_failed_qualification_task.reference, + "3333333", + "TSLR", + "Approved awaiting payroll", + "01/11/2024", + "Some admin", + nil, + nil, + nil, + "physics", + "02/08/2022", + "01/10/2023", + "Graduate Diploma" + ] + ]) + end + end +end From 7cc144b915a3a725e7cd392f6d83b7e998f3aa6e Mon Sep 17 00:00:00 2001 From: Richard Lynch Date: Mon, 16 Dec 2024 14:36:21 +0000 Subject: [PATCH 4/5] Add csv extension Noticed we were missing the csv extension on the report --- .../admin/reports/approved_claims_failing_qualification_task.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/admin/reports/approved_claims_failing_qualification_task.rb b/app/models/admin/reports/approved_claims_failing_qualification_task.rb index b33363886f..c870c05ae0 100644 --- a/app/models/admin/reports/approved_claims_failing_qualification_task.rb +++ b/app/models/admin/reports/approved_claims_failing_qualification_task.rb @@ -18,7 +18,7 @@ class ApprovedClaimsFailingQualificationTask ] def filename - "approved_claims_failing_qualification_task" + "approved_claims_failing_qualification_task.csv" end def to_csv From 07013acfa2371717e4ee56848e7ae04da12c48ee Mon Sep 17 00:00:00 2001 From: Richard Lynch Date: Thu, 19 Dec 2024 15:43:19 +0000 Subject: [PATCH 5/5] Add duplicate claims check We're soon going to want to generate a report for the ops team of the duplicate claims. This commit introduces a new claim verifier that records the duplicate claims, if any. Once existing duplicates on production have been back filled we can replace other uses of `MatchingAttributeFinder#matching_claims` with `claim.duplicates` and also join claims to their duplicates when generating the duplicate claims report. --- .../claim_verifiers/duplicate_claims_check.rb | 37 ++++++++ app/models/claim.rb | 20 ++++ app/models/claims.rb | 5 + app/models/claims/claim_duplicate.rb | 28 ++++++ app/models/policies/early_career_payments.rb | 3 +- app/models/policies/early_years_payments.rb | 3 +- .../policies/further_education_payments.rb | 3 +- .../international_relocation_payments.rb | 3 +- .../policies/levelling_up_premium_payments.rb | 3 +- app/models/policies/student_loans.rb | 3 +- ...19115223_create_claims_claim_duplicates.rb | 33 +++++++ db/schema.rb | 15 ++- .../duplicate_claims_check_spec.rb | 82 +++++++++++++++++ spec/models/claims/claim_duplicate_spec.rb | 92 +++++++++++++++++++ 14 files changed, 323 insertions(+), 7 deletions(-) create mode 100644 app/models/automated_checks/claim_verifiers/duplicate_claims_check.rb create mode 100644 app/models/claims.rb create mode 100644 app/models/claims/claim_duplicate.rb create mode 100644 db/migrate/20241219115223_create_claims_claim_duplicates.rb create mode 100644 spec/models/automated_checks/claim_verifiers/duplicate_claims_check_spec.rb create mode 100644 spec/models/claims/claim_duplicate_spec.rb diff --git a/app/models/automated_checks/claim_verifiers/duplicate_claims_check.rb b/app/models/automated_checks/claim_verifiers/duplicate_claims_check.rb new file mode 100644 index 0000000000..10faaa2c2a --- /dev/null +++ b/app/models/automated_checks/claim_verifiers/duplicate_claims_check.rb @@ -0,0 +1,37 @@ +module AutomatedChecks + module ClaimVerifiers + class DuplicateClaimsCheck + def initialize(claim:) + @claim = claim + end + + def perform + matching_attribute_finder.matching_claims.each do |existing_claim| + if existing_claim.created_at < claim.created_at + original_claim = existing_claim + duplicate_claim = claim + else + original_claim = claim + duplicate_claim = existing_claim + end + + unless Claims::ClaimDuplicate.exists?(original_claim: original_claim, duplicate_claim: duplicate_claim) + Claims::ClaimDuplicate.create!( + original_claim: original_claim, + duplicate_claim: duplicate_claim, + matching_attributes: matching_attribute_finder.matching_attributes(existing_claim) + ) + end + end + end + + private + + attr_reader :claim + + def matching_attribute_finder + @matching_attribute_finder ||= Claim::MatchingAttributeFinder.new(claim) + end + end + end +end diff --git a/app/models/claim.rb b/app/models/claim.rb index bb0491bc35..6705561c01 100644 --- a/app/models/claim.rb +++ b/app/models/claim.rb @@ -142,6 +142,26 @@ class Claim < ApplicationRecord inverse_of: :assigned_claims, optional: true + has_many :claim_duplicates_as_original_claim, + class_name: "Claims::ClaimDuplicate", + foreign_key: :original_claim_id, + dependent: :destroy + + has_many :claim_duplicates_as_duplicate_claim, + class_name: "Claims::ClaimDuplicate", + foreign_key: :duplicate_claim_id, + dependent: :destroy + + has_many :duplicates, + through: :claim_duplicates_as_original_claim, + source: :duplicate_claim, + class_name: "Claim" + + has_many :originals, + through: :claim_duplicates_as_duplicate_claim, + source: :original_claim, + class_name: "Claim" + enum :payroll_gender, { dont_know: 0, female: 1, diff --git a/app/models/claims.rb b/app/models/claims.rb new file mode 100644 index 0000000000..bd683b58be --- /dev/null +++ b/app/models/claims.rb @@ -0,0 +1,5 @@ +module Claims + def self.table_name_prefix + "claims_" + end +end diff --git a/app/models/claims/claim_duplicate.rb b/app/models/claims/claim_duplicate.rb new file mode 100644 index 0000000000..e84bf0501d --- /dev/null +++ b/app/models/claims/claim_duplicate.rb @@ -0,0 +1,28 @@ +module Claims + class ClaimDuplicate < ApplicationRecord + belongs_to :original_claim, class_name: "Claim" + belongs_to :duplicate_claim, class_name: "Claim" + + validates :duplicate_claim, uniqueness: { + scope: :original_claim, + message: "has already been registered as a duplicate" + } + validate :claims_are_not_the_same, if: -> { original_claim && duplicate_claim } + validate :original_claim_is_older, if: -> { original_claim && duplicate_claim } + validates :matching_attributes, presence: true + + private + + def claims_are_not_the_same + return unless original_claim == duplicate_claim + + errors.add(:duplicate_claim, "can't be the same as the original claim") + end + + def original_claim_is_older + return unless original_claim.created_at > duplicate_claim.created_at + + errors.add(:original_claim, "must be older than the duplicate claim") + end + end +end diff --git a/app/models/policies/early_career_payments.rb b/app/models/policies/early_career_payments.rb index bc5f165769..50f9461210 100644 --- a/app/models/policies/early_career_payments.rb +++ b/app/models/policies/early_career_payments.rb @@ -20,7 +20,8 @@ module EarlyCareerPayments AutomatedChecks::ClaimVerifiers::CensusSubjectsTaught, AutomatedChecks::ClaimVerifiers::Employment, AutomatedChecks::ClaimVerifiers::StudentLoanPlan, - AutomatedChecks::ClaimVerifiers::FraudRisk + AutomatedChecks::ClaimVerifiers::FraudRisk, + AutomatedChecks::ClaimVerifiers::DuplicateClaimsCheck ].freeze POLICY_START_YEAR = AcademicYear.new(2021).freeze diff --git a/app/models/policies/early_years_payments.rb b/app/models/policies/early_years_payments.rb index 379385bd7c..b7e8e6204f 100644 --- a/app/models/policies/early_years_payments.rb +++ b/app/models/policies/early_years_payments.rb @@ -18,7 +18,8 @@ module EarlyYearsPayments VERIFIERS = [ AutomatedChecks::ClaimVerifiers::StudentLoanPlan, - AutomatedChecks::ClaimVerifiers::EarlyYearsPayments::Identity + AutomatedChecks::ClaimVerifiers::EarlyYearsPayments::Identity, + AutomatedChecks::ClaimVerifiers::DuplicateClaimsCheck ] # Attributes to delete from claims submitted before the current academic diff --git a/app/models/policies/further_education_payments.rb b/app/models/policies/further_education_payments.rb index 323fdaa035..e6a329474d 100644 --- a/app/models/policies/further_education_payments.rb +++ b/app/models/policies/further_education_payments.rb @@ -22,7 +22,8 @@ module FurtherEducationPayments AutomatedChecks::ClaimVerifiers::ProviderVerification, AutomatedChecks::ClaimVerifiers::Employment, AutomatedChecks::ClaimVerifiers::StudentLoanPlan, - AutomatedChecks::ClaimVerifiers::FraudRisk + AutomatedChecks::ClaimVerifiers::FraudRisk, + AutomatedChecks::ClaimVerifiers::DuplicateClaimsCheck ] # Options shown to admins when rejecting a claim diff --git a/app/models/policies/international_relocation_payments.rb b/app/models/policies/international_relocation_payments.rb index 963bb5d07d..2fde881a10 100644 --- a/app/models/policies/international_relocation_payments.rb +++ b/app/models/policies/international_relocation_payments.rb @@ -4,7 +4,8 @@ module InternationalRelocationPayments extend self VERIFIERS = [ - AutomatedChecks::ClaimVerifiers::FraudRisk + AutomatedChecks::ClaimVerifiers::FraudRisk, + AutomatedChecks::ClaimVerifiers::DuplicateClaimsCheck ].freeze ELIGIBILITY_MATCHING_ATTRIBUTES = [["passport_number"]].freeze diff --git a/app/models/policies/levelling_up_premium_payments.rb b/app/models/policies/levelling_up_premium_payments.rb index e2a41761a7..de0a89f938 100644 --- a/app/models/policies/levelling_up_premium_payments.rb +++ b/app/models/policies/levelling_up_premium_payments.rb @@ -10,7 +10,8 @@ module LevellingUpPremiumPayments AutomatedChecks::ClaimVerifiers::CensusSubjectsTaught, AutomatedChecks::ClaimVerifiers::Employment, AutomatedChecks::ClaimVerifiers::StudentLoanPlan, - AutomatedChecks::ClaimVerifiers::FraudRisk + AutomatedChecks::ClaimVerifiers::FraudRisk, + AutomatedChecks::ClaimVerifiers::DuplicateClaimsCheck ].freeze # Used in diff --git a/app/models/policies/student_loans.rb b/app/models/policies/student_loans.rb index 16862dba0f..e79e351288 100644 --- a/app/models/policies/student_loans.rb +++ b/app/models/policies/student_loans.rb @@ -19,7 +19,8 @@ module StudentLoans AutomatedChecks::ClaimVerifiers::CensusSubjectsTaught, AutomatedChecks::ClaimVerifiers::Employment, AutomatedChecks::ClaimVerifiers::StudentLoanAmount, - AutomatedChecks::ClaimVerifiers::FraudRisk + AutomatedChecks::ClaimVerifiers::FraudRisk, + AutomatedChecks::ClaimVerifiers::DuplicateClaimsCheck ].freeze POLICY_START_YEAR = AcademicYear.new(2013).freeze diff --git a/db/migrate/20241219115223_create_claims_claim_duplicates.rb b/db/migrate/20241219115223_create_claims_claim_duplicates.rb new file mode 100644 index 0000000000..649f84f0b4 --- /dev/null +++ b/db/migrate/20241219115223_create_claims_claim_duplicates.rb @@ -0,0 +1,33 @@ +class CreateClaimsClaimDuplicates < ActiveRecord::Migration[8.0] + def change + create_table :claims_claim_duplicates, id: :uuid do |t| + t.belongs_to( + :original_claim, + null: false, + foreign_key: { + to_table: :claims + }, + type: :uuid + ) + + t.belongs_to( + :duplicate_claim, + null: false, + foreign_key: { + to_table: :claims + }, + type: :uuid + ) + + t.jsonb :matching_attributes, default: [] + + t.timestamps + end + + add_index( + :claims_claim_duplicates, + [:original_claim_id, :duplicate_claim_id], + unique: true + ) + end +end diff --git a/db/schema.rb b/db/schema.rb index ff5c7e9cb2..18b85d679e 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2024_11_26_105650) do +ActiveRecord::Schema[8.0].define(version: 2024_12_19_115223) do # These are extensions that must be enabled in order to support this database enable_extension "citext" enable_extension "pg_catalog.plpgsql" @@ -123,6 +123,17 @@ t.index ["submitted_at"], name: "index_claims_on_submitted_at" end + create_table "claims_claim_duplicates", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + t.uuid "original_claim_id", null: false + t.uuid "duplicate_claim_id", null: false + t.jsonb "matching_attributes", default: [] + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["duplicate_claim_id"], name: "index_claims_claim_duplicates_on_duplicate_claim_id" + t.index ["original_claim_id", "duplicate_claim_id"], name: "idx_on_original_claim_id_duplicate_claim_id_5ce2c01567", unique: true + t.index ["original_claim_id"], name: "index_claims_claim_duplicates_on_original_claim_id" + end + create_table "decisions", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| t.integer "result" t.uuid "claim_id" @@ -585,6 +596,8 @@ add_foreign_key "claim_payments", "claims" add_foreign_key "claim_payments", "payments" add_foreign_key "claims", "journeys_sessions" + add_foreign_key "claims_claim_duplicates", "claims", column: "duplicate_claim_id" + add_foreign_key "claims_claim_duplicates", "claims", column: "original_claim_id" add_foreign_key "decisions", "dfe_sign_in_users", column: "created_by_id" add_foreign_key "early_career_payments_eligibilities", "schools", column: "current_school_id" add_foreign_key "eligible_ey_providers", "local_authorities" diff --git a/spec/models/automated_checks/claim_verifiers/duplicate_claims_check_spec.rb b/spec/models/automated_checks/claim_verifiers/duplicate_claims_check_spec.rb new file mode 100644 index 0000000000..8020c659e3 --- /dev/null +++ b/spec/models/automated_checks/claim_verifiers/duplicate_claims_check_spec.rb @@ -0,0 +1,82 @@ +require "rails_helper" + +RSpec.describe AutomatedChecks::ClaimVerifiers::DuplicateClaimsCheck do + describe "#perform" do + context "when the claim has duplicates" do + it "marks the claim as having duplicates" do + existing_claim = create( + :claim, + :submitted, + email_address: "test@example.com", + first_name: "one" + ) + + new_claim = create( + :claim, + :submitted, + email_address: "test@example.com", + first_name: "two" + ) + + described_class.new(claim: new_claim).perform + + expect(existing_claim.reload.duplicates).to include(new_claim) + expect(new_claim.reload.originals).to include(existing_claim) + + claim_duplicate = new_claim.claim_duplicates_as_duplicate_claim.last + + expect(claim_duplicate.original_claim).to eq(existing_claim) + expect(claim_duplicate.duplicate_claim).to eq(new_claim) + expect(claim_duplicate.matching_attributes).to eq(["email_address"]) + end + + it "is idempotent" do + existing_claim = create( + :claim, + :submitted, + email_address: "test@example.com", + first_name: "one" + ) + + new_claim = create( + :claim, + :submitted, + email_address: "test@example.com", + first_name: "two" + ) + + Claims::ClaimDuplicate.create!( + original_claim: existing_claim, + duplicate_claim: new_claim, + matching_attributes: ["email_address"] + ) + + expect { described_class.new(claim: new_claim).perform }.not_to( + change(existing_claim.duplicates, :count) + ) + end + end + + context "when the claim has no duplicates" do + it "does not mark the claim as having duplicates" do + existing_claim = create( + :claim, + :submitted, + email_address: "test1@example.com", + first_name: "one" + ) + + new_claim = create( + :claim, + :submitted, + email_address: "test2@example.com", + first_name: "two" + ) + + described_class.new(claim: new_claim).perform + + expect(existing_claim.reload.duplicates).not_to include(new_claim) + end + end + end +end diff --git a/spec/models/claims/claim_duplicate_spec.rb b/spec/models/claims/claim_duplicate_spec.rb new file mode 100644 index 0000000000..9eb89b4117 --- /dev/null +++ b/spec/models/claims/claim_duplicate_spec.rb @@ -0,0 +1,92 @@ +require "rails_helper" + +RSpec.describe Claims::ClaimDuplicate, type: :model do + describe "validations" do + it { is_expected.to validate_presence_of(:matching_attributes) } + + describe "uniquness" do + it "doesn't allow a duplicate to be registered more than once" do + original_claim = create(:claim, created_at: 1.day.ago) + duplicate_claim = create(:claim, created_at: Time.zone.now) + + described_class.create!( + original_claim: original_claim, + duplicate_claim: duplicate_claim, + matching_attributes: ["email_address"] + ) + + claim_duplicate = described_class.new( + original_claim: original_claim, + duplicate_claim: duplicate_claim, + matching_attributes: ["email_address"] + ) + + expect(claim_duplicate).not_to be_valid + expect(claim_duplicate.errors[:duplicate_claim]).to include( + "has already been registered as a duplicate" + ) + end + end + + describe "original_claim_is_older" do + it "is valid when the original claim is older" do + original_claim = create(:claim, created_at: 1.day.ago) + duplicate_claim = create(:claim, created_at: Time.zone.now) + + claim_duplicate = described_class.new( + original_claim: original_claim, + duplicate_claim: duplicate_claim, + matching_attributes: ["email_address"] + ) + + expect(claim_duplicate).to be_valid + end + + it "is invalid when the original claim is newer" do + original_claim = create(:claim, created_at: Time.zone.now) + duplicate_claim = create(:claim, created_at: 1.day.ago) + + claim_duplicate = described_class.new( + original_claim: original_claim, + duplicate_claim: duplicate_claim, + matching_attributes: ["email_address"] + ) + + expect(claim_duplicate).not_to be_valid + expect(claim_duplicate.errors[:original_claim]).to include( + "must be older than the duplicate claim" + ) + end + end + + describe "claims_are_not_the_same" do + it "is valid when the claims are different" do + original_claim = create(:claim) + duplicate_claim = create(:claim) + + claim_duplicate = described_class.new( + original_claim: original_claim, + duplicate_claim: duplicate_claim, + matching_attributes: ["email_address"] + ) + + expect(claim_duplicate).to be_valid + end + + it "is invalid when the claims are the same" do + claim = create(:claim) + + claim_duplicate = described_class.new( + original_claim: claim, + duplicate_claim: claim, + matching_attributes: ["email_address"] + ) + + expect(claim_duplicate).not_to be_valid + expect(claim_duplicate.errors[:duplicate_claim]).to include( + "can't be the same as the original claim" + ) + end + end + end +end