Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CAPT-1962: Sync batch database operations with BigQuery #3378

Merged
merged 2 commits into from
Nov 8, 2024
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
CAPT-1962: Sync batch database operations with BigQuery
slorek committed Nov 8, 2024
commit 08bcb7ef3228d08e3a990219e96a45a40543cd64
2 changes: 1 addition & 1 deletion app/jobs/employment_check_job.rb
Original file line number Diff line number Diff line change
@@ -14,7 +14,7 @@ def delete_employment_tasks
claim_ids = claims_awaiting_decision_without_passed_check.pluck(:id)

claim_ids.each_slice(500) do |ids|
Task.where(claim_id: ids, name: "employment").delete_all
Task.where(claim_id: ids, name: "employment").destroy_all
end
end

5 changes: 4 additions & 1 deletion app/jobs/import_census_job.rb
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
class ImportCensusJob < FileImporterJob
import_with SchoolWorkforceCensusDataImporter
rescue_with -> { SchoolWorkforceCensus.delete_all }
rescue_with -> do
SchoolWorkforceCensus.delete_all
Rake::Task["dfe:analytics:import_entity"].invoke(SchoolWorkforceCensus.table_name) if DfE::Analytics.enabled?
end
notify_with AdminMailer, success: :census_csv_processing_success, failure: :census_csv_processing_error
end
5 changes: 4 additions & 1 deletion app/jobs/import_student_loans_data_job.rb
Original file line number Diff line number Diff line change
@@ -5,5 +5,8 @@ class ImportStudentLoansDataJob < FileImporterJob
StudentLoanAmountCheckJob.perform_later
StudentLoanPlanCheckJob.perform_later
end
rescue_with -> { StudentLoansData.delete_all }
rescue_with -> do
StudentLoansData.delete_all
Rake::Task["dfe:analytics:import_entity"].invoke(StudentLoansData.table_name) if DfE::Analytics.enabled?
end
end
2 changes: 1 addition & 1 deletion app/jobs/qualifications_no_match_check_job.rb
Original file line number Diff line number Diff line change
@@ -18,7 +18,7 @@ def perform(filter: nil)
claims.each_slice(250).with_index do |cl, index|
sleep 60 unless index.zero?

Task.where(claim_id: cl.pluck(:id), name: "qualifications").delete_all
Task.where(claim_id: cl.pluck(:id), name: "qualifications").destroy_all

cl.each do |claim|
AutomatedChecks::ClaimVerifiers::Qualifications.new(
2 changes: 1 addition & 1 deletion app/jobs/student_loan_amount_check_job.rb
Original file line number Diff line number Diff line change
@@ -15,7 +15,7 @@ def delete_no_data_student_loan_amount_tasks
claim_ids = current_year_tslr_claims_with_no_data_tasks.pluck(:id)

claim_ids.each_slice(500) do |ids|
Task.where(claim_id: ids, name: "student_loan_amount", claim_verifier_match: nil, manual: false).delete_all
Task.where(claim_id: ids, name: "student_loan_amount", claim_verifier_match: nil, manual: false).destroy_all
end
end

2 changes: 1 addition & 1 deletion app/jobs/student_loan_plan_check_job.rb
Original file line number Diff line number Diff line change
@@ -21,7 +21,7 @@ def delete_no_data_student_loan_plan_tasks
claim_ids = current_year_claims_with_no_data_tasks.pluck(:id)

claim_ids.each_slice(500) do |ids|
Task.where(claim_id: ids, name: "student_loan_plan", claim_verifier_match: nil, manual: false).delete_all
Task.where(claim_id: ids, name: "student_loan_plan", claim_verifier_match: nil, manual: false).destroy_all
end
end

2 changes: 1 addition & 1 deletion app/models/dfe_sign_in/user.rb
Original file line number Diff line number Diff line change
@@ -65,7 +65,7 @@ def mark_as_deleted!
private

def unassign_claims
assigned_claims.update_all(assigned_to_id: nil)
assigned_claims.update(assigned_to_id: nil)
end
end
end
Original file line number Diff line number Diff line change
@@ -29,7 +29,7 @@ def process

def commit!
LevellingUpPremiumPayments::Award.transaction do
LevellingUpPremiumPayments::Award.where(academic_year: academic_year.to_s).delete_all
LevellingUpPremiumPayments::Award.where(academic_year: academic_year.to_s).destroy_all
@records.each(&:save!)
end
end
3 changes: 1 addition & 2 deletions lib/claim_allocator.rb
Original file line number Diff line number Diff line change
@@ -8,7 +8,6 @@ def initialize(claim_ids:, admin_user_id:)

def call
# Avoid users clashing
Claim.where(id: claim_ids, assigned_to: nil)
.update_all(assigned_to_id: admin_user_id)
Claim.where(id: claim_ids, assigned_to: nil).update(assigned_to_id: admin_user_id)
end
end
3 changes: 1 addition & 2 deletions lib/claim_deallocator.rb
Original file line number Diff line number Diff line change
@@ -7,7 +7,6 @@ def initialize(claim_ids:, admin_user_id:)
end

def call
Claim.where(id: claim_ids, assigned_to: admin_user_id)
.update_all(assigned_to_id: nil)
Claim.where(id: claim_ids, assigned_to: admin_user_id).update(assigned_to_id: nil)
end
end
2 changes: 2 additions & 0 deletions lib/csv_importer/base.rb
Original file line number Diff line number Diff line change
@@ -32,6 +32,8 @@ def run

target_data_model.insert_all(record_hashes) unless record_hashes.empty?
end

Rake::Task["dfe:analytics:import_entity"].invoke(target_data_model.table_name) if DfE::Analytics.enabled?
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would be tempted to wrap calling this rake task with our own model (DfEAnalytics.sync(SomeClass)) - we could then try stubbing the call to that class in the tests and potentially that would work on CI 🤷

end

def rows_with_data_count
9 changes: 9 additions & 0 deletions spec/jobs/import_census_job_spec.rb
Original file line number Diff line number Diff line change
@@ -46,6 +46,15 @@
# keeps the file upload for debugging
expect(FileUpload.find_by_id(file_upload.id)).to be_present
end

describe "dfe-analytics syncing", :with_dfe_analytics_enabled do
let(:dbl) { double(run: true) }
it "invokes the relevant import entity job" do
expect(DfE::Analytics::LoadEntities).to receive(:new).with(entity_name: SchoolWorkforceCensus.table_name).and_return(dbl)
expect(dbl).to receive(:run)
subject.perform(file_upload.id)
end
end
end
end
end
9 changes: 9 additions & 0 deletions spec/jobs/import_student_loans_data_job_spec.rb
Original file line number Diff line number Diff line change
@@ -53,6 +53,15 @@
it "does not enqueue StudentLoanPlanCheckJob" do
expect { upload }.not_to have_enqueued_job(StudentLoanPlanCheckJob)
end

describe "dfe-analytics syncing", :with_dfe_analytics_enabled do
let(:dbl) { double(run: true) }
it "invokes the relevant import entity job" do
expect(DfE::Analytics::LoadEntities).to receive(:new).with(entity_name: StudentLoansData.table_name).and_return(dbl)
expect(dbl).to receive(:run)
upload
end
end
end
end
end
11 changes: 10 additions & 1 deletion spec/lib/csv_importer/base_spec.rb
Original file line number Diff line number Diff line change
@@ -23,7 +23,7 @@
skip_row_if:
}
end
let(:target_data_model) { double("TargetDataModel", delete_all: nil, insert_all: nil, is_a?: true) }
let(:target_data_model) { double("TargetDataModel", delete_all: nil, insert_all: nil, is_a?: true, table_name: "schools") }
let(:append_only) { false }
let(:batch_size) { nil }
let(:parse_headers) { true }
@@ -79,6 +79,15 @@
end
end

describe "dfe-analytics syncing", :with_dfe_analytics_enabled do
let(:dbl) { double(run: true) }
it "invokes the relevant import entity job" do
expect(DfE::Analytics::LoadEntities).to receive(:new).with(entity_name: target_data_model.table_name).and_return(dbl)
expect(dbl).to receive(:run)
importer.run
end
end

describe "data transformation and importing" do
before { importer.run }

2 changes: 2 additions & 0 deletions spec/rails_helper.rb
Original file line number Diff line number Diff line change
@@ -91,3 +91,5 @@
config.filter_run_excluding js: true unless ENV["RUN_JS_SPECS"] == "true"
config.filter_run_excluding slow: true unless ENV["RUN_SLOW_SPECS"] == "true"
end

Rails.application.load_tasks
8 changes: 8 additions & 0 deletions spec/support/dfe_analytics.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
RSpec.shared_context "with DfE Analytics enabled", shared_context: :metadata do
before { allow(DfE::Analytics).to receive(:enabled?).and_return(true) }
after { allow(DfE::Analytics).to receive(:enabled?).and_return(false) }
end

RSpec.configure do |rspec|
rspec.include_context "with DfE Analytics enabled", with_dfe_analytics_enabled: true
end