From 49f43b741c0e7dec7b9cc56a2db174f15630d5e3 Mon Sep 17 00:00:00 2001 From: Vincent Pochet Date: Tue, 26 Nov 2024 18:18:03 +0100 Subject: [PATCH] Move clocks into recurring config --- app/jobs/clock/events_validation_job.rb | 5 +- .../refresh_wallets_ongoing_balance_job.rb | 2 + clock.rb | 144 -------------- config/initializers/rsa_keys.rb | 2 +- config/recurring.yml | 72 +++++++ scripts/start.clock.dev.sh | 5 - scripts/start.clock.sh | 4 - spec/clockwork_spec.rb | 181 ------------------ 8 files changed, 78 insertions(+), 337 deletions(-) delete mode 100644 clock.rb delete mode 100755 scripts/start.clock.dev.sh delete mode 100755 scripts/start.clock.sh delete mode 100644 spec/clockwork_spec.rb diff --git a/app/jobs/clock/events_validation_job.rb b/app/jobs/clock/events_validation_job.rb index 7c20e08feda..85a85f3f73a 100644 --- a/app/jobs/clock/events_validation_job.rb +++ b/app/jobs/clock/events_validation_job.rb @@ -5,10 +5,11 @@ class EventsValidationJob < ApplicationJob include SentryCronConcern queue_as 'clock' - - unique :until_executed + limits_concurrency to: 1, key: 'post_validate_events', duration: 1.hour def perform + return if ActiveModel::Type::Boolean.new.cast(ENV['LAGO_DISABLE_EVENTS_VALIDATION']) + # NOTE: refresh the last hour events materialized view Scenic.database.refresh_materialized_view( Events::LastHourMv.table_name, diff --git a/app/jobs/clock/refresh_wallets_ongoing_balance_job.rb b/app/jobs/clock/refresh_wallets_ongoing_balance_job.rb index 725d739ab6b..37734ce7d52 100644 --- a/app/jobs/clock/refresh_wallets_ongoing_balance_job.rb +++ b/app/jobs/clock/refresh_wallets_ongoing_balance_job.rb @@ -9,6 +9,8 @@ class RefreshWalletsOngoingBalanceJob < ApplicationJob def perform return unless License.premium? + return if ActiveModel::Type::Boolean.new.cast(ENV['LAGO_DISABLE_WALLET_REFRESH']) + return unless ENV['LAGO_MEMCACHE_SERVERS'].present? || ENV['LAGO_REDIS_CACHE_URL'].present? Wallet.active.ready_to_be_refreshed.find_each do |wallet| Wallets::RefreshOngoingBalanceJob.perform_later(wallet) diff --git a/clock.rb b/clock.rb deleted file mode 100644 index a1dda09e8bd..00000000000 --- a/clock.rb +++ /dev/null @@ -1,144 +0,0 @@ -# frozen_string_literal: true - -require 'clockwork' -require './config/boot' -require './config/environment' - -# TODO(solid_queue): Remove file -module Clockwork - handler do |job, time| - puts "Running #{job} at #{time}" - end - - error_handler do |error| - Rails.logger.error(error.message) - Rails.logger.error(error.backtrace.join("\n")) - - Sentry.capture_exception(error) - end - - # NOTE: All clocks run every hour to take customer timezones into account - - # every(5.minutes, 'schedule:activate_subscriptions') do - # Clock::ActivateSubscriptionsJob - # .set(sentry: {"slug" => 'lago_activate_subscriptions', "cron" => '*/5 * * * *'}) - # .perform_later - # end - - every(5.minutes, 'schedule:refresh_draft_invoices') do - Clock::RefreshDraftInvoicesJob - .set(sentry: {"slug" => 'lago_refresh_draft_invoices', "cron" => '*/5 * * * *'}) - .perform_later - end - - lifetime_usage_refresh_interval = ENV["LAGO_LIFETIME_USAGE_REFRESH_INTERVAL_SECONDS"].presence || 5.minutes - every(lifetime_usage_refresh_interval.to_i.seconds, 'schedule:refresh_lifetime_usages') do - Clock::RefreshLifetimeUsagesJob - .set(sentry: {"slug" => 'lago_refresh_lifetime_usages', "cron" => "#{lifetime_usage_refresh_interval} interval"}) - .perform_later - end - - if ENV['LAGO_MEMCACHE_SERVERS'].present? || ENV['LAGO_REDIS_CACHE_URL'].present? - every(5.minutes, 'schedule:refresh_wallets_ongoing_balance') do - unless ENV['LAGO_DISABLE_WALLET_REFRESH'] == 'true' - Clock::RefreshWalletsOngoingBalanceJob - .set(sentry: {"slug" => 'lago_refresh_wallets_ongoing_balance', "cron" => '*/5 * * * *'}) - .perform_later - end - end - end - - every(1.hour, 'schedule:terminate_ended_subscriptions', at: '*:05') do - Clock::TerminateEndedSubscriptionsJob - .set(sentry: {"slug" => 'lago_terminate_ended_subscriptions', "cron" => '5 */1 * * *'}) - .perform_later - end - - every(1.hour, 'schedule:bill_customers', at: '*:10') do - Clock::SubscriptionsBillerJob - .set(sentry: {"slug" => 'lago_bill_customers', "cron" => '10 */1 * * *'}) - .perform_later - end - - every(1.hour, 'schedule:api_keys_track_usage', at: '*:15') do - Clock::ApiKeys::TrackUsageJob - .set(sentry: {"slug" => 'lago_api_keys_track_usage', "cron" => '15 */1 * * *'}) - .perform_later - end - - every(1.hour, 'schedule:retry_generating_subscription_invoices', at: '*:30') do - Clock::RetryGeneratingSubscriptionInvoicesJob - .set(sentry: {"slug" => 'lago_retry_invoices', "cron" => '30 */1 * * *'}) - .perform_later - end - - every(1.hour, 'schedule:finalize_invoices', at: '*:20') do - Clock::FinalizeInvoicesJob - .set(sentry: {"slug" => 'lago_finalize_invoices', "cron" => '20 */1 * * *'}) - .perform_later - end - - every(1.hour, 'schedule:mark_invoices_as_payment_overdue', at: '*:25') do - Clock::MarkInvoicesAsPaymentOverdueJob - .set(sentry: {"slug" => 'lago_mark_invoices_as_payment_overdue', "cron" => '25 */1 * * *'}) - .perform_later - end - - every(1.hour, 'schedule:terminate_coupons', at: '*:30') do - Clock::TerminateCouponsJob - .set(sentry: {"slug" => 'lago_terminate_coupons', "cron" => '30 */1 * * *'}) - .perform_later - end - - every(1.hour, 'schedule:bill_ended_trial_subscriptions', at: '*:35') do - Clock::FreeTrialSubscriptionsBillerJob - .set(sentry: {"slug" => 'lago_bill_ended_trial_subscriptions', "cron" => '35 */1 * * *'}) - .perform_later - end - - every(1.hour, 'schedule:terminate_wallets', at: '*:45') do - Clock::TerminateWalletsJob - .set(sentry: {"slug" => 'lago_terminate_wallets', "cron" => '45 */1 * * *'}) - .perform_later - end - - every(1.hour, 'schedule:termination_alert', at: '*:50') do - Clock::SubscriptionsToBeTerminatedJob - .set(sentry: {"slug" => 'lago_termination_alert', "cron" => '50 */1 * * *'}) - .perform_later - end - - every(1.hour, 'schedule:top_up_wallet_interval_credits', at: '*:55') do - Clock::CreateIntervalWalletTransactionsJob - .set(sentry: {"slug" => 'lago_top_up_wallet_interval_credits', "cron" => '55 */1 * * *'}) - .perform_later - end - - every(1.day, 'schedule:clean_webhooks', at: '01:00') do - Clock::WebhooksCleanupJob - .set(sentry: {"slug" => 'lago_clean_webhooks', "cron" => '0 1 * * *'}) - .perform_later - end - - unless ActiveModel::Type::Boolean.new.cast(ENV['LAGO_DISABLE_EVENTS_VALIDATION']) - every(1.hour, 'schedule:post_validate_events', at: '*:05') do - Clock::EventsValidationJob - .set(sentry: {"slug" => 'lago_post_validate_events', "cron" => '5 */1 * * *'}) - .perform_later - rescue => e - Sentry.capture_exception(e) - end - end - - every(1.hour, 'schedule:compute_daily_usage', at: '*:15') do - Clock::ComputeAllDailyUsagesJob - .set(sentry: {"slug" => 'lago_compute_daily_usage', "cron" => '15 */1 * * *'}) - .perform_later - end - - every(1.hour, "schedule:process_dunning_campaigns", at: "*:45") do - Clock::ProcessDunningCampaignsJob - .set(sentry: {"slug" => "lago_process_dunning_campaigns", "cron" => "45 */1 * * *"}) - .perform_later - end -end diff --git a/config/initializers/rsa_keys.rb b/config/initializers/rsa_keys.rb index bcd91b9dba0..d71626ab715 100644 --- a/config/initializers/rsa_keys.rb +++ b/config/initializers/rsa_keys.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # NOTE: Turn off the private key while building assets for mission_control -unless ENV['SECRET_KEY_BASE_DUMMY'] != 1 +if ENV['SECRET_KEY_BASE_DUMMY'] != 1 private_key_string = if Rails.env.development? || Rails.env.test? File.read(Rails.root.join('.rsa_private.pem')) else diff --git a/config/recurring.yml b/config/recurring.yml index 73e8e04f4fe..83d60c93dec 100644 --- a/config/recurring.yml +++ b/config/recurring.yml @@ -3,6 +3,78 @@ default: &default class: Clock::ActivateSubscriptionsJob queue: clock schedule: every 5 minutes + refresh_draft_invoices: + class: Clock::RefreshDraftInvoicesJob + queue: clock + schedule: every 5 minutes + terminate_ended_subscriptions: + class: Clock::TerminateEndedSubscriptionsJob + queue: clock + schedule: "5 */1 * * *" + bill_customers: + class: Clock::SubscriptionsBillerJob + queue: clock + schedule: "10 */1 * * *" + api_keys_track_usage: + class: Clock::ApiKeys::TrackUsageJob + queue: clock + schedule: "15 */1 * * *" + retry_generating_subscription_invoices: + class: Clock::RetryGeneratingSubscriptionInvoicesJob + queue: clock + schedule: "30 */1 * * *" + finalize_invoices: + class: Clock::FinalizeInvoicesJob + queue: clock + schedule: "20 */1 * * *" + mark_invoices_as_payment_overdue: + class: Clock::MarkInvoicesAsPaymentOverdueJob + queue: clock + schedule: "25 */1 * * *" + terminate_coupons: + class: Clock::TerminateCouponsJob + queue: clock + schedule: "30 */1 * * *" + bill_ended_trial_subscriptions: + class: Clock::FreeTrialSubscriptionsBillerJob + queue: clock + schedule: "35 */1 * * *" + terminate_wallets: + class: Clock::TerminateWalletsJob + queue: clock + schedule: "45 */1 * * *" + termination_alert: + class: Clock::SubscriptionsToBeTerminatedJob + queue: clock + schedule: "50 */1 * * *" + top_up_wallet_interval_credits: + class: Clock::CreateIntervalWalletTransactionsJob + queue: clock + schedule: "55 */1 * * *" + clean_webhooks: + class: Clock::WebhooksCleanupJob + queue: clock + schedule: every 1 day at 01:00 + compute_daily_usage: + class: Clock::ComputeAllDailyUsagesJob + queue: clock + schedule: "15 */1 * * *" + process_dunning_campaigns: + class: Clock::ProcessDunningCampaignsJob + queue: clock + schedule: "45 */1 * * *" + refresh_lifetime_usages: + class: Clock::RefreshLifetimeUsagesJob + queue: clock + schedule: <%= "every " + (ENV.fetch('LAGO_LIFETIME_USAGE_REFRESH_INTERVAL_SECONDS', 5.minutes).to_i / 60).to_s + " minutes" %> + post_validate_events: + class: Clock::EventsValidationJob + queue: clock + schedule: "5 */1 * * *" + refresh_wallets_ongoing_balance: + class: Clock::RefreshWalletsOngoingBalanceJob + queue: clock + schedule: every 5 minutes development: <<: *default diff --git a/scripts/start.clock.dev.sh b/scripts/start.clock.dev.sh deleted file mode 100755 index 757131ef65a..00000000000 --- a/scripts/start.clock.dev.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -# TODO(solid_queue): Remove -bundle install -bundle exec clockwork ./clock.rb diff --git a/scripts/start.clock.sh b/scripts/start.clock.sh deleted file mode 100755 index 7efa5d553c9..00000000000 --- a/scripts/start.clock.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -# TODO(solid_queue): Remove -bundle exec clockwork ./clock.rb diff --git a/spec/clockwork_spec.rb b/spec/clockwork_spec.rb deleted file mode 100644 index a5046b06f34..00000000000 --- a/spec/clockwork_spec.rb +++ /dev/null @@ -1,181 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -# TODO(solid_queue): Remove -describe Clockwork do - after { Clockwork::Test.clear! } - - let(:clock_file) { Rails.root.join('clock.rb') } - - describe 'schedule:bill_customers' do - let(:job) { 'schedule:bill_customers' } - let(:start_time) { Time.zone.parse('1 Apr 2022 00:01:00') } - let(:end_time) { Time.zone.parse('1 Apr 2022 01:01:00') } - - it 'enqueue a subscription biller job' do - Clockwork::Test.run( - file: clock_file, - start_time:, - end_time:, - tick_speed: 1.second - ) - - expect(Clockwork::Test).to be_ran_job(job) - expect(Clockwork::Test.times_run(job)).to eq(1) - - Clockwork::Test.block_for(job).call - expect(Clock::SubscriptionsBillerJob).to have_been_enqueued - end - end - - describe 'schedule:activate_subscriptions' do - let(:job) { 'schedule:activate_subscriptions' } - let(:start_time) { Time.zone.parse('1 Apr 2022 00:01:00') } - let(:end_time) { Time.zone.parse('1 Apr 2022 00:31:00') } - - it 'enqueue a activate subscriptions job' do - Clockwork::Test.run( - file: clock_file, - start_time:, - end_time:, - tick_speed: 1.second - ) - - expect(Clockwork::Test).to be_ran_job(job) - expect(Clockwork::Test.times_run(job)).to eq(6) - - Clockwork::Test.block_for(job).call - expect(Clock::ActivateSubscriptionsJob).to have_been_enqueued - end - end - - describe 'schedule:post_validate_events' do - let(:job) { 'schedule:post_validate_events' } - let(:start_time) { Time.zone.parse('1 Apr 2022 01:00:00') } - let(:end_time) { Time.zone.parse('1 Apr 2022 03:00:00') } - - it 'enqueue a activate subscriptions job' do - Clockwork::Test.run( - file: clock_file, - start_time:, - end_time:, - tick_speed: 1.second - ) - - expect(Clockwork::Test).to be_ran_job(job) - expect(Clockwork::Test.times_run(job)).to eq(2) - - Clockwork::Test.block_for(job).call - expect(Clock::EventsValidationJob).to have_been_enqueued - end - end - - describe 'schedule:refresh_lifetime_usages' do - let(:job) { 'schedule:refresh_lifetime_usages' } - let(:start_time) { Time.zone.parse('1 Apr 2022 00:01:00') } - let(:end_time) { Time.zone.parse('1 Apr 2022 00:31:00') } - - it 'enqueue a refresh lifetime usages job' do - Clockwork::Test.run( - file: clock_file, - start_time:, - end_time:, - tick_speed: 1.second - ) - - expect(Clockwork::Test).to be_ran_job(job) - expect(Clockwork::Test.times_run(job)).to eq(6) - - Clockwork::Test.block_for(job).call - expect(Clock::RefreshLifetimeUsagesJob).to have_been_enqueued - end - - context "with a custom refresh interval configured" do - before do - allow(ENV).to receive(:[]).and_call_original - allow(ENV).to receive(:[]).with('LAGO_LIFETIME_USAGE_REFRESH_INTERVAL_SECONDS').and_return('150') - end - - it 'uses the ENV["LAGO_LIFETIME_USAGE_REFRESH_INTERVAL_SECONDS"] to set a custom period' do - Clockwork::Test.run( - file: clock_file, - start_time:, - end_time:, - tick_speed: 1.second - ) - - expect(Clockwork::Test).to be_ran_job(job) - expect(Clockwork::Test.times_run(job)).to eq(12) - - Clockwork::Test.block_for(job).call - expect(Clock::RefreshLifetimeUsagesJob).to have_been_enqueued - - expect(ENV).to have_received(:[]).with('LAGO_LIFETIME_USAGE_REFRESH_INTERVAL_SECONDS') - end - end - end - - describe 'schedule:retry_generating_subscription_invoices' do - let(:job) { 'schedule:retry_generating_subscription_invoices' } - let(:start_time) { Time.zone.parse('1 Apr 2022 00:01:00') } - let(:end_time) { Time.zone.parse('1 Apr 2022 01:01:00') } - - it 'enqueues a Clock::RetryGeneratingSubscriptionInvoiceJob' do - Clockwork::Test.run( - file: clock_file, - start_time:, - end_time:, - tick_speed: 1.second - ) - - expect(Clockwork::Test).to be_ran_job(job) - expect(Clockwork::Test.times_run(job)).to eq(1) - - Clockwork::Test.block_for(job).call - expect(Clock::RetryGeneratingSubscriptionInvoicesJob).to have_been_enqueued - end - end - - describe 'schedule:compute_daily_usage' do - let(:job) { 'schedule:compute_daily_usage' } - let(:start_time) { Time.zone.parse('1 Apr 2022 00:01:00') } - let(:end_time) { Time.zone.parse('1 Apr 2022 01:01:00') } - - it 'enqueue a activate subscriptions job' do - Clockwork::Test.run( - file: clock_file, - start_time:, - end_time:, - tick_speed: 1.second - ) - - expect(Clockwork::Test).to be_ran_job(job) - expect(Clockwork::Test.times_run(job)).to eq(1) - - Clockwork::Test.block_for(job).call - expect(Clock::ComputeAllDailyUsagesJob).to have_been_enqueued - end - end - - describe "schedule:process_dunning_campaigns" do - let(:job) { "schedule:process_dunning_campaigns" } - let(:start_time) { Time.zone.parse("1 Apr 2022 00:01:00") } - let(:end_time) { Time.zone.parse("1 Apr 2022 01:01:00") } - - it "enqueue a process dunning campaigns job" do - Clockwork::Test.run( - file: clock_file, - start_time:, - end_time:, - tick_speed: 1.second - ) - - expect(Clockwork::Test).to be_ran_job(job) - expect(Clockwork::Test.times_run(job)).to eq(1) - - Clockwork::Test.block_for(job).call - expect(Clock::ProcessDunningCampaignsJob).to have_been_enqueued - end - end -end