add config env vars to other pruning workers

- this also improves performance by limiting the date range of the pruning op
keygen-sh · Oct 19, 2023 · 36cd61d · 36cd61d
1 parent 2d69b1b
commit 36cd61d
Show file tree

Hide file tree

Showing 5 changed files with 100 additions and 18 deletions.
diff --git a/app/workers/prune_metrics_worker.rb b/app/workers/prune_metrics_worker.rb
@@ -1,5 +1,6 @@
 class PruneMetricsWorker < BaseWorker
   BACKLOG_DAYS = ENV.fetch('KEYGEN_PRUNE_METRIC_BACKLOG_DAYS') { 30 }.to_i
+  TARGET_DAYS  = ENV.fetch('KEYGEN_PRUNE_METRIC_TARGET_DAYS')  { 1 }.to_i
   BATCH_SIZE   = ENV.fetch('KEYGEN_PRUNE_BATCH_SIZE')          { 1_000 }.to_i
   BATCH_WAIT   = ENV.fetch('KEYGEN_PRUNE_BATCH_WAIT')          { 1 }.to_f
 
@@ -11,15 +12,19 @@ def perform
     return if
       BACKLOG_DAYS <= 0
 
-    accounts = Account.where(<<~SQL.squish, BACKLOG_DAYS.days.ago.beginning_of_day)
+    end_date   = BACKLOG_DAYS.days.ago.beginning_of_day
+    start_date = (end_date - TARGET_DAYS.day).beginning_of_day
+
+    accounts = Account.where(<<~SQL.squish, start_date:, end_date:)
       EXISTS (
         SELECT
           1
         FROM
           "metrics"
         WHERE
-          "metrics"."account_id" = "accounts"."id" AND
-          "metrics"."created_at" < ?
+          "metrics"."account_id"  = "accounts"."id" AND
+          "metrics"."created_at" >= :start_date     AND
+          "metrics"."created_at" <  :end_date
         LIMIT
           1
       )
@@ -34,8 +39,8 @@ def perform
       Keygen.logger.info "[workers.prune-metrics] Pruning rows: account_id=#{account_id}"
 
       loop do
-        metrics = account.metrics
-                         .where('created_at < ?', BACKLOG_DAYS.days.ago.beginning_of_day)
+        metrics = account.metrics.where('created_at >= ?', start_date)
+                                 .where('created_at < ?', end_date)
 
         batch += 1
         count = metrics.limit(BATCH_SIZE)

diff --git a/app/workers/prune_request_logs_worker.rb b/app/workers/prune_request_logs_worker.rb
@@ -1,5 +1,6 @@
 class PruneRequestLogsWorker < BaseWorker
   BACKLOG_DAYS = ENV.fetch('KEYGEN_PRUNE_REQUEST_BACKLOG_DAYS') { 30 }.to_i
+  TARGET_DAYS  = ENV.fetch('KEYGEN_PRUNE_REQUEST_TARGET_DAYS')  { 1 }.to_i
   BATCH_SIZE   = ENV.fetch('KEYGEN_PRUNE_BATCH_SIZE')           { 1_000 }.to_i
   BATCH_WAIT   = ENV.fetch('KEYGEN_PRUNE_BATCH_WAIT')           { 1 }.to_f
 
@@ -11,15 +12,19 @@ def perform
     return if
       BACKLOG_DAYS <= 0
 
-    accounts = Account.where(<<~SQL.squish, BACKLOG_DAYS.days.ago.beginning_of_day)
+    end_date   = BACKLOG_DAYS.days.ago.beginning_of_day
+    start_date = (end_date - TARGET_DAYS.day).beginning_of_day
+
+    accounts = Account.where(<<~SQL.squish, start_date:, end_date:)
       EXISTS (
         SELECT
           1
         FROM
           "request_logs"
         WHERE
-          "request_logs"."account_id" = "accounts"."id" AND
-          "request_logs"."created_at" < ?
+          "request_logs"."account_id"  = "accounts"."id" AND
+          "request_logs"."created_at" >= :start_date     AND
+          "request_logs"."created_at" <  :end_date
         LIMIT
           1
       )
@@ -34,8 +39,8 @@ def perform
       Keygen.logger.info "[workers.prune-request-logs] Pruning rows: account_id=#{account_id}"
 
       loop do
-        logs = account.request_logs
-                      .where('created_at < ?', BACKLOG_DAYS.days.ago.beginning_of_day)
+        logs = account.request_logs.where('created_at >= ?', start_date)
+                                   .where('created_at < ?', end_date)
 
         batch += 1
         count = logs.limit(BATCH_SIZE)

diff --git a/app/workers/prune_webhook_events_worker.rb b/app/workers/prune_webhook_events_worker.rb
@@ -1,16 +1,34 @@
 class PruneWebhookEventsWorker < BaseWorker
-  BATCH_SIZE     = ENV.fetch('PRUNE_BATCH_SIZE')     { 1_000 }.to_i
-  SLEEP_DURATION = ENV.fetch('PRUNE_SLEEP_DURATION') { 1 }.to_f
+  BACKLOG_DAYS   = ENV.fetch('KEYGEN_PRUNE_WEBHOOK_BACKLOG_DAYS') { 30 }.to_i
+  TARGET_DAYS    = ENV.fetch('KEYGEN_PRUNE_WEBHOOK_TARGET_DAYS')  { 1 }.to_i
+  BATCH_SIZE     = ENV.fetch('PRUNE_BATCH_SIZE')                  { 1_000 }.to_i
+  SLEEP_DURATION = ENV.fetch('PRUNE_SLEEP_DURATION')              { 1 }.to_f
 
   sidekiq_options queue: :cron,
                   lock: :until_executed,
                   cronitor_disabled: false
 
   def perform
-    accounts = Account.joins(:webhook_events)
-                      .where('webhook_events.created_at < ?', 30.days.ago)
-                      .group('accounts.id')
-                      .having('count(webhook_events.id) > 0')
+    return if
+      BACKLOG_DAYS <= 0
+
+    end_date   = BACKLOG_DAYS.days.ago.beginning_of_day
+    start_date = (end_date - TARGET_DAYS.day).beginning_of_day
+
+    accounts = Account.where(<<~SQL.squish, start_date:, end_date:)
+      EXISTS (
+        SELECT
+          1
+        FROM
+          "webhook_events"
+        WHERE
+          "webhook_events"."account_id"  = "accounts"."id" AND
+          "webhook_events"."created_at" >= :start_date     AND
+          "webhook_events"."created_at" <  :end_date
+        LIMIT
+          1
+      )
+    SQL
 
     Keygen.logger.info "[workers.prune-webhook-events] Starting: accounts=#{accounts.count}"
 
@@ -21,8 +39,8 @@ def perform
       Keygen.logger.info "[workers.prune-webhook-events] Pruning rows: account_id=#{account_id}"
 
       loop do
-        events = account.webhook_events
-                        .where('created_at < ?', 30.days.ago.beginning_of_day)
+        events = account.webhook_events.where('created_at >= ?', start_date)
+                                       .where('created_at < ?', end_date)
 
         batch += 1
         count = events.limit(BATCH_SIZE)

diff --git a/spec/workers/prune_request_logs_worker_spec.rb b/spec/workers/prune_request_logs_worker_spec.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+require 'spec_helper'
+
+describe PruneRequestLogsWorker do
+  let(:worker)  { PruneRequestLogsWorker }
+  let(:account) { create(:account) }
+
+  # See: https://github.com/mhenrixon/sidekiq-unique-jobs#testing
+  before do
+    Sidekiq::Testing.inline!
+  end
+
+  after do
+    Sidekiq::Worker.clear_all
+  end
+
+  it 'should prune backlog of request logs' do
+    create_list(:request_log, 50, account:, created_at: (worker::BACKLOG_DAYS + 1).days.ago)
+    create_list(:request_log, 50, account:)
+
+    expect { worker.perform_async }.to(
+      change { account.request_logs.count }.from(100).to(50),
+    )
+  end
+end
diff --git a/spec/workers/prune_webhook_events_worker_spec.rb b/spec/workers/prune_webhook_events_worker_spec.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+require 'spec_helper'
+
+describe PruneWebhookEventsWorker do
+  let(:worker)  { PruneWebhookEventsWorker }
+  let(:account) { create(:account) }
+
+  # See: https://github.com/mhenrixon/sidekiq-unique-jobs#testing
+  before do
+    Sidekiq::Testing.inline!
+  end
+
+  after do
+    Sidekiq::Worker.clear_all
+  end
+
+  it 'should prune backlog of webhook events' do
+    create_list(:webhook_event, 50, account:, created_at: (worker::BACKLOG_DAYS + 1).days.ago)
+    create_list(:webhook_event, 50, account:)
+
+    expect { worker.perform_async }.to(
+      change { account.webhook_events.count }.from(100).to(50),
+    )
+  end
+end