Merge tag '2023-06-26-01' into feature/import-2023-06-26

# Conflicts:
#	db/schema.rb

app/jobs/cron/expired_dossiers_brouillon_deletion_job.rb

@@ -0,0 +1,7 @@
+class Cron::ExpiredDossiersBrouillonDeletionJob < Cron::CronJob
+  self.schedule_expression = "every day at 10 pm"
+
+  def perform(*args)
+    ExpiredDossiersDeletionService.new.process_expired_dossiers_brouillon
+  end
+end

app/jobs/cron/expired_dossiers_en_construction_deletion_job.rb

@@ -0,0 +1,7 @@
+class Cron::ExpiredDossiersEnConstructionDeletionJob < Cron::CronJob
+  self.schedule_expression = "every day at 3 pm"
+
+  def perform(*args)
+    ExpiredDossiersDeletionService.new.process_expired_dossiers_en_construction
+  end
+end

app/jobs/cron/expired_dossiers_termine_deletion_job.rb

@@ -0,0 +1,7 @@
+class Cron::ExpiredDossiersTermineDeletionJob < Cron::CronJob
+  self.schedule_expression = "every day at 7 am"
+
+  def perform(*args)
+    ExpiredDossiersDeletionService.new.process_expired_dossiers_termine
+  end
+end

app/jobs/cron/weekly_overview_job.rb

@@ -1,16 +1,13 @@
 class Cron::WeeklyOverviewJob < Cron::CronJob
-  self.schedule_expression = "every monday at 7 am"
+  self.schedule_expression = "every monday at 4 am"
 
   def perform
     # Feature flipped to avoid mails in staging due to unprocessed dossier
     return unless Rails.application.config.ds_weekly_overview
 
     Instructeur.find_each do |instructeur|
-      # NOTE: it's not exactly accurate because rate limit is not shared between jobs processes
-      Dolist::API.sleep_until_limit_reset if Dolist::API.near_rate_limit?
-
       # mailer won't send anything if overview if empty
-      InstructeurMailer.last_week_overview(instructeur)&.deliver_later
+      InstructeurMailer.last_week_overview(instructeur)&.deliver_later(wait: rand(0..3.hours))
     end
   end
 end

app/mailers/application_mailer.rb

@@ -7,6 +7,8 @@ class ApplicationMailer < ActionMailer::Base
   default from: "#{APPLICATION_NAME} <#{CONTACT_EMAIL}>"
   layout 'mailer'
 
+  before_action -> { Sentry.set_tags(mailer: mailer_name, action: action_name) }
+
   # Attach the procedure logo to the email (if any).
   # Returns the attachment url.
   def attach_logo(procedure)

app/services/expired_dossiers_deletion_service.rb

@@ -1,20 +1,28 @@
 class ExpiredDossiersDeletionService
-  def self.process_expired_dossiers_brouillon
+  def initialize(rate_limiter: MailRateLimiter.new(limit: 200, window: 10.minutes))
+    @rate_limiter = rate_limiter
+  end
+
+  def process_expired_dossiers_brouillon
     send_brouillon_expiration_notices
     delete_expired_brouillons_and_notify
   end
 
-  def self.process_expired_dossiers_en_construction
+  def process_expired_dossiers_en_construction
     send_en_construction_expiration_notices
     delete_expired_en_construction_and_notify
   end
 
-  def self.process_expired_dossiers_termine
+  def process_expired_dossiers_termine
     send_termine_expiration_notices
     delete_expired_termine_and_notify
   end
 
-  def self.send_brouillon_expiration_notices
+  def safe_send_email(mail)
+    @rate_limiter.send_with_delay(mail)
+  end
+
+  def send_brouillon_expiration_notices
     dossiers_close_to_expiration = Dossier
       .brouillon_close_to_expiration
       .without_brouillon_expiration_notice_sent
@@ -24,66 +32,70 @@ def self.send_brouillon_expiration_notices
     dossiers_close_to_expiration.in_batches.update_all(brouillon_close_to_expiration_notice_sent_at: Time.zone.now)
 
     user_notifications.each do |(email, dossiers)|
-      DossierMailer.notify_brouillon_near_deletion(
+      mail = DossierMailer.notify_brouillon_near_deletion(
         dossiers,
         email
-      ).deliver_later
+      )
+      safe_send_email(mail)
     end
   end
 
-  def self.send_en_construction_expiration_notices
+  def send_en_construction_expiration_notices
     send_expiration_notices(
       Dossier.en_construction_close_to_expiration.without_en_construction_expiration_notice_sent,
       :en_construction_close_to_expiration_notice_sent_at
     )
   end
 
-  def self.send_termine_expiration_notices
+  def send_termine_expiration_notices
     send_expiration_notices(
       Dossier.termine_close_to_expiration.without_termine_expiration_notice_sent,
       :termine_close_to_expiration_notice_sent_at
     )
   end
 
-  def self.delete_expired_brouillons_and_notify
+  def delete_expired_brouillons_and_notify
     user_notifications = group_by_user_email(Dossier.brouillon_expired)
       .map { |(email, dossiers)| [email, dossiers.map(&:hash_for_deletion_mail)] }
 
     Dossier.brouillon_expired.in_batches.destroy_all
 
     user_notifications.each do |(email, dossiers_hash)|
-      DossierMailer.notify_brouillon_deletion(
+      mail = DossierMailer.notify_brouillon_deletion(
         dossiers_hash,
         email
-      ).deliver_later
+      )
+      safe_send_email(mail)
     end
   end
 
-  def self.delete_expired_en_construction_and_notify
+  def delete_expired_en_construction_and_notify
     delete_expired_and_notify(Dossier.en_construction_expired)
   end
 
-  def self.delete_expired_termine_and_notify
+  def delete_expired_termine_and_notify
     delete_expired_and_notify(Dossier.termine_expired, notify_on_closed_procedures_to_user: true)
   end
 
   private
 
-  def self.send_expiration_notices(dossiers_close_to_expiration, close_to_expiration_flag)
+  def send_expiration_notices(dossiers_close_to_expiration, close_to_expiration_flag)
     user_notifications = group_by_user_email(dossiers_close_to_expiration)
     administration_notifications = group_by_fonctionnaire_email(dossiers_close_to_expiration)
 
     dossiers_close_to_expiration.in_batches.update_all(close_to_expiration_flag => Time.zone.now)
 
     user_notifications.each do |(email, dossiers)|
-      DossierMailer.notify_near_deletion_to_user(dossiers, email).deliver_later
+      mail = DossierMailer.notify_near_deletion_to_user(dossiers, email)
+      safe_send_email(mail)
     end
     administration_notifications.each do |(email, dossiers)|
-      DossierMailer.notify_near_deletion_to_administration(dossiers, email).deliver_later
+      mail = DossierMailer.notify_near_deletion_to_administration(dossiers, email)
+      safe_send_email(mail)
     end
   end
 
-  def self.delete_expired_and_notify(dossiers_to_remove, notify_on_closed_procedures_to_user: false)
+  def delete_expired_and_notify(dossiers_to_remove, notify_on_closed_procedures_to_user: false)
     user_notifications = group_by_user_email(dossiers_to_remove, notify_on_closed_procedures_to_user: notify_on_closed_procedures_to_user)
       .map { |(email, dossiers)| [email, dossiers.map(&:id)] }
     administration_notifications = group_by_fonctionnaire_email(dossiers_to_remove)
@@ -98,24 +110,26 @@ def self.delete_expired_and_notify(dossiers_to_remove, notify_on_closed_procedur
     user_notifications.each do |(email, dossier_ids)|
       dossier_ids = dossier_ids.intersection(deleted_dossier_ids)
       if dossier_ids.present?
-        DossierMailer.notify_automatic_deletion_to_user(
+        mail = DossierMailer.notify_automatic_deletion_to_user(
           DeletedDossier.where(dossier_id: dossier_ids).to_a,
           email
-        ).deliver_later
+        )
+        safe_send_email(mail)
       end
     end
     administration_notifications.each do |(email, dossier_ids)|
       dossier_ids = dossier_ids.intersection(deleted_dossier_ids)
       if dossier_ids.present?
-        DossierMailer.notify_automatic_deletion_to_administration(
+        mail = DossierMailer.notify_automatic_deletion_to_administration(
           DeletedDossier.where(dossier_id: dossier_ids).to_a,
           email
-        ).deliver_later
+        )
+        safe_send_email(mail)
       end
     end
   end
 
-  def self.group_by_user_email(dossiers, notify_on_closed_procedures_to_user: false)
+  def group_by_user_email(dossiers, notify_on_closed_procedures_to_user: false)
     dossiers
       .visible_by_user
       .with_notifiable_procedure(notify_on_closed: notify_on_closed_procedures_to_user)
@@ -124,7 +138,7 @@ def self.group_by_user_email(dossiers, notify_on_closed_procedures_to_user: fals
       .map { |(user, dossiers)| [user.email, dossiers] }
   end
 
-  def self.group_by_fonctionnaire_email(dossiers)
+  def group_by_fonctionnaire_email(dossiers)
     dossiers
       .visible_by_administration
       .with_notifiable_procedure(notify_on_closed: true)

app/services/mail_rate_limiter.rb

@@ -0,0 +1,32 @@
+class MailRateLimiter
+  attr_reader :delay, :current_window
+
+  def send_with_delay(mail)
+    if current_window_full?
+      @delay += @window
+    end
+    if current_window_full? || current_window_expired?
+      @current_window = { started_at: Time.current, sent: 0 }
+    end
+    @current_window[:sent] += 1
+
+    mail.deliver_later(wait: delay)
+  end
+
+  private
+
+  def initialize(limit:, window:)
+    @limit = limit
+    @window = window
+    @current_window = { started_at: Time.current, sent: 0 }
+    @delay = 0
+  end
+
+  def current_window_expired?
+    (@current_window[:started_at] + @window).past?
+  end
+
+  def current_window_full?
+    @current_window[:sent] >= @limit
+  end
+end

db/migrate/20230623160831_add_index_to_parent_dossier_id.rb

@@ -0,0 +1,7 @@
+class AddIndexToParentDossierId < ActiveRecord::Migration[7.0]
+  disable_ddl_transaction!
+
+  def change
+    add_index :dossiers, :parent_dossier_id, algorithm: :concurrently
+  end
+end

db/schema.rb

@@ -418,6 +418,7 @@
     t.index ["editing_fork_origin_id"], name: "index_dossiers_on_editing_fork_origin_id"
     t.index ["groupe_instructeur_id"], name: "index_dossiers_on_groupe_instructeur_id"
     t.index ["hidden_at"], name: "index_dossiers_on_hidden_at"
+    t.index ["parent_dossier_id"], name: "index_dossiers_on_parent_dossier_id"
     t.index ["prefill_token"], name: "index_dossiers_on_prefill_token", unique: true
     t.index ["revision_id"], name: "index_dossiers_on_revision_id"
     t.index ["state"], name: "index_dossiers_on_state"

spec/lib/mail_rate_limiter_spec.rb

@@ -0,0 +1,26 @@
+describe MailRateLimiter do
+  describe 'hits limits' do
+    let(:limit) { 10 }
+    let(:window) { 2.seconds }
+    let(:rate_limiter) { MailRateLimiter.new(limit:, window:) }
+    let(:mail) { DossierMailer.notify_automatic_deletion_to_user([], '[email protected]') }
+
+    it 'decreases current_window[:limit]' do
+      expect { rate_limiter.send_with_delay(mail) }.to change { rate_limiter.current_window[:sent] }.by(1)
+    end
+
+    it 'increases the delay by window when it reaches the max number of call' do
+      expect do
+        (limit + 1).times { rate_limiter.send_with_delay(mail) }
+      end.to change { rate_limiter.delay }.by(window)
+    end
+
+    it 'renews current_window when it expires' do
+      rate_limiter.send_with_delay(mail)
+      travel_to(Time.current + window + 1.second) do
+        rate_limiter.send_with_delay(mail)
+        expect(rate_limiter.current_window[:sent]).to eq(1)
+      end
+    end
+  end
+end