Merge pull request #1871 from internetee/645-2-gracefully-drop-que-em…

…ail-jobs-sidekiq

Change Que for Sidekiq as job backend

Gemfile

@@ -67,6 +67,7 @@ gem 'epp-xml', '1.1.0', github: 'internetee/epp-xml'
 gem 'que'
 gem 'daemons-rails', '1.2.1'
 gem 'que-web'
+gem 'sidekiq'
 gem 'pdfkit'
 gem 'jquery-ui-rails', '5.0.5'
 gem 'airbrake'

Gemfile.lock

@@ -190,6 +190,7 @@ GEM
       execjs
     coffee-script-source (1.12.2)
     concurrent-ruby (1.1.8)
+    connection_pool (2.2.3)
     countries (3.1.0)
       i18n_data (~> 0.11.0)
       sixarm_ruby_unaccent (~> 1.1)
@@ -384,6 +385,7 @@ GEM
       activesupport (>= 5.2.4)
       i18n
     rbtree3 (0.6.0)
+    redis (4.2.5)
     regexp_parser (2.1.1)
     request_store (1.5.0)
       rack (>= 1.4)
@@ -421,6 +423,10 @@ GEM
     selenium-webdriver (3.142.7)
       childprocess (>= 0.5, < 4.0)
       rubyzip (>= 1.2.2)
+    sidekiq (6.1.3)
+      connection_pool (>= 2.2.2)
+      rack (~> 2.0)
+      redis (>= 4.2.0)
     simplecov (0.17.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
@@ -554,6 +560,7 @@ DEPENDENCIES
   sass-rails
   select2-rails (= 4.0.13)
   selectize-rails (= 0.12.1)
+  sidekiq
   simplecov (= 0.17.1)
   simpleidn (= 0.1.1)
   truemail (~> 2.2)
@@ -565,4 +572,4 @@ DEPENDENCIES
   wkhtmltopdf-binary (~> 0.12.5.1)
 
 BUNDLED WITH
-   2.2.2
+   2.2.15

app/interactions/domains/expire_period/process_expired.rb

@@ -14,7 +14,7 @@ def execute
         return unless saved
 
         recipients.each do |recipient|
-          DomainExpireEmailJob.enqueue(domain.id, recipient, run_at: send_time)
+          DomainExpireEmailJob.set(wait_until: send_time).perform_later(domain.id, recipient)
         end
       end
 

app/interactions/domains/expired_pendings/process_clean.rb

@@ -9,7 +9,7 @@ def execute
         clean_pendings
 
         to_stdout("DomainCron.clean_expired_pendings: ##{domain.id} (#{domain.name})")
-        UpdateWhoisRecordJob.enqueue domain.name, 'domain'
+        UpdateWhoisRecordJob.perform_later domain.name, 'domain'
       end
 
       private

app/interactions/domains/update_confirm/process_update_confirmed.rb

@@ -17,7 +17,7 @@ def apply_pending_update!
         update_domain
         clean_pendings!
 
-        WhoisRecord.find_by(domain_id: domain.id).save # need to reload model
+        WhoisRecord.find_by(domain_id: domain.id)&.save # need to reload model
       end
 
       def update_domain

app/jobs/application_job.rb

@@ -1,2 +1,4 @@
 class ApplicationJob < ActiveJob::Base
+  discard_on NoMethodError
+  queue_as :default
 end

app/jobs/csync_job.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-class CsyncJob < Que::Job
-  def run(generate: false)
+class CsyncJob < ApplicationJob
+  def perform(generate: false)
     @store = {}
     @input_store = { secure: {}, insecure: {} }
     @results = {}

app/jobs/directo_invoice_forward_job.rb

@@ -1,5 +1,5 @@
-class DirectoInvoiceForwardJob < Que::Job
-  def run(monthly: false, dry: false)
+class DirectoInvoiceForwardJob < ApplicationJob
+  def perform(monthly: false, dry: false)
     @dry = dry
     (@month = Time.zone.now - 1.month) if monthly
 

app/jobs/dispute_status_update_job.rb

@@ -1,5 +1,5 @@
-class DisputeStatusUpdateJob < Que::Job
-  def run(logger: Logger.new(STDOUT))
+class DisputeStatusUpdateJob < ApplicationJob
+  def perform(logger: Logger.new(STDOUT))
     @logger = logger
 
     @backlog = { 'activated': 0, 'closed': 0, 'activate_fail': [], 'close_fail': [] }

app/jobs/domain_delete_confirm_job.rb

@@ -1,11 +1,39 @@
 class DomainDeleteConfirmJob < ApplicationJob
-  queue_as :default
-
   def perform(domain_id, action, initiator = nil)
     domain = Epp::Domain.find(domain_id)
 
     Domains::DeleteConfirm::ProcessAction.run(domain: domain,
                                               action: action,
                                               initiator: initiator)
   end
+
+  private
+
+  def action_confirmed(domain)
+    domain.notify_registrar(:poll_pending_delete_confirmed_by_registrant)
+    domain.apply_pending_delete!
+    raise_errors!(domain)
+  end
+
+  def action_rejected(domain)
+    domain.statuses.delete(DomainStatus::PENDING_DELETE_CONFIRMATION)
+    domain.notify_registrar(:poll_pending_delete_rejected_by_registrant)
+
+    domain.cancel_pending_delete
+    domain.save(validate: false)
+    raise_errors!(domain)
+    notify_on_domain(domain)
+  end
+
+  def notify_on_domain(domain)
+    if domain.registrant_verification_token.blank?
+      Rails.logger.warn 'EMAIL NOT DELIVERED: registrant_verification_token is missing for '\
+                        "#{domain.name}"
+    elsif domain.registrant_verification_asked_at.blank?
+      Rails.logger.warn 'EMAIL NOT DELIVERED: registrant_verification_asked_at is missing for '\
+                        "#{domain.name}"
+    else
+      DomainDeleteMailer.rejected(domain).deliver_now
+    end
+  end
 end

app/jobs/domain_delete_job.rb

@@ -1,6 +1,5 @@
-class DomainDeleteJob < Que::Job
-
-  def run(domain_id)
+class DomainDeleteJob < ApplicationJob
+  def perform(domain_id)
     domain = Domain.find(domain_id)
 
     Domains::Delete::DoDelete.run(domain: domain)

app/jobs/domain_expire_email_job.rb

@@ -1,5 +1,5 @@
-class DomainExpireEmailJob < Que::Job
-  def run(domain_id, email)
+class DomainExpireEmailJob < ApplicationJob
+  def perform(domain_id, email)
     domain = Domain.find(domain_id)
 
     return if domain.registered?

app/jobs/domain_update_confirm_job.rb

@@ -1,6 +1,4 @@
 class DomainUpdateConfirmJob < ApplicationJob
-  queue_as :default
-
   def perform(domain_id, action, initiator = nil)
     domain = Epp::Domain.find(domain_id)
     Domains::UpdateConfirm::ProcessAction.run(domain: domain,

app/jobs/regenerate_registrar_whoises_job.rb

@@ -1,10 +1,12 @@
-class RegenerateRegistrarWhoisesJob < Que::Job
-  def run(registrar_id)
+class RegenerateRegistrarWhoisesJob < ApplicationJob
+  retry_on StandardError, wait: 2.seconds, attempts: 3
+
+  def perform(registrar_id)
     # no return as we want restart job if fails
     registrar = Registrar.find(registrar_id)
 
     registrar.whois_records.select(:name).find_in_batches(batch_size: 20) do |group|
-      UpdateWhoisRecordJob.enqueue group.map(&:name), 'domain'
+      UpdateWhoisRecordJob.perform_later group.map(&:name), 'domain'
     end
   end
-end
+end

app/jobs/regenerate_subzone_whoises_job.rb

@@ -1,11 +1,11 @@
-class RegenerateSubzoneWhoisesJob < Que::Job
-  def run
+class RegenerateSubzoneWhoisesJob < ApplicationJob
+  def perform
     subzones = DNS::Zone.all
 
     subzones.each do |zone|
       next unless zone.subzone?
 
-      UpdateWhoisRecordJob.enqueue zone.origin, 'zone'
+      UpdateWhoisRecordJob.perform_later zone.origin, 'zone'
     end
   end
 end

app/jobs/registrant_change_confirm_email_job.rb

@@ -1,5 +1,5 @@
-class RegistrantChangeConfirmEmailJob < Que::Job
-  def run(domain_id, new_registrant_id)
+class RegistrantChangeConfirmEmailJob < ApplicationJob
+  def perform(domain_id, new_registrant_id)
     domain = Domain.find(domain_id)
     new_registrant = Registrant.find(new_registrant_id)
 

app/jobs/registrant_change_expired_email_job.rb

@@ -0,0 +1,23 @@
+class RegistrantChangeExpiredEmailJob < ApplicationJob
+  def perform(domain_id)
+    domain = Domain.find(domain_id)
+    log(domain)
+    RegistrantChangeMailer.expired(domain: domain,
+                                   registrar: domain.registrar,
+                                   registrant: domain.registrant,
+                                   send_to: [domain.new_registrant_email,
+                                             domain.registrant.email]).deliver_now
+  end
+
+  private
+
+  def log(domain)
+    message = 'Send RegistrantChangeMailer#expired email for domain '\
+               "#{domain.name} (##{domain.id}) to #{domain.new_registrant_email}"
+    logger.info(message)
+  end
+
+  def logger
+    Rails.logger
+  end
+end

app/jobs/registrant_change_notice_email_job.rb

@@ -1,5 +1,5 @@
-class RegistrantChangeNoticeEmailJob < Que::Job
-  def run(domain_id, new_registrant_id)
+class RegistrantChangeNoticeEmailJob < ApplicationJob
+  def perform(domain_id, new_registrant_id)
     domain = Domain.find(domain_id)
     new_registrant = Registrant.find(new_registrant_id)
     log(domain, new_registrant)

app/jobs/send_e_invoice_job.rb

@@ -1,24 +1,29 @@
-class SendEInvoiceJob < Que::Job
-  def run(invoice_id, payable = true)
-    invoice = run_condition(Invoice.find_by(id: invoice_id), payable: payable)
+class SendEInvoiceJob < ApplicationJob
+  discard_on HTTPClient::TimeoutError
 
-    invoice.to_e_invoice(payable: payable).deliver
-    ActiveRecord::Base.transaction do
-      invoice.update(e_invoice_sent_at: Time.zone.now)
-      log_success(invoice)
-      destroy
-    end
+  def perform(invoice_id, payable = true)
+    invoice = Invoice.find_by(id: invoice_id)
+    return unless need_to_process_invoice?(invoice: invoice, payable: payable)
+
+    process(invoice: invoice, payable: payable)
   rescue StandardError => e
     log_error(invoice: invoice, error: e)
     raise e
   end
 
   private
 
-  def run_condition(invoice, payable: true)
-    destroy unless invoice
-    destroy if invoice.do_not_send_e_invoice? && payable
-    invoice
+  def need_to_process_invoice?(invoice:, payable:)
+    return false if invoice.blank?
+    return false if invoice.do_not_send_e_invoice? && payable
+
+    true
+  end
+
+  def process(invoice:, payable:)
+    invoice.to_e_invoice(payable: payable).deliver
+    invoice.update(e_invoice_sent_at: Time.zone.now)
+    log_success(invoice)
   end
 
   def log_success(invoice)

app/jobs/update_whois_record_job.rb

@@ -1,5 +1,5 @@
-class UpdateWhoisRecordJob < Que::Job
-  def run(names, type)
+class UpdateWhoisRecordJob < ApplicationJob
+  def perform(names, type)
     Whois::Update.run(names: [names].flatten, type: type)
   end
 end

app/jobs/verify_emails_job.rb

@@ -1,26 +1,28 @@
-class VerifyEmailsJob < Que::Job
-  def run(verification_id)
-    email_address_verification = run_condition(EmailAddressVerification.find(verification_id))
+class VerifyEmailsJob < ApplicationJob
+  discard_on StandardError
 
-    return if email_address_verification.recently_verified?
+  def perform(verification_id)
+    email_address_verification = EmailAddressVerification.find(verification_id)
+    return unless need_to_verify?(email_address_verification)
 
-    ActiveRecord::Base.transaction do
-      email_address_verification.verify
-      log_success(email_address_verification)
-      destroy
-    end
+    process(email_address_verification)
   rescue StandardError => e
     log_error(verification: email_address_verification, error: e)
     raise e
   end
 
   private
 
-  def run_condition(email_address_verification)
-    destroy unless email_address_verification
-    destroy if email_address_verification.recently_verified?
+  def need_to_verify?(email_address_verification)
+    return false if email_address_verification.blank?
+    return false if email_address_verification.recently_verified?
 
-    email_address_verification
+    true
+  end
+
+  def process(email_address_verification)
+    email_address_verification.verify
+    log_success(email_address_verification)
   end
 
   def logger

app/models/blocked_domain.rb

@@ -34,6 +34,6 @@ def generate_json
   end
 
   def remove_data
-    UpdateWhoisRecordJob.enqueue name, 'blocked'
+    UpdateWhoisRecordJob.perform_later name, 'blocked'
   end
 end

app/models/concerns/domain/deletable.rb

@@ -7,17 +7,24 @@ module Domain::Deletable
     DomainStatus::FORCE_DELETE,
   ].freeze
 
+  def deletion_time
+    @deletion_time ||= Time.zone.at(rand(deletion_time_span))
+  end
+
   private
 
   def delete_later
-    deletion_time = Time.zone.at(rand(deletion_time_span))
-    DomainDeleteJob.enqueue(id, run_at: deletion_time, priority: 1)
+    DomainDeleteJob.set(wait_until: deletion_time).perform_later(id)
     logger.info "Domain #{name} is scheduled to be deleted around #{deletion_time}"
   end
 
   def do_not_delete_later
-    # Que job can be manually deleted in admin area UI
-    QueJob.find_by("args->>0 = '#{id}'", job_class: DomainDeleteJob.name)&.destroy
+    return if Rails.env.test?
+
+    jobs = Sidekiq::ScheduledSet.new.select do |job|
+      job.args.first['job_class'] == 'DomainDeleteJob' && job.args.first['arguments'] == [id]
+    end
+    jobs.each(&:delete)
   end
 
   def deletion_time_span

app/models/concerns/invoice/payable.rb

@@ -11,7 +11,7 @@ def payable?
   end
 
   def paid?
-    account_activity
+    account_activity.present?
   end
 
   def receipt_date

app/models/concerns/zone/whois_queryable.rb

@@ -11,7 +11,7 @@ def subzone?
   end
 
   def update_whois_record
-    UpdateWhoisRecordJob.enqueue origin, 'zone'
+    UpdateWhoisRecordJob.perform_later origin, 'zone'
   end
 
   def generate_data