diff --git a/Gemfile b/Gemfile index 56e595cb..53aae9ab 100644 --- a/Gemfile +++ b/Gemfile @@ -14,6 +14,7 @@ gem 'rails-patterns' gem 'friendly_id', '~> 5.5.0' gem 'nilify_blanks', '~> 1.4' gem 'solid_cable' +gem 'solid_queue' # frontend gem 'haml-rails', '~> 2.0' diff --git a/Gemfile.lock b/Gemfile.lock index 2dab5e34..4223fada 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -139,6 +139,8 @@ GEM email_validator (2.2.4) activemodel erubi (1.13.0) + et-orbi (1.2.11) + tzinfo factory_bot (6.5.0) activesupport (>= 5.0.0) factory_bot_rails (6.4.4) @@ -162,6 +164,9 @@ GEM flamegraph (0.9.5) friendly_id (5.5.1) activerecord (>= 4.0.0) + fugit (1.11.1) + et-orbi (~> 1, >= 1.2.11) + raabro (~> 1.4) globalid (1.2.1) activesupport (>= 6.1) haml (6.3.0) @@ -304,6 +309,7 @@ GEM public_suffix (6.0.1) puma (6.4.3) nio4r (~> 2.0) + raabro (1.4.0) racc (1.8.1) rack (3.1.8) rack-mini-profiler (3.3.1) @@ -463,6 +469,13 @@ GEM activejob (>= 7.2) activerecord (>= 7.2) railties (>= 7.2) + solid_queue (1.0.1) + activejob (>= 7.1) + activerecord (>= 7.1) + concurrent-ruby (>= 1.3.1) + fugit (~> 1.11.0) + railties (>= 7.1) + thor (~> 1.3.1) sqlite3 (2.2.0-x86_64-linux-gnu) sqlite3 (2.2.0-x86_64-linux-musl) stackprof (0.2.26) @@ -579,6 +592,7 @@ DEPENDENCIES silencer simple_form (~> 5.3) solid_cable + solid_queue sqlite3 stackprof stringex (~> 2.8) diff --git a/bin/jobs b/bin/jobs new file mode 100755 index 00000000..dcf59f30 --- /dev/null +++ b/bin/jobs @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby + +require_relative "../config/environment" +require "solid_queue/cli" + +SolidQueue::Cli.start(ARGV) diff --git a/config/environments/production.rb b/config/environments/production.rb index 4020cee1..e3488312 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -60,7 +60,9 @@ } # Replace the default in-process and non-durable queuing backend for Active Job. - # config.active_job.queue_adapter = :resque + config.active_job.queue_adapter = :solid_queue + config.solid_queue.connects_to = { database: { writing: :queue } } + # Ignore bad email addresses and do not raise email delivery errors. # Set this to true and configure the email server for immediate delivery to raise delivery errors. diff --git a/config/queue.yml b/config/queue.yml new file mode 100644 index 00000000..9eace59c --- /dev/null +++ b/config/queue.yml @@ -0,0 +1,18 @@ +default: &default + dispatchers: + - polling_interval: 1 + batch_size: 500 + workers: + - queues: "*" + threads: 3 + processes: <%= ENV.fetch("JOB_CONCURRENCY", 1) %> + polling_interval: 0.1 + +development: + <<: *default + +test: + <<: *default + +production: + <<: *default diff --git a/config/recurring.yml b/config/recurring.yml new file mode 100644 index 00000000..d045b191 --- /dev/null +++ b/config/recurring.yml @@ -0,0 +1,10 @@ +# production: +# periodic_cleanup: +# class: CleanSoftDeletedRecordsJob +# queue: background +# args: [ 1000, { batch_size: 500 } ] +# schedule: every hour +# periodic_command: +# command: "SoftDeletedRecord.due.delete_all" +# priority: 2 +# schedule: at 5am every day diff --git a/db/queue_schema.rb b/db/queue_schema.rb new file mode 100644 index 00000000..ef165500 --- /dev/null +++ b/db/queue_schema.rb @@ -0,0 +1,143 @@ +# frozen_string_literal: true + +# This file is auto-generated from the current state of the database. Instead +# of editing this file, please use the migrations feature of Active Record to +# incrementally modify your database, and then regenerate this schema definition. +# +# This file is the source Rails uses to define your schema when running `bin/rails +# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to +# be faster and is potentially less error prone than running all of your +# migrations from scratch. Old migrations may fail to apply correctly if those +# migrations use external dependencies or application code. +# +# It's strongly recommended that you check this file into your version control system. + +ActiveRecord::Schema[8.0].define(version: 1) do + create_table 'solid_queue_blocked_executions', force: :cascade do |t| + t.bigint 'job_id', null: false + t.string 'queue_name', null: false + t.integer 'priority', default: 0, null: false + t.string 'concurrency_key', null: false + t.datetime 'expires_at', null: false + t.datetime 'created_at', null: false + t.index ['concurrency_key', 'priority', 'job_id'], name: 'index_solid_queue_blocked_executions_for_release' + t.index ['expires_at', 'concurrency_key'], name: 'index_solid_queue_blocked_executions_for_maintenance' + t.index ['job_id'], name: 'index_solid_queue_blocked_executions_on_job_id', unique: true + end + + create_table 'solid_queue_claimed_executions', force: :cascade do |t| + t.bigint 'job_id', null: false + t.bigint 'process_id' + t.datetime 'created_at', null: false + t.index ['job_id'], name: 'index_solid_queue_claimed_executions_on_job_id', unique: true + t.index ['process_id', 'job_id'], name: 'index_solid_queue_claimed_executions_on_process_id_and_job_id' + end + + create_table 'solid_queue_failed_executions', force: :cascade do |t| + t.bigint 'job_id', null: false + t.text 'error' + t.datetime 'created_at', null: false + t.index ['job_id'], name: 'index_solid_queue_failed_executions_on_job_id', unique: true + end + + create_table 'solid_queue_jobs', force: :cascade do |t| + t.string 'queue_name', null: false + t.string 'class_name', null: false + t.text 'arguments' + t.integer 'priority', default: 0, null: false + t.string 'active_job_id' + t.datetime 'scheduled_at' + t.datetime 'finished_at' + t.string 'concurrency_key' + t.datetime 'created_at', null: false + t.datetime 'updated_at', null: false + t.index ['active_job_id'], name: 'index_solid_queue_jobs_on_active_job_id' + t.index ['class_name'], name: 'index_solid_queue_jobs_on_class_name' + t.index ['finished_at'], name: 'index_solid_queue_jobs_on_finished_at' + t.index ['queue_name', 'finished_at'], name: 'index_solid_queue_jobs_for_filtering' + t.index ['scheduled_at', 'finished_at'], name: 'index_solid_queue_jobs_for_alerting' + end + + create_table 'solid_queue_pauses', force: :cascade do |t| + t.string 'queue_name', null: false + t.datetime 'created_at', null: false + t.index ['queue_name'], name: 'index_solid_queue_pauses_on_queue_name', unique: true + end + + create_table 'solid_queue_processes', force: :cascade do |t| + t.string 'kind', null: false + t.datetime 'last_heartbeat_at', null: false + t.bigint 'supervisor_id' + t.integer 'pid', null: false + t.string 'hostname' + t.text 'metadata' + t.datetime 'created_at', null: false + t.string 'name', null: false + t.index ['last_heartbeat_at'], name: 'index_solid_queue_processes_on_last_heartbeat_at' + t.index ['name', 'supervisor_id'], name: 'index_solid_queue_processes_on_name_and_supervisor_id', unique: true + t.index ['supervisor_id'], name: 'index_solid_queue_processes_on_supervisor_id' + end + + create_table 'solid_queue_ready_executions', force: :cascade do |t| + t.bigint 'job_id', null: false + t.string 'queue_name', null: false + t.integer 'priority', default: 0, null: false + t.datetime 'created_at', null: false + t.index ['job_id'], name: 'index_solid_queue_ready_executions_on_job_id', unique: true + t.index ['priority', 'job_id'], name: 'index_solid_queue_poll_all' + t.index ['queue_name', 'priority', 'job_id'], name: 'index_solid_queue_poll_by_queue' + end + + create_table 'solid_queue_recurring_executions', force: :cascade do |t| + t.bigint 'job_id', null: false + t.string 'task_key', null: false + t.datetime 'run_at', null: false + t.datetime 'created_at', null: false + t.index ['job_id'], name: 'index_solid_queue_recurring_executions_on_job_id', unique: true + t.index ['task_key', 'run_at'], name: 'index_solid_queue_recurring_executions_on_task_key_and_run_at', unique: true + end + + create_table 'solid_queue_recurring_tasks', force: :cascade do |t| + t.string 'key', null: false + t.string 'schedule', null: false + t.string 'command', limit: 2048 + t.string 'class_name' + t.text 'arguments' + t.string 'queue_name' + t.integer 'priority', default: 0 + t.boolean 'static', default: true, null: false + t.text 'description' + t.datetime 'created_at', null: false + t.datetime 'updated_at', null: false + t.index ['key'], name: 'index_solid_queue_recurring_tasks_on_key', unique: true + t.index ['static'], name: 'index_solid_queue_recurring_tasks_on_static' + end + + create_table 'solid_queue_scheduled_executions', force: :cascade do |t| + t.bigint 'job_id', null: false + t.string 'queue_name', null: false + t.integer 'priority', default: 0, null: false + t.datetime 'scheduled_at', null: false + t.datetime 'created_at', null: false + t.index ['job_id'], name: 'index_solid_queue_scheduled_executions_on_job_id', unique: true + t.index ['scheduled_at', 'priority', 'job_id'], name: 'index_solid_queue_dispatch_all' + end + + create_table 'solid_queue_semaphores', force: :cascade do |t| + t.string 'key', null: false + t.integer 'value', default: 1, null: false + t.datetime 'expires_at', null: false + t.datetime 'created_at', null: false + t.datetime 'updated_at', null: false + t.index ['expires_at'], name: 'index_solid_queue_semaphores_on_expires_at' + t.index ['key', 'value'], name: 'index_solid_queue_semaphores_on_key_and_value' + t.index ['key'], name: 'index_solid_queue_semaphores_on_key', unique: true + end + + add_foreign_key 'solid_queue_blocked_executions', 'solid_queue_jobs', column: 'job_id', on_delete: :cascade + add_foreign_key 'solid_queue_claimed_executions', 'solid_queue_jobs', column: 'job_id', on_delete: :cascade + add_foreign_key 'solid_queue_failed_executions', 'solid_queue_jobs', column: 'job_id', on_delete: :cascade + add_foreign_key 'solid_queue_ready_executions', 'solid_queue_jobs', column: 'job_id', on_delete: :cascade + add_foreign_key 'solid_queue_recurring_executions', 'solid_queue_jobs', column: 'job_id', on_delete: :cascade + add_foreign_key 'solid_queue_scheduled_executions', 'solid_queue_jobs', column: 'job_id', on_delete: :cascade +end