Separate the queue database

Gemfile

@@ -23,16 +23,13 @@ gem "jsbundling-rails", "~> 1.3.0"
 gem "puma", "~> 6.4.0"
 
 # Default database
-gem "sqlite3", "~> 1.7.0"
-
-# Enhanced SQLite3 adapter
-gem "activerecord-enhancedsqlite3-adapter", "~> 0.7.0"
+gem "sqlite3", "~> 2.1.0"
 
 # Cache store
 gem "solid_cache", "~> 1.0.0"
 
 # Background job processing
-gem "solid_queue", "~> 0.2.1"
+gem "solid_queue", "~> 1.0.0"
 
 # Action Cable adapter
 gem "solid_cable", "~> 3.0.0"

Gemfile.lock

@@ -54,9 +54,6 @@ GEM
       activemodel (= 7.2.1)
       activesupport (= 7.2.1)
       timeout (>= 0.4.0)
-    activerecord-enhancedsqlite3-adapter (0.7.0)
-      activerecord (>= 7.1)
-      sqlite3 (>= 1.6)
     activestorage (7.2.1)
       actionpack (= 7.2.1)
       activejob (= 7.2.1)
@@ -139,12 +136,17 @@ GEM
       rubocop
       smart_properties
     erubi (1.13.0)
+    et-orbi (1.2.11)
+      tzinfo
     ferrum (0.13)
       addressable (~> 2.5)
       concurrent-ruby (~> 1.1)
       webrick (~> 1.7)
       websocket-driver (>= 0.6, < 0.8)
     ffi (1.16.3)
+    fugit (1.11.1)
+      et-orbi (~> 1, >= 1.2.11)
+      raabro (~> 1.4)
     globalid (1.2.1)
       activesupport (>= 6.1)
     hashdiff (1.1.0)
@@ -237,6 +239,7 @@ GEM
     public_suffix (5.0.5)
     puma (6.4.3)
       nio4r (~> 2.0)
+    raabro (1.4.0)
     racc (1.8.1)
     rack (3.1.7)
     rack-session (2.0.0)
@@ -332,9 +335,14 @@ GEM
       activejob (>= 7.2)
       activerecord (>= 7.2)
       railties (>= 7.2)
-    solid_queue (0.2.2)
-      rails (~> 7.1)
-    sqlite3 (1.7.3)
+    solid_queue (1.0.0)
+      activejob (>= 7.1)
+      activerecord (>= 7.1)
+      concurrent-ruby (>= 1.3.1)
+      fugit (~> 1.11.0)
+      railties (>= 7.1)
+      thor (~> 1.3.1)
+    sqlite3 (2.1.0)
       mini_portile2 (~> 2.8.0)
     sshkit (1.22.2)
       base64
@@ -385,7 +393,6 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  activerecord-enhancedsqlite3-adapter (~> 0.7.0)
   acts_as_list (~> 1.2.0)
   bcrypt (~> 3.1.11)
   bootsnap (~> 1.18.0)
@@ -416,8 +423,8 @@ DEPENDENCIES
   simplecov-lcov (~> 0.8.0)
   solid_cable (~> 3.0.0)
   solid_cache (~> 1.0.0)
-  solid_queue (~> 0.2.1)
-  sqlite3 (~> 1.7.0)
+  solid_queue (~> 1.0.0)
+  sqlite3 (~> 2.1.0)
   standard (~> 1.25.0)
   standard-rails
   stimulus-rails (~> 1.3.4)

bin/jobs

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require_relative "../config/environment"
+require "solid_queue/cli"
+
+SolidQueue::Cli.start(ARGV)

config/application.rb

@@ -30,6 +30,7 @@ module BlackCandy
   has_config :db_url
   has_config :cache_db_url
   has_config :cable_db_url
+  has_config :queue_db_url
   has_config :media_path
   has_config :db_adapter, default: "sqlite"
   has_config :nginx_sendfile, default: false
@@ -43,8 +44,8 @@ module BlackCandy
 
     if value == "postgresql" &&
         ENV["RAILS_ENV"] == "production" &&
-        (config.db_url.blank? || config.cache_db_url.blank? || config.cable_db_url.blank?)
-      raise_config_validation_error "DB_URL, CABLE_DB_URL and CACHE_DB_URL are required if database adapter is postgresql"
+        (config.db_url.blank? || config.cache_db_url.blank? || config.cable_db_url.blank? || config.queue_db_url.blank?)
+      raise_config_validation_error "DB_URL, CABLE_DB_URL, QUEUE_DB_URL and CACHE_DB_URL are required if database adapter is postgresql"
     end
   end
 
@@ -61,7 +62,6 @@ class Application < Rails::Application
 
     config.active_storage.resolve_model_to_route = :rails_storage_proxy
 
-    config.solid_queue.silence_polling = true
     config.solid_queue.preserve_finished_jobs = false
 
     # Configuration for the application, engines, and railties goes here.

config/database.yml

@@ -2,6 +2,7 @@ sqlite_default: &sqlite_default
   adapter: sqlite3
   pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %>
   timeout: 5000
+  default_transaction_mode: immediate
 
 pg_default: &pg_default
   adapter: postgresql
@@ -25,6 +26,10 @@ development:
     <<: *primary_development
     database: blackcandy_development_cable
     migrations_paths: db/cable_migrate
+  queue:
+    <<: *primary_development
+    database: blackcandy_development_queue
+    migrations_paths: db/queue_migrate
 
 test:
   primary: &primary_test
@@ -41,6 +46,10 @@ test:
     <<: *primary_test
     database: blackcandy_test_cable
     migrations_paths: db/cable_migrate
+  queue:
+    <<: *primary_test
+    database: blackcandy_test_queue
+    migrations_paths: db/queue_migrate
 
 production:
   primary: &primary_production
@@ -54,6 +63,10 @@ production:
     <<: *primary_production
     url: <%= BlackCandy.config.cable_db_url %>
     migrations_paths: db/cable_migrate
+  queue:
+    <<: *primary_production
+    url: <%= BlackCandy.config.queue_db_url %>
+    migrations_paths: db/queue_migrate
 <% else %>
 development:
   primary:
@@ -67,6 +80,10 @@ development:
     <<: *sqlite_default
     database: storage/development_cable.sqlite3
     migrations_paths: db/cable_migrate
+  queue:
+    <<: *sqlite_default
+    database: storage/development_queue.sqlite3
+    migrations_paths: db/queue_migrate
 
 test:
   primary:
@@ -80,6 +97,10 @@ test:
     <<: *sqlite_default
     database: storage/test_cable.sqlite3
     migrations_paths: db/cable_migrate
+  queue:
+    <<: *sqlite_default
+    database: storage/test_queue.sqlite3
+    migrations_paths: db/queue_migrate
 
 production:
   primary:
@@ -93,4 +114,8 @@ production:
     <<: *sqlite_default
     database: storage/production_cable.sqlite3
     migrations_paths: db/cable_migrate
+  queue:
+    <<: *sqlite_default
+    database: storage/production_queue.sqlite3
+    migrations_paths: db/queue_migrate
 <% end %>

config/environments/development.rb

@@ -72,6 +72,7 @@
   config.action_controller.action_on_unpermitted_parameters = :raise
 
   config.active_job.queue_adapter = :solid_queue
+  config.solid_queue.connects_to = {database: {writing: :queue}}
 
   config.after_initialize do
     Bullet.enable = true

config/environments/production.rb

@@ -69,6 +69,7 @@
 
   # Use a real queuing backend for Active Job (and separate queues per environment).
   config.active_job.queue_adapter = :solid_queue
+  config.solid_queue.connects_to = {database: {writing: :queue}}
 
   # Enable locale fallbacks for I18n (makes lookups for any locale fall back to
   # the I18n.default_locale when a translation cannot be found).

config/solid_queue.yml → config/queue.yml

@@ -5,9 +5,8 @@ default: &default
       concurrency_maintenance_interval: 30
   workers:
     - queues: [critical, high, default, low]
-      # 2 more threads are used by worker and heartbeat task in solid queue
-      threads: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } - 2 %>
-      processes: 1
+      threads: 3
+      processes: <%= ENV.fetch("JOB_CONCURRENCY", 1) %>
       polling_interval: 0.1
 
 development:

config/recurring.yml

@@ -0,0 +1,10 @@
+# production:
+#   periodic_cleanup:
+#     class: CleanSoftDeletedRecordsJob
+#     queue: background
+#     args: [ 1000, { batch_size: 500 } ]
+#     schedule: every hour
+#   periodic_command:
+#     command: "SoftDeletedRecord.due.delete_all"
+#     priority: 2
+#     schedule: at 5am every day

db/cable_schema.rb

@@ -1,4 +1,16 @@
-ActiveRecord::Schema[7.1].define(version: 1) do
+# This file is auto-generated from the current state of the database. Instead
+# of editing this file, please use the migrations feature of Active Record to
+# incrementally modify your database, and then regenerate this schema definition.
+#
+# This file is the source Rails uses to define your schema when running `bin/rails
+# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to
+# be faster and is potentially less error prone than running all of your
+# migrations from scratch. Old migrations may fail to apply correctly if those
+# migrations use external dependencies or application code.
+#
+# It's strongly recommended that you check this file into your version control system.
+
+ActiveRecord::Schema[7.2].define(version: 1) do
   create_table "solid_cable_messages", force: :cascade do |t|
     t.binary "channel", limit: 1024, null: false
     t.binary "payload", limit: 536870912, null: false

db/cache_schema.rb

@@ -1,4 +1,14 @@
-# frozen_string_literal: true
+# This file is auto-generated from the current state of the database. Instead
+# of editing this file, please use the migrations feature of Active Record to
+# incrementally modify your database, and then regenerate this schema definition.
+#
+# This file is the source Rails uses to define your schema when running `bin/rails
+# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to
+# be faster and is potentially less error prone than running all of your
+# migrations from scratch. Old migrations may fail to apply correctly if those
+# migrations use external dependencies or application code.
+#
+# It's strongly recommended that you check this file into your version control system.
 
 ActiveRecord::Schema[7.2].define(version: 1) do
   create_table "solid_cache_entries", force: :cascade do |t|

db/migrate/20241018134323_drop_solid_queue_tables.rb

@@ -0,0 +1,96 @@
+class DropSolidQueueTables < ActiveRecord::Migration[7.2]
+  def change
+    drop_table :solid_queue_blocked_executions do |t|
+      t.integer "job_id", null: false
+      t.string "queue_name", null: false
+      t.integer "priority", default: 0, null: false
+      t.string "concurrency_key", null: false
+      t.datetime "expires_at", null: false
+      t.datetime "created_at", null: false
+      t.index ["expires_at", "concurrency_key"], name: "index_solid_queue_blocked_executions_for_maintenance"
+      t.index ["job_id"], name: "index_solid_queue_blocked_executions_on_job_id", unique: true
+    end
+
+    drop_table :solid_queue_claimed_executions do |t|
+      t.integer "job_id", null: false
+      t.bigint "process_id"
+      t.datetime "created_at", null: false
+      t.index ["job_id"], name: "index_solid_queue_claimed_executions_on_job_id", unique: true
+      t.index ["process_id", "job_id"], name: "index_solid_queue_claimed_executions_on_process_id_and_job_id"
+    end
+
+    drop_table :solid_queue_failed_executions do |t|
+      t.integer "job_id", null: false
+      t.text "error"
+      t.datetime "created_at", null: false
+      t.index ["job_id"], name: "index_solid_queue_failed_executions_on_job_id", unique: true
+    end
+
+    drop_table :solid_queue_jobs do |t|
+      t.string "queue_name", null: false
+      t.string "class_name", null: false
+      t.text "arguments"
+      t.integer "priority", default: 0, null: false
+      t.string "active_job_id"
+      t.datetime "scheduled_at"
+      t.datetime "finished_at"
+      t.string "concurrency_key"
+      t.datetime "created_at", null: false
+      t.datetime "updated_at", null: false
+      t.index ["active_job_id"], name: "index_solid_queue_jobs_on_active_job_id"
+      t.index ["class_name"], name: "index_solid_queue_jobs_on_class_name"
+      t.index ["finished_at"], name: "index_solid_queue_jobs_on_finished_at"
+      t.index ["queue_name", "finished_at"], name: "index_solid_queue_jobs_for_filtering"
+      t.index ["scheduled_at", "finished_at"], name: "index_solid_queue_jobs_for_alerting"
+    end
+
+    drop_table :solid_queue_pauses do |t|
+      t.string "queue_name", null: false
+      t.datetime "created_at", null: false
+      t.index ["queue_name"], name: "index_solid_queue_pauses_on_queue_name", unique: true
+    end
+
+    drop_table :solid_queue_processes do |t|
+      t.string "kind", null: false
+      t.datetime "last_heartbeat_at", null: false
+      t.bigint "supervisor_id"
+      t.integer "pid", null: false
+      t.string "hostname"
+      t.text "metadata"
+      t.datetime "created_at", null: false
+      t.index ["last_heartbeat_at"], name: "index_solid_queue_processes_on_last_heartbeat_at"
+      t.index ["supervisor_id"], name: "index_solid_queue_processes_on_supervisor_id"
+    end
+
+    drop_table :solid_queue_ready_executions do |t|
+      t.integer "job_id", null: false
+      t.string "queue_name", null: false
+      t.integer "priority", default: 0, null: false
+      t.datetime "created_at", null: false
+      t.index ["job_id"], name: "index_solid_queue_ready_executions_on_job_id", unique: true
+      t.index ["priority", "job_id"], name: "index_solid_queue_poll_all"
+      t.index ["queue_name", "priority", "job_id"], name: "index_solid_queue_poll_by_queue"
+    end
+
+    drop_table :solid_queue_scheduled_executions do |t|
+      t.integer "job_id", null: false
+      t.string "queue_name", null: false
+      t.integer "priority", default: 0, null: false
+      t.datetime "scheduled_at", null: false
+      t.datetime "created_at", null: false
+      t.index ["job_id"], name: "index_solid_queue_scheduled_executions_on_job_id", unique: true
+      t.index ["scheduled_at", "priority", "job_id"], name: "index_solid_queue_dispatch_all"
+    end
+
+    drop_table :solid_queue_semaphores do |t|
+      t.string "key", null: false
+      t.integer "value", default: 1, null: false
+      t.datetime "expires_at", null: false
+      t.datetime "created_at", null: false
+      t.datetime "updated_at", null: false
+      t.index ["expires_at"], name: "index_solid_queue_semaphores_on_expires_at"
+      t.index ["key", "value"], name: "index_solid_queue_semaphores_on_key_and_value"
+      t.index ["key"], name: "index_solid_queue_semaphores_on_key", unique: true
+    end
+  end
+end