diff --git a/Gemfile b/Gemfile index a9f12fb..c4a5aea 100644 --- a/Gemfile +++ b/Gemfile @@ -49,7 +49,7 @@ gem "bootsnap", require: false gem "image_processing", "~> 1.2" # for background jobs -gem "resque" +gem "solid_queue" group :development, :test do # See https://guides.rubyonrails.org/debugging_rails_applications.html#debugging-with-the-debug-gem diff --git a/Gemfile.lock b/Gemfile.lock index 08cb3db..e66dab5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -94,7 +94,7 @@ GEM rack-test (>= 0.6.3) regexp_parser (>= 1.5, < 3.0) xpath (~> 3.2) - concurrent-ruby (1.3.3) + concurrent-ruby (1.3.4) connection_pool (2.4.1) crass (1.0.6) cuprite (0.15.1) @@ -106,6 +106,8 @@ GEM reline (>= 0.3.8) drb (2.2.1) erubi (1.13.0) + et-orbi (1.2.11) + tzinfo ferrum (0.15) addressable (~> 2.5) concurrent-ruby (~> 1.1) @@ -113,6 +115,9 @@ GEM websocket-driver (~> 0.7) ffi (1.17.0-x86_64-darwin) ffi (1.17.0-x86_64-linux-gnu) + fugit (1.11.0) + et-orbi (~> 1, >= 1.2.11) + raabro (~> 1.4) globalid (1.2.1) activesupport (>= 6.1) i18n (1.14.5) @@ -145,11 +150,7 @@ GEM mini_magick (4.13.2) mini_mime (1.1.5) minitest (5.24.1) - mono_logger (1.1.2) msgpack (1.7.2) - multi_json (1.15.0) - mustermann (3.0.2) - ruby2_keywords (~> 0.0.1) mutex_m (0.2.0) net-imap (0.4.14) date @@ -171,11 +172,9 @@ GEM public_suffix (6.0.1) puma (6.4.2) nio4r (~> 2.0) + raabro (1.4.0) racc (1.8.1) rack (3.1.7) - rack-protection (4.0.0) - base64 (>= 0.1.0) - rack (>= 3.0.0, < 4) rack-session (2.0.0) rack (>= 3.0.0) rack-test (2.1.0) @@ -216,26 +215,18 @@ GEM rdoc (6.7.0) psych (>= 4.0.0) redis (4.8.1) - redis-namespace (1.11.0) - redis (>= 4) regexp_parser (2.9.2) reline (0.5.9) io-console (~> 0.5) - resque (2.6.0) - mono_logger (~> 1.0) - multi_json (~> 1.0) - redis-namespace (~> 1.6) - sinatra (>= 0.9.2) ruby-vips (2.2.2) ffi (~> 1.12) logger - ruby2_keywords (0.0.5) - sinatra (4.0.0) - mustermann (~> 3.0) - rack (>= 3.0.0, < 4) - rack-protection (= 4.0.0) - rack-session (>= 2.0.0, < 3) - tilt (~> 2.0) + solid_queue (0.4.1) + activejob (>= 7.1) + activerecord (>= 7.1) + concurrent-ruby (>= 1.3.1) + fugit (~> 1.11.0) + railties (>= 7.1) sprockets (4.2.1) concurrent-ruby (~> 1.0) rack (>= 2.2.4, < 4) @@ -247,7 +238,6 @@ GEM railties (>= 6.0.0) stringio (3.1.1) thor (1.3.1) - tilt (2.4.0) timeout (0.4.1) turbo-rails (2.0.6) actionpack (>= 6.0.0) @@ -285,7 +275,7 @@ DEPENDENCIES puma (~> 6.0) rails (= 7.1) redis (~> 4.0) - resque + solid_queue sprockets-rails stimulus-rails turbo-rails diff --git a/config/application.rb b/config/application.rb index 75133cd..4777180 100644 --- a/config/application.rb +++ b/config/application.rb @@ -23,5 +23,7 @@ class Application < Rails::Application # # config.time_zone = "Central Time (US & Canada)" # config.eager_load_paths << Rails.root.join("extras") + + config.active_job.queue_adapter = :solid_queue end end diff --git a/config/environments/development.rb b/config/environments/development.rb index c33db6b..3e0cc1c 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -75,7 +75,7 @@ config.action_controller.raise_on_missing_callback_actions = true # Use Resque as ActiveJob Adapter - config.active_job.queue_adapter = :resque + # config.active_job.queue_adapter = :resque # ActiveStorageのURLの期限切れを伸ばす # - 動画の再生が5分で途切れる問題のワークアラウンド diff --git a/config/environments/production.rb b/config/environments/production.rb index 7e083a2..2e593bc 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -69,7 +69,7 @@ # config.cache_store = :mem_cache_store # Use a real queuing backend for Active Job (and separate queues per environment). - config.active_job.queue_adapter = :resque + # config.active_job.queue_adapter = :solid_queue # config.active_job.queue_name_prefix = "ytdlor_production" config.action_mailer.perform_caching = false diff --git a/config/initializers/resque.rb b/config/initializers/resque.rb deleted file mode 100644 index f49b542..0000000 --- a/config/initializers/resque.rb +++ /dev/null @@ -1,6 +0,0 @@ -rails_root = ENV['RAILS_ROOT'] || File.dirname(__FILE__) + '/../..' -rails_env = ENV['RAILS_ENV'] || 'development' -config_file = rails_root + '/config/resque.yml' - -resque_config = YAML::load(ERB.new(IO.read(config_file)).result) -Resque.redis = resque_config[rails_env] diff --git a/config/resque.yml b/config/resque.yml deleted file mode 100644 index 43cab42..0000000 --- a/config/resque.yml +++ /dev/null @@ -1,3 +0,0 @@ -development: <%= ENV['REDIS_URL'] %> -test: <%= ENV['REDIS_URL'] %> -production: <%= ENV['REDIS_URL'] %> diff --git a/config/solid_queue.yml b/config/solid_queue.yml new file mode 100644 index 0000000..569ca5e --- /dev/null +++ b/config/solid_queue.yml @@ -0,0 +1,26 @@ +default: &default + dispatchers: + - polling_interval: 1 + batch_size: 500 + workers: + - queues: "default" + threads: 1 + processes: 1 + polling_interval: 1 + - queues: "thumbnail" + threads: 1 + processes: 1 + polling_interval: 1 + - queues: "video" + threads: 1 + processes: 1 + polling_interval: 1 + +development: + <<: *default + +test: + <<: *default + +production: + <<: *default diff --git a/db/migrate/20240810224902_create_solid_queue_tables.solid_queue.rb b/db/migrate/20240810224902_create_solid_queue_tables.solid_queue.rb new file mode 100644 index 0000000..6d8c4f6 --- /dev/null +++ b/db/migrate/20240810224902_create_solid_queue_tables.solid_queue.rb @@ -0,0 +1,101 @@ +# This migration comes from solid_queue (originally 20231211200639) +class CreateSolidQueueTables < ActiveRecord::Migration[7.0] + def change + create_table :solid_queue_jobs do |t| + t.string :queue_name, null: false + t.string :class_name, null: false, index: true + t.text :arguments + t.integer :priority, default: 0, null: false + t.string :active_job_id, index: true + t.datetime :scheduled_at + t.datetime :finished_at, index: true + t.string :concurrency_key + + t.timestamps + + t.index [ :queue_name, :finished_at ], name: "index_solid_queue_jobs_for_filtering" + t.index [ :scheduled_at, :finished_at ], name: "index_solid_queue_jobs_for_alerting" + end + + create_table :solid_queue_scheduled_executions do |t| + t.references :job, index: { unique: true }, null: false + t.string :queue_name, null: false + t.integer :priority, default: 0, null: false + t.datetime :scheduled_at, null: false + + t.datetime :created_at, null: false + + t.index [ :scheduled_at, :priority, :job_id ], name: "index_solid_queue_dispatch_all" + end + + create_table :solid_queue_ready_executions do |t| + t.references :job, index: { unique: true }, null: false + t.string :queue_name, null: false + t.integer :priority, default: 0, null: false + + t.datetime :created_at, null: false + + t.index [ :priority, :job_id ], name: "index_solid_queue_poll_all" + t.index [ :queue_name, :priority, :job_id ], name: "index_solid_queue_poll_by_queue" + end + + create_table :solid_queue_claimed_executions do |t| + t.references :job, index: { unique: true }, null: false + t.bigint :process_id + t.datetime :created_at, null: false + + t.index [ :process_id, :job_id ] + end + + create_table :solid_queue_blocked_executions do |t| + t.references :job, index: { unique: true }, null: false + t.string :queue_name, null: false + t.integer :priority, default: 0, null: false + t.string :concurrency_key, null: false + t.datetime :expires_at, null: false + + t.datetime :created_at, null: false + + t.index [ :expires_at, :concurrency_key ], name: "index_solid_queue_blocked_executions_for_maintenance" + end + + create_table :solid_queue_failed_executions do |t| + t.references :job, index: { unique: true }, null: false + t.text :error + t.datetime :created_at, null: false + end + + create_table :solid_queue_pauses do |t| + t.string :queue_name, null: false, index: { unique: true } + t.datetime :created_at, null: false + end + + create_table :solid_queue_processes do |t| + t.string :kind, null: false + t.datetime :last_heartbeat_at, null: false, index: true + t.bigint :supervisor_id, index: true + + t.integer :pid, null: false + t.string :hostname + t.text :metadata + + t.datetime :created_at, null: false + end + + create_table :solid_queue_semaphores do |t| + t.string :key, null: false, index: { unique: true } + t.integer :value, default: 1, null: false + t.datetime :expires_at, null: false, index: true + + t.timestamps + + t.index [ :key, :value ], name: "index_solid_queue_semaphores_on_key_and_value" + end + + add_foreign_key :solid_queue_blocked_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade + add_foreign_key :solid_queue_claimed_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade + add_foreign_key :solid_queue_failed_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade + add_foreign_key :solid_queue_ready_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade + add_foreign_key :solid_queue_scheduled_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade + end +end diff --git a/db/migrate/20240810224903_add_missing_index_to_blocked_executions.solid_queue.rb b/db/migrate/20240810224903_add_missing_index_to_blocked_executions.solid_queue.rb new file mode 100644 index 0000000..2d96b10 --- /dev/null +++ b/db/migrate/20240810224903_add_missing_index_to_blocked_executions.solid_queue.rb @@ -0,0 +1,6 @@ +# This migration comes from solid_queue (originally 20240110143450) +class AddMissingIndexToBlockedExecutions < ActiveRecord::Migration[7.1] + def change + add_index :solid_queue_blocked_executions, [ :concurrency_key, :priority, :job_id ], name: "index_solid_queue_blocked_executions_for_release" + end +end diff --git a/db/migrate/20240810224904_create_recurring_executions.solid_queue.rb b/db/migrate/20240810224904_create_recurring_executions.solid_queue.rb new file mode 100644 index 0000000..4c49821 --- /dev/null +++ b/db/migrate/20240810224904_create_recurring_executions.solid_queue.rb @@ -0,0 +1,15 @@ +# This migration comes from solid_queue (originally 20240218110712) +class CreateRecurringExecutions < ActiveRecord::Migration[7.1] + def change + create_table :solid_queue_recurring_executions do |t| + t.references :job, index: { unique: true }, null: false + t.string :task_key, null: false + t.datetime :run_at, null: false + t.datetime :created_at, null: false + + t.index [ :task_key, :run_at ], unique: true + end + + add_foreign_key :solid_queue_recurring_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade + end +end diff --git a/db/schema.rb b/db/schema.rb index 4ccffa0..e545a0b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2023_09_23_194012) do +ActiveRecord::Schema[7.1].define(version: 2024_08_10_224904) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -50,6 +50,115 @@ t.datetime "updated_at", null: false end + create_table "solid_queue_blocked_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.string "concurrency_key", null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.index ["concurrency_key", "priority", "job_id"], name: "index_solid_queue_blocked_executions_for_release" + t.index ["expires_at", "concurrency_key"], name: "index_solid_queue_blocked_executions_for_maintenance" + t.index ["job_id"], name: "index_solid_queue_blocked_executions_on_job_id", unique: true + end + + create_table "solid_queue_claimed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.bigint "process_id" + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_claimed_executions_on_job_id", unique: true + t.index ["process_id", "job_id"], name: "index_solid_queue_claimed_executions_on_process_id_and_job_id" + end + + create_table "solid_queue_failed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.text "error" + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_failed_executions_on_job_id", unique: true + end + + create_table "solid_queue_jobs", force: :cascade do |t| + t.string "queue_name", null: false + t.string "class_name", null: false + t.text "arguments" + t.integer "priority", default: 0, null: false + t.string "active_job_id" + t.datetime "scheduled_at" + t.datetime "finished_at" + t.string "concurrency_key" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["active_job_id"], name: "index_solid_queue_jobs_on_active_job_id" + t.index ["class_name"], name: "index_solid_queue_jobs_on_class_name" + t.index ["finished_at"], name: "index_solid_queue_jobs_on_finished_at" + t.index ["queue_name", "finished_at"], name: "index_solid_queue_jobs_for_filtering" + t.index ["scheduled_at", "finished_at"], name: "index_solid_queue_jobs_for_alerting" + end + + create_table "solid_queue_pauses", force: :cascade do |t| + t.string "queue_name", null: false + t.datetime "created_at", null: false + t.index ["queue_name"], name: "index_solid_queue_pauses_on_queue_name", unique: true + end + + create_table "solid_queue_processes", force: :cascade do |t| + t.string "kind", null: false + t.datetime "last_heartbeat_at", null: false + t.bigint "supervisor_id" + t.integer "pid", null: false + t.string "hostname" + t.text "metadata" + t.datetime "created_at", null: false + t.index ["last_heartbeat_at"], name: "index_solid_queue_processes_on_last_heartbeat_at" + t.index ["supervisor_id"], name: "index_solid_queue_processes_on_supervisor_id" + end + + create_table "solid_queue_ready_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_ready_executions_on_job_id", unique: true + t.index ["priority", "job_id"], name: "index_solid_queue_poll_all" + t.index ["queue_name", "priority", "job_id"], name: "index_solid_queue_poll_by_queue" + end + + create_table "solid_queue_recurring_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "task_key", null: false + t.datetime "run_at", null: false + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_recurring_executions_on_job_id", unique: true + t.index ["task_key", "run_at"], name: "index_solid_queue_recurring_executions_on_task_key_and_run_at", unique: true + end + + create_table "solid_queue_scheduled_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "scheduled_at", null: false + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_scheduled_executions_on_job_id", unique: true + t.index ["scheduled_at", "priority", "job_id"], name: "index_solid_queue_dispatch_all" + end + + create_table "solid_queue_semaphores", force: :cascade do |t| + t.string "key", null: false + t.integer "value", default: 1, null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["expires_at"], name: "index_solid_queue_semaphores_on_expires_at" + t.index ["key", "value"], name: "index_solid_queue_semaphores_on_key_and_value" + t.index ["key"], name: "index_solid_queue_semaphores_on_key", unique: true + end + add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" + add_foreign_key "solid_queue_blocked_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_claimed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_failed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_ready_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_recurring_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_scheduled_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade end diff --git a/docker-compose-development.yml b/docker-compose-development.yml index d33f9f5..659cfce 100644 --- a/docker-compose-development.yml +++ b/docker-compose-development.yml @@ -8,13 +8,7 @@ services: web: volumes: *rails_volumes - default_worker: - volumes: - *rails_volumes - thumbnail_worker: - volumes: - *rails_volumes - video_worker: + worker: volumes: *rails_volumes test: @@ -27,6 +21,7 @@ services: depends_on: - db - redis + - worker environment: SECRET_KEY_BASE: ${SECRET_KEY_BASE} REDIS_URL: "redis://redis:6379/1" diff --git a/docker-compose-production.yml b/docker-compose-production.yml index ab478fb..9d7be6f 100644 --- a/docker-compose-production.yml +++ b/docker-compose-production.yml @@ -26,26 +26,13 @@ services: environment: << : *rails_environment RAILS_SERVE_STATIC_FILES: 1 - default_worker: - build: - *rails_build - volumes: - *rails_volumes - environment: - *rails_environment - thumbnail_worker: - build: - *rails_build - volumes: - *rails_volumes - environment: - *rails_environment - video_worker: + worker: build: *rails_build volumes: *rails_volumes environment: *rails_environment + volumes: storage: diff --git a/docker-compose.yml b/docker-compose.yml index 6e3baf5..302dea7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,7 +27,7 @@ services: environment: *rails_environment command: ./server.sh - default_worker: + worker: build: *rails_build image: ${COMPOSE_PROJECT_NAME}:latest @@ -35,31 +35,8 @@ services: - db - redis environment: - << : *resque_environment - QUEUE: default - command: rake resque:work - thumbnail_worker: - build: - *rails_build - image: ${COMPOSE_PROJECT_NAME}:latest - depends_on: - - db - - redis - environment: - << : *resque_environment - QUEUE: thumbnail - command: rake resque:work - video_worker: - build: - *rails_build - image: ${COMPOSE_PROJECT_NAME}:latest - depends_on: - - db - - redis - environment: - << : *resque_environment - QUEUE: video - command: rake resque:work + *rails_environment + command: rake solid_queue:start redis: image: redis db: @@ -70,5 +47,6 @@ services: target: /var/lib/postgresql/data environment: POSTGRES_PASSWORD: password + volumes: postgres: