Allow multiple workers for the same queue and configuration

From Donal:

> A single worker won't use more than 1 CPU core, so if you need to
> dedicate more than on CPU core on a VM to a queue you'll need multiple
> workers. I think I'll need this for the new campfire

This changes the configuration format to be like this:

```
workers:
    - queues: haystack_production_incineration
      processes: 3
      threads: 4
      polling_interval: 2
    - queues: haystack_production_recycling
      threads: 3
      polling_interval: 5
```

`processes` is 1 by default, and `queues` should be now an array of
queues (which can include wildcards).

lib/solid_queue/configuration.rb

@@ -3,7 +3,8 @@
 module SolidQueue
  class Configuration
  WORKER_DEFAULTS = {
- pool_size: 5,
+ threads: 5,
+ processes: 1,
  polling_interval: 0.1
  }
 
@@ -28,7 +29,10 @@ def runners
 
  def workers
  if mode.in? %i[ work all]
- workers_options.values.map { |worker_options| SolidQueue::Worker.new(**worker_options) }
+ workers_options.flat_map do |worker_options|
+ processes = worker_options.fetch(:processes, WORKER_DEFAULTS[:processes])
+ processes.times.collect { SolidQueue::Worker.new(**worker_options.with_defaults(WORKER_DEFAULTS)) }
+ end
  else
  []
  end
@@ -41,8 +45,8 @@ def scheduler
  end
 
  def max_number_of_threads
- # At most pool_size thread in each worker + 1 thread for the worker + 1 thread for the heartbeat task
- workers_options.values.map { |options| options[:pool_size] }.max + 2
+ # At most "threads" in each worker + 1 thread for the worker + 1 thread for the heartbeat task
+ workers_options.map { |options| options[:threads] }.max + 2
  end
 
  private
@@ -54,9 +58,7 @@ def config_from(file_or_hash, env: Rails.env)
  end
 
  def workers_options
- @workers_options ||= (raw_config[:workers] || {}).each_with_object({}) do |(queue_string, options), hsh|
- hsh[queue_string] = options.merge(queues: queue_string.to_s).with_defaults(WORKER_DEFAULTS)
- end.deep_symbolize_keys
+ @workers_options ||= (raw_config[:workers] || {}).map { |options| options.dup.symbolize_keys }
  end
 
  def scheduler_options

lib/solid_queue/queue_parser.rb

@@ -4,8 +4,8 @@ module SolidQueue
  class QueueParser
  attr_reader :raw_queues, :relation
 
- def initialize(queue_string, relation)
- @raw_queues = queue_string.split(",").map(&:strip).presence || [ "*" ]
+ def initialize(queue_list, relation)
+ @raw_queues = Array(queue_list).map(&:strip).presence || [ "*" ]
  @relation = relation
  end
 

lib/solid_queue/worker.rb

@@ -10,8 +10,8 @@ def initialize(**options)
  options = options.dup.with_defaults(SolidQueue::Configuration::WORKER_DEFAULTS)
 
  @polling_interval = options[:polling_interval]
- @queues = options[:queues].to_s
- @pool = Pool.new(options[:pool_size], on_idle: -> { wake_up })
+ @queues = options[:queues]
+ @pool = Pool.new(options[:threads], on_idle: -> { wake_up })
  end
 
  private
@@ -42,7 +42,7 @@ def all_work_completed?
  end
 
  def metadata
- super.merge(queues: queues, pool_size: pool.size, idle_threads: pool.idle_threads, polling_interval: polling_interval)
+ super.merge(queues: queues, thread_pool_size: pool.size, idle_threads: pool.idle_threads, polling_interval: polling_interval)
  end
  end
 end

test/dummy/config/solid_queue.yml

@@ -1,9 +1,9 @@
 default: &default
  workers:
- background:
- pool_size: 3
- default:
- pool_size: 5
+ - queues: background
+ threads: 3
+ - queues: default
+ threads: 5
  scheduler:
  polling_interval: 1
  batch_size: 500

test/integration/jobs_lifecycle_test.rb

@@ -3,7 +3,7 @@
 
 class JobsLifecycleTest < ActiveSupport::TestCase
  setup do
- @worker = SolidQueue::Worker.new(queue_name: "background", pool_size: 3, polling_interval: 1)
+ @worker = SolidQueue::Worker.new(queues: "background", threads: 3, polling_interval: 1)
  @scheduler = SolidQueue::Scheduler.new(batch_size: 10, polling_interval: 1)
  end
 

test/models/solid_queue/ready_execution_test.rb

@@ -55,7 +55,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase
  (SolidQueue::Job.all - @jobs).each(&:prepare_for_execution)
 
  assert_claimed_jobs(SolidQueue::Job.count) do
- SolidQueue::ReadyExecution.claim("fixtures,background", SolidQueue::Job.count + 1, 42)
+ SolidQueue::ReadyExecution.claim(%w[ fixtures background ], SolidQueue::Job.count + 1, 42)
  end
  end
 
@@ -93,7 +93,7 @@ class SolidQueue::ReadyExecutionTest < ActiveSupport::TestCase
  (SolidQueue::Job.all - @jobs).each(&:prepare_for_execution)
 
  assert_claimed_jobs(SolidQueue::Job.count) do
- SolidQueue::ReadyExecution.claim("fix*,background", SolidQueue::Job.count + 1, 42)
+ SolidQueue::ReadyExecution.claim(%w[ fix* background ], SolidQueue::Job.count + 1, 42)
  end
  end
 

test/unit/configuration_test.rb

@@ -9,15 +9,28 @@ class ConfigurationTest < ActiveSupport::TestCase
  end
 
  test "provide configuration as a hash and fill defaults" do
- config_as_hash = { workers: { background: { polling_interval: 10 } } }
+ background_worker = { queues: "background", polling_interval: 10 }
+ config_as_hash = { workers: [ background_worker, background_worker ] }
  configuration = SolidQueue::Configuration.new(mode: :all, load_from: config_as_hash)
 
  assert_equal SolidQueue::Configuration::SCHEDULER_DEFAULTS[:polling_interval], configuration.scheduler.polling_interval
- assert configuration.workers.detect { |w| w.queues == "background" }.pool.size > 0
+ assert_equal 2, configuration.workers.count
+ assert_equal [ "background" ], configuration.workers.map(&:queues).uniq
+ assert_equal [ 10 ], configuration.workers.map(&:polling_interval).uniq
  end
 
  test "max number of threads" do
  configuration = SolidQueue::Configuration.new(mode: :all)
  assert 7, configuration.max_number_of_threads
  end
+
+ test "mulitple workers with the same configuration" do
+ background_worker = { queues: "background", polling_interval: 10, processes: 3 }
+ config_as_hash = { workers: [ background_worker ] }
+ configuration = SolidQueue::Configuration.new(mode: :work, load_from: config_as_hash)
+
+ assert_equal 3, configuration.workers.count
+ assert_equal [ "background" ], configuration.workers.map(&:queues).uniq
+ assert_equal [ 10 ], configuration.workers.map(&:polling_interval).uniq
+ end
 end

test/unit/worker_test.rb

@@ -5,7 +5,7 @@ class WorkerTest < ActiveSupport::TestCase
  include ActiveSupport::Testing::MethodCallAssertions
 
  setup do
- @worker = SolidQueue::Worker.new(queue_name: "background", pool_size: 3, polling_interval: 10)
+ @worker = SolidQueue::Worker.new(queues: "background", threads: 3, polling_interval: 10)
  end
 
  teardown do