Merge pull request #29 from basecamp/sqlite

Run solid_queue with SQLite

.github/workflows/main.yml

@@ -9,7 +9,7 @@ jobs:
  fail-fast: false
  matrix:
  ruby-version: [3.2.2]
- database: [mysql]
+ database: [mysql, sqlite]
  services:
  mysql:
  image: mysql:8.0.31

test/dummy/config/database.yml

@@ -5,6 +5,22 @@
 # gem "mysql2"
 #
 
+<% if ENV["TARGET_DB"] == "sqlite" %>
+default: &default
+ adapter: sqlite3
+ pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 50 } %>
+ retries: 100
+
+development:
+ <<: *default
+ database: db/development.sqlite3
+
+test:
+ <<: *default
+ pool: 20
+ database: db/test.sqlite3
+
+<% else %>
 default: &default
  adapter: mysql2
  username: root
@@ -20,3 +36,5 @@ test:
  <<: *default
  pool: 20
  database: solid_queue_test
+<% end %>
+

test/dummy/config/initializers/sqlite3.rb

@@ -0,0 +1,30 @@
+module SqliteImmediateTransactions
+ def begin_db_transaction
+ log("begin immediate transaction", "TRANSACTION") do
+ with_raw_connection(allow_retry: true, materialize_transactions: false) do |conn|
+ conn.transaction(:immediate)
+ end
+ end
+ end
+end
+
+module SQLite3Configuration
+ private
+ def configure_connection
+ super
+
+ if @config[:retries]
+ retries = self.class.type_cast_config_to_integer(@config[:retries])
+ raw_connection.busy_handler do |count|
+ (count <= retries).tap { |result| sleep count * 0.001 if result }
+ end
+ end
+ end
+end
+
+ActiveSupport.on_load :active_record do
+ if defined?(ActiveRecord::ConnectionAdapters::SQLite3Adapter)
+ ActiveRecord::ConnectionAdapters::SQLite3Adapter.prepend SqliteImmediateTransactions
+ ActiveRecord::ConnectionAdapters::SQLite3Adapter.prepend SQLite3Configuration
+ end
+end

test/dummy/config/solid_queue.yml

@@ -5,7 +5,7 @@ default: &default
  default:
  pool_size: 5
  scheduler:
- polling_interval: 300
+ polling_interval: 1
  batch_size: 500
 
 development:

test/integration/jobs_lifecycle_test.rb

@@ -44,7 +44,7 @@ class JobsLifecycleTest < ActiveSupport::TestCase
 
  travel_to 5.days.from_now
 
- wait_for_jobs_to_finish_for(0.5.seconds)
+ wait_for_jobs_to_finish_for(5.seconds)
 
  assert_equal 2, JobBuffer.size
  assert_equal "I'm scheduled later", JobBuffer.last_value

test/integration/processes_lifecycle_test.rb

@@ -113,7 +113,7 @@ class ProcessLifecycleTest < ActiveSupport::TestCase
 
  test "term supervisor exceeding timeout while there are jobs in-flight" do
  no_pause = enqueue_store_result_job("no pause")
- pause = enqueue_store_result_job("pause", pause: SolidQueue.shutdown_timeout + 0.1.second)
+ pause = enqueue_store_result_job("pause", pause: SolidQueue.shutdown_timeout + 1.second)
 
  signal_process(@pid, :TERM, wait: 0.1.second)
  wait_for_jobs_to_finish_for(SolidQueue.shutdown_timeout + 0.1.second)

test/test_helper.rb

@@ -15,6 +15,16 @@
  ActiveSupport::TestCase.fixtures :all
 end
 
+module BlockLogDeviceTimeoutExceptions
+ def write(...)
+ # Prevents Timeout exceptions from occurring during log writing, where they will be swallowed
+ # See https://bugs.ruby-lang.org/issues/9115
+ Thread.handle_interrupt(Timeout::Error => :never, Timeout::ExitException => :never) { super }
+ end
+end
+
+Logger::LogDevice.prepend(BlockLogDeviceTimeoutExceptions)
+
 class ActiveSupport::TestCase
  setup do
  SolidQueue.logger = ActiveSupport::Logger.new(nil)

test/unit/supervisor_test.rb

@@ -47,6 +47,7 @@ class SupervisorTest < ActiveSupport::TestCase
  end
 
  test "abort if there's already a pidfile for a supervisor" do
+ FileUtils.mkdir_p(File.dirname(@pidfile))
  File.write(@pidfile, ::Process.pid.to_s)
 
  pid = run_supervisor_as_fork(mode: :all)