From db404e0f856d64560231ff321f377b3e01e800d2 Mon Sep 17 00:00:00 2001 From: Dmitry Bochkarev Date: Wed, 7 Apr 2021 10:28:35 +0500 Subject: [PATCH 01/15] Add Github Actions --- .github/workflows/ruby.yml | 44 ++++++++++++++++++++++++++++++++++++++ .travis.yml | 19 ---------------- exe/qless-config | 6 +++++- exe/qless-stats | 6 +++++- spec/spec_helper.rb | 14 ++++++------ utils/exe/qless-campfire | 18 ++++++++++------ utils/exe/qless-growl | 18 ++++++++++------ 7 files changed, 83 insertions(+), 42 deletions(-) create mode 100644 .github/workflows/ruby.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml new file mode 100644 index 00000000..afcd04ef --- /dev/null +++ b/.github/workflows/ruby.yml @@ -0,0 +1,44 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. +# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake +# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby + +name: Ruby + +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + ruby-version: ["2.5.5"] + redis-version: ["2.8"] + + services: + redis: + image: redis:${{ matrix.redis-version }} + # Map TCP port 6379 on Docker host to a random free port on the Redis container + ports: + - 6379/tcp + + steps: + - uses: actions/checkout@v2 + - name: Set up Ruby + # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby, + # change this to (see https://github.com/ruby/setup-ruby#versioning): + # uses: ruby/setup-ruby@v1 + uses: ruby/setup-ruby@473e4d8fe5dd94ee328fdfca9f8c9c7afc9dae5e + with: + ruby-version: ${{ matrix.ruby-version }} + bundler-cache: true # runs 'bundle install' and caches installed gems automatically + - name: Run tests + env: + REDIS_PORT: ${{ job.services.redis.ports['6379'] }} + run: bundle exec rspec diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 291754de..00000000 --- a/.travis.yml +++ /dev/null @@ -1,19 +0,0 @@ -dist: trusty -sudo: false -bundler_args: --without extras -# before_script: -# - bundle exec rake core:verify -branches: - only: - - master -cache: - bundler: true -services: - - redis-server -script: bundle exec rake -rvm: - - 2.3.5 - - 2.4.2 - - 2.5 - - 2.6 - - 2.7 diff --git a/exe/qless-config b/exe/qless-config index 3886fa4b..718a68b4 100755 --- a/exe/qless-config +++ b/exe/qless-config @@ -6,7 +6,11 @@ require 'thor' class Configurator < Thor - class_option :redis, :default => 'redis://localhost:6379/0' + REDIS_HOST = ENV.fetch('REDIS_HOST', 'localhost') + REDIS_PORT = Integer(ENV.fetch('REDIS_PORT', '6379')) + REDIS_DB = Integer(ENV.fetch('REDIS_DB', '0')) + + class_option :redis, :default => "redis://#{REDIS_HOST}:#{REDIS_PORT}/#{REDIS_DB}" no_commands do def qless diff --git a/exe/qless-stats b/exe/qless-stats index b6cf80cc..4dc1648a 100755 --- a/exe/qless-stats +++ b/exe/qless-stats @@ -6,7 +6,11 @@ require 'thor' class Stats < Thor - class_option :redis, :default => 'redis://localhost:6379/0' + REDIS_HOST = ENV.fetch('REDIS_HOST', 'localhost') + REDIS_PORT = Integer(ENV.fetch('REDIS_PORT', '6379')) + REDIS_DB = Integer(ENV.fetch('REDIS_DB', '0')) + + class_option :redis, :default => "redis://#{REDIS_HOST}:#{REDIS_PORT}/#{REDIS_DB}" class_option :interval, :type => :numeric, :default => 60, :desc => 'Interval (in seconds) between stats collections' class_option :count, :type => :numeric, :default => 0, diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 737712fe..d0f3e142 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -38,16 +38,16 @@ module RedisHelpers extend self def redis_config - return @redis_config unless @redis_config.nil? - if File.exist?('./spec/redis.config.yml') - @redis_config = YAML.load_file('./spec/redis.config.yml') - else - @redis_config = {} - end + return @redis_config if defined?(@redis_config) + + @redis_config = { + host: ENV.fetch('REDIS_HOST', 'localhost'), + port: Integer(ENV.fetch('REDIS_PORT', '6379')), + db: Integer(ENV.fetch('REDIS_DB', '0')) + } end def redis_url - return 'redis://localhost:6379/0' if redis_config.empty? c = redis_config "redis://#{c[:host]}:#{c[:port]}/#{c.fetch(:db, 0)}" end diff --git a/utils/exe/qless-campfire b/utils/exe/qless-campfire index 343b8597..e08258ab 100755 --- a/utils/exe/qless-campfire +++ b/utils/exe/qless-campfire @@ -6,12 +6,16 @@ require 'qless' require 'tinder' require 'micro-optparse' +REDIS_HOST = ENV.fetch('REDIS_HOST', 'localhost') +REDIS_PORT = Integer(ENV.fetch('REDIS_PORT', '6379')) +REDIS_DB = Integer(ENV.fetch('REDIS_DB', '0')) + @options = Parser.new do |p| p.banner = 'This agent lets you get campfire notifications for the progress of tracked jobs' p.option :subdomain, 'campfire subdomain' , :default => '', :value_satisfies => lambda { |subdomain| subdomain.is_a?(String) } p.option :token , 'campfire token for bot', :default => '', :value_satisfies => lambda { |subdomain| subdomain.is_a?(String) } p.option :room , 'campfire room to talk in (defaults to first room)', :default => '' - p.option :host , 'host:port for your qless redis instance', :default => 'localhost:6379' + p.option :host , 'host:port for your qless redis instance', :default => "#{REDIS_HOST}:#{REDIS_PORT}/#{REDIS_DB}" p.option :web , 'host:port for your qless web ui', :default => 'localhost:5678' end.process! @@ -52,7 +56,7 @@ end on.stalled { |jid| event(jid) { |job| 'stalled' } } on.track { |jid| event(jid) { |job| 'is being tracked' } } on.untrack { |jid| event(jid) { |job| 'not longer tracked' } } - + on.completed do |jid| event(jid) do |job| if job @@ -66,7 +70,7 @@ end end end end - + on.popped do |jid| event(jid) do |job| if job @@ -76,7 +80,7 @@ end end end end - + on.put do |jid| event(jid) do |job| if job @@ -86,7 +90,7 @@ end end end end - + on.failed do |jid| job = @client.jobs[jid] if job @@ -96,9 +100,9 @@ end speak("#{jid} failed") end end - + puts 'Listening...' - + Signal.trap("INT") do puts 'Exiting' Process.exit(0) diff --git a/utils/exe/qless-growl b/utils/exe/qless-growl index 65788f5a..7c202819 100755 --- a/utils/exe/qless-growl +++ b/utils/exe/qless-growl @@ -6,11 +6,15 @@ require 'qless' require 'ruby-growl' require 'micro-optparse' +REDIS_HOST = ENV.fetch('REDIS_HOST', 'localhost') +REDIS_PORT = Integer(ENV.fetch('REDIS_PORT', '6379')) +REDIS_DB = Integer(ENV.fetch('REDIS_DB', '0')) + @options = Parser.new do |p| p.banner = 'This agent lets you get growl notifications for the progress of tracked jobs' p.option :growl , 'host for the growl daemon', :default => 'localhost' p.option :app , 'application name for notifications', :default => 'qless' - p.option :host , 'host:port for your qless redis instance', :default => 'localhost:6379' + p.option :host , 'host:port for your qless redis instance', :default => "#{REDIS_HOST}:#{REDIS_PORT}/#{REDIS_DB}" p.option :web , 'host:port for your qless web ui', :default => 'localhost:5678' end.process! @@ -45,7 +49,7 @@ end on.stalled { |jid| notify(jid, 'canceled') { |job| 'stalled' } } on.track { |jid| notify(jid, 'canceled') { |job| 'is being tracked' } } on.untrack { |jid| notify(jid, 'canceled') { |job| 'no longer tracked' } } - + on.completed do |jid| notify(jid, 'completed') do |job| if job @@ -59,7 +63,7 @@ end end end end - + on.failed do |jid| notify(jid, 'failed') do |job| if job @@ -69,7 +73,7 @@ end end end end - + on.popped do |jid| notify(jid, 'popped') do |job| if job @@ -79,7 +83,7 @@ end end end end - + on.put do |jid| notify(jid, 'put') do |job| if job @@ -89,9 +93,9 @@ end end end end - + puts 'Listening...' - + Signal.trap("INT") do puts 'Exiting' Process.exit(0) From 02ddf2d63ba7d6b008cdd567c4e1a65496bcf3d4 Mon Sep 17 00:00:00 2001 From: Dmitry Bochkarev Date: Wed, 7 Apr 2021 11:28:13 +0500 Subject: [PATCH 02/15] Skip js specs on ci for now --- .github/workflows/ruby.yml | 1 + spec/spec_helper.rb | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index afcd04ef..4c73f633 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -41,4 +41,5 @@ jobs: - name: Run tests env: REDIS_PORT: ${{ job.services.redis.ports['6379'] }} + SKIP_JS_SPEC: true run: bundle exec rspec diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index d0f3e142..fc300101 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -75,7 +75,7 @@ def new_redis_for_alternate_db c.before(:each, :js) do pending 'Skipping JS test because JS tests have been flaky on Travis.' - end if ENV['TRAVIS'] + end if ENV['SKIP_JS_SPEC'] end using_integration_context = false From c035dea3207580dab4b69e647565e0cfb00d49e6 Mon Sep 17 00:00:00 2001 From: Dmitry Bochkarev Date: Wed, 7 Apr 2021 11:38:52 +0500 Subject: [PATCH 03/15] The semantics of `RSpec::Core::Pending#pending` are changing in Use pending instead of skip in test RSpec 3. In RSpec 2.x, it caused the example to be skipped. In RSpec 3, the rest of the example will still be run but is expected to fail, and will be marked as a failure (rather than as pending) if the example passes. Any passed block will no longer be executed. This feature is being removed since it was semantically inconsistent, and the behaviour it offered is being made available with the other ways of marking an example pending. To keep the same skip semantics, change `pending` to `skip`. Otherwise, if you want the new RSpec 3 behavior, you can safely ignore this warning and continue to upgrade to RSpec 3 without addressing it. --- spec/integration/job_spec.rb | 2 +- spec/integration/middleware/memory_usage_monitor_spec.rb | 2 +- spec/integration/queue_spec.rb | 4 ++-- spec/integration/workers/forking_spec.rb | 2 +- spec/integration/workers/serial_spec.rb | 2 +- spec/spec_helper.rb | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/spec/integration/job_spec.rb b/spec/integration/job_spec.rb index 41264321..a923c00f 100644 --- a/spec/integration/job_spec.rb +++ b/spec/integration/job_spec.rb @@ -355,7 +355,7 @@ def self.perform(job) end it 'exposes when the next job will run' do - pending('This is implemented only in the python client') + skip('This is implemented only in the python client') queue.recur('Foo', {}, 60, jid: 'jid') nxt = client.jobs['jid'].next queue.pop diff --git a/spec/integration/middleware/memory_usage_monitor_spec.rb b/spec/integration/middleware/memory_usage_monitor_spec.rb index 1223a20c..ca8b3c49 100644 --- a/spec/integration/middleware/memory_usage_monitor_spec.rb +++ b/spec/integration/middleware/memory_usage_monitor_spec.rb @@ -126,7 +126,7 @@ def self.puts(msg) load "qless/middleware/memory_usage_monitor.rb" unless Process.respond_to?(:rusage) - pending "Could not load the rusage gem" + skip "Could not load the rusage gem" end end end diff --git a/spec/integration/queue_spec.rb b/spec/integration/queue_spec.rb index a5178f59..0a2db24c 100644 --- a/spec/integration/queue_spec.rb +++ b/spec/integration/queue_spec.rb @@ -16,7 +16,7 @@ module Qless expect(queue.jobs.send(cmd)).to eq([]) end end - + it 'provides access to job counts' do queue.put('Foo', {}) expect(queue.counts).to eq({ @@ -97,7 +97,7 @@ def remembered_queue_names end it 'can optionally stop all running jobs when pausing' do - pending('this is specific to ruby') + skip('this is specific to ruby') end it 'exposes max concurrency' do diff --git a/spec/integration/workers/forking_spec.rb b/spec/integration/workers/forking_spec.rb index 39feb073..1c2b947e 100644 --- a/spec/integration/workers/forking_spec.rb +++ b/spec/integration/workers/forking_spec.rb @@ -168,7 +168,7 @@ def self.perform(job) context 'when a job times out', :uses_threads do it 'fails the job with an error containing the job backtrace' do - pending('I do not think this is actually the desired behavior') + skip('I do not think this is actually the desired behavior') end end end diff --git a/spec/integration/workers/serial_spec.rb b/spec/integration/workers/serial_spec.rb index fd7bacc5..5ee4d13e 100644 --- a/spec/integration/workers/serial_spec.rb +++ b/spec/integration/workers/serial_spec.rb @@ -197,7 +197,7 @@ def self.perform(job) end it 'fails the job with an error containing the job backtrace' do - pending('I do not think this is actually the desired behavior') + skip('I do not think this is actually the desired behavior') end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index fc300101..4eb5d36e 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -74,7 +74,7 @@ def new_redis_for_alternate_db c.include QlessSpecHelpers c.before(:each, :js) do - pending 'Skipping JS test because JS tests have been flaky on Travis.' + skip 'Skipping JS test because JS tests have been flaky on Travis.' end if ENV['SKIP_JS_SPEC'] end From 51e16b3139bdc7c96dbd6a448f6dfe96ddb84223 Mon Sep 17 00:00:00 2001 From: Dmitry Bochkarev Date: Wed, 7 Apr 2021 11:41:21 +0500 Subject: [PATCH 04/15] Use eq false instead of be_false `be_false` is deprecated. Use `be_falsey` (for Ruby's conditional semantics) or `be false` (for exact `== false` equality) instead. Called from /home/runner/work/qless/qless/spec/unit/job_spec.rb:170:in `block (4 levels) in '. --- spec/unit/job_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/unit/job_spec.rb b/spec/unit/job_spec.rb index b3d2da22..7c63239d 100644 --- a/spec/unit/job_spec.rb +++ b/spec/unit/job_spec.rb @@ -167,7 +167,7 @@ class MyCustomError < StandardError; end job.send(meth, *args) end.to raise_error(MyCustomError) - job.state_changed?.should be_false + job.state_changed?.should eq(false) end it 'triggers before and after callbacks' do From a0122525631e2d163c0baaf6b0b8368db0f0ceda Mon Sep 17 00:00:00 2001 From: Dmitry Bochkarev Date: Wed, 7 Apr 2021 17:49:03 +0500 Subject: [PATCH 05/15] skip not core speccs --- .github/workflows/ruby.yml | 1 + spec/integration/exe/qless_stats_spec.rb | 2 +- spec/integration/middleware/memory_usage_monitor_spec.rb | 2 +- spec/spec_helper.rb | 6 +++++- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 4c73f633..eaf9c480 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -42,4 +42,5 @@ jobs: env: REDIS_PORT: ${{ job.services.redis.ports['6379'] }} SKIP_JS_SPEC: true + SKIP_NOT_CORE: true run: bundle exec rspec diff --git a/spec/integration/exe/qless_stats_spec.rb b/spec/integration/exe/qless_stats_spec.rb index a2e72014..82c3fc4d 100644 --- a/spec/integration/exe/qless_stats_spec.rb +++ b/spec/integration/exe/qless_stats_spec.rb @@ -6,7 +6,7 @@ require 'spec_helper' -describe 'qless-stats', :integration do +describe 'qless-stats', :integration, :not_core do let(:path) { './exe/qless-stats' } def run(*args) diff --git a/spec/integration/middleware/memory_usage_monitor_spec.rb b/spec/integration/middleware/memory_usage_monitor_spec.rb index ca8b3c49..c17da224 100644 --- a/spec/integration/middleware/memory_usage_monitor_spec.rb +++ b/spec/integration/middleware/memory_usage_monitor_spec.rb @@ -4,7 +4,7 @@ module Qless module Middleware - describe MemoryUsageMonitor do + describe MemoryUsageMonitor, :not_core do include_context "forking worker" mem_usage_from_other_technique = nil diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 4eb5d36e..e8255f04 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -74,8 +74,12 @@ def new_redis_for_alternate_db c.include QlessSpecHelpers c.before(:each, :js) do - skip 'Skipping JS test because JS tests have been flaky on Travis.' + skip 'Skipping JS test because JS tests have been flaky on CI.' end if ENV['SKIP_JS_SPEC'] + + c.before(:each, :not_core) do + skip 'Skipping not core test' + end if ENV['SKIP_NOT_CORE'] end using_integration_context = false From 0d4368751eab7a084a88ac9cf6e5f68ae2a5e1b3 Mon Sep 17 00:00:00 2001 From: Dmitry Bochkarev Date: Wed, 7 Apr 2021 17:51:41 +0500 Subject: [PATCH 06/15] add redis 5, 6 to matrix --- .github/workflows/ruby.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index eaf9c480..1b7457ef 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -19,7 +19,7 @@ jobs: strategy: matrix: ruby-version: ["2.5.5"] - redis-version: ["2.8"] + redis-version: ["2.8", "5", "6"] services: redis: From 31fbb64d77d7690e04f8c7a608abf2080cc26734 Mon Sep 17 00:00:00 2001 From: Dmitry Bochkarev Date: Wed, 7 Apr 2021 17:53:59 +0500 Subject: [PATCH 07/15] fix job dependencies spec --- .github/workflows/ruby.yml | 1 - spec/integration/job_spec.rb | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 1b7457ef..88721ea5 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -24,7 +24,6 @@ jobs: services: redis: image: redis:${{ matrix.redis-version }} - # Map TCP port 6379 on Docker host to a random free port on the Redis container ports: - 6379/tcp diff --git a/spec/integration/job_spec.rb b/spec/integration/job_spec.rb index a923c00f..d10b59fc 100644 --- a/spec/integration/job_spec.rb +++ b/spec/integration/job_spec.rb @@ -170,7 +170,7 @@ class NoPerformJob; end queue.put('Foo', {}, jid: 'c', depends: ['a']) expect(client.jobs['c'].dependencies).to eq(['a']) client.jobs['c'].depend('b') - expect(client.jobs['c'].dependencies).to eq(%w{a b}) + expect(client.jobs['c'].dependencies).to match_array(%w{a b}) client.jobs['c'].undepend('a') expect(client.jobs['c'].dependencies).to eq(['b']) end From 92dc35c3b91d623c7f20b2d80c900f023e3350ca Mon Sep 17 00:00:00 2001 From: Dmitry Bochkarev Date: Wed, 7 Apr 2021 17:55:54 +0500 Subject: [PATCH 08/15] add ruby 2.3, 2.6, 2.7, 3 to matrix --- .github/workflows/ruby.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 88721ea5..a6a69107 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - ruby-version: ["2.5.5"] + ruby-version: ["2.3", "2.5.5", "2.6", "2.7", "3"] redis-version: ["2.8", "5", "6"] services: From a49456ab1ffde86ed07347775d2e2c7d4b83605f Mon Sep 17 00:00:00 2001 From: Dmitry Bochkarev Date: Wed, 7 Apr 2021 18:01:28 +0500 Subject: [PATCH 09/15] fix time dependent specs --- spec/integration/events_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/integration/events_spec.rb b/spec/integration/events_spec.rb index a1cc01b4..dbf28277 100644 --- a/spec/integration/events_spec.rb +++ b/spec/integration/events_spec.rb @@ -34,12 +34,12 @@ module Qless end end end - end.tap { |t| t.join(0.01) } + end.tap { |t| t.join(0.1) } end # Wait until all the threads have sent their events def jids_for_event(event) - thread.join(0.1) + thread.join(0.3) events[event] end From 78ed44cabb2b34fd49d3badcd2310a42bb7a028d Mon Sep 17 00:00:00 2001 From: Dmitry Bochkarev Date: Thu, 8 Apr 2021 17:50:23 +0500 Subject: [PATCH 10/15] Adapt for redis gem version 4 --- .github/workflows/ruby.yml | 4 ++++ .gitignore | 1 + Gemfile | 2 ++ lib/qless.rb | 3 +++ lib/qless/job.rb | 2 +- lib/qless/middleware/redis_reconnect.rb | 4 +++- lib/qless/redis_underline_client.rb | 18 ++++++++++++++++++ lib/qless/worker/base.rb | 2 +- qless.gemspec | 2 +- spec/integration/job_spec.rb | 2 +- spec/integration/workers/forking_spec.rb | 8 ++++---- spec/integration/workers/serial_spec.rb | 12 ++++++------ spec/unit/middleware/redis_reconnect_spec.rb | 7 ++++++- 13 files changed, 51 insertions(+), 16 deletions(-) create mode 100644 lib/qless/redis_underline_client.rb diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index a6a69107..cf394f94 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -20,6 +20,7 @@ jobs: matrix: ruby-version: ["2.3", "2.5.5", "2.6", "2.7", "3"] redis-version: ["2.8", "5", "6"] + redis-gem-version: ["< 4", ">= 4"] services: redis: @@ -37,9 +38,12 @@ jobs: with: ruby-version: ${{ matrix.ruby-version }} bundler-cache: true # runs 'bundle install' and caches installed gems automatically + env: + REDIS_GEM_VERSION: ${{ matrix.redis-gem-version }} - name: Run tests env: REDIS_PORT: ${{ job.services.redis.ports['6379'] }} + REDIS_GEM_VERSION: ${{ matrix.redis-gem-version }} SKIP_JS_SPEC: true SKIP_NOT_CORE: true run: bundle exec rspec diff --git a/.gitignore b/.gitignore index d635a869..7c1b62e6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .vagrant/ +Gemfile.lock *.gem *~ .DS_STORE diff --git a/Gemfile b/Gemfile index 270a49b5..dadd622d 100644 --- a/Gemfile +++ b/Gemfile @@ -3,6 +3,8 @@ source "http://rubygems.org" # Specify your gem's dependencies in qless.gemspec gemspec +gem 'redis', ENV['REDIS_GEM_VERSION'] if ENV['REDIS_GEM_VERSION'] + group :extras do gem 'debugger', :platform => :mri_19 end diff --git a/lib/qless.rb b/lib/qless.rb index 383ddc5d..95b71867 100644 --- a/lib/qless.rb +++ b/lib/qless.rb @@ -20,6 +20,7 @@ module Qless USING_LEGACY_REDIS_VERSION = ::Redis::VERSION.to_f < 3.0 end +require 'qless/redis_underline_client' require 'qless/version' require 'qless/config' require 'qless/queue' @@ -31,6 +32,8 @@ module Qless module Qless UnsupportedRedisVersionError = Class.new(Error) + extend RedisUnderlineClient + def generate_jid SecureRandom.uuid.gsub('-', '') end diff --git a/lib/qless/job.rb b/lib/qless/job.rb index 0437bbbc..de17585a 100644 --- a/lib/qless/job.rb +++ b/lib/qless/job.rb @@ -172,7 +172,7 @@ def ttl end def reconnect_to_redis - @client.redis._client.reconnect + Qless.redis_underline_client(@client.redis).reconnect end def history diff --git a/lib/qless/middleware/redis_reconnect.rb b/lib/qless/middleware/redis_reconnect.rb index 6fb62df3..32e1e0d8 100644 --- a/lib/qless/middleware/redis_reconnect.rb +++ b/lib/qless/middleware/redis_reconnect.rb @@ -1,5 +1,7 @@ # Encoding: utf-8 +require 'qless/redis_underline_client' + module Qless module Middleware # A module for reconnecting to redis for each job @@ -15,7 +17,7 @@ def self.new(*redis_connections, &block) define_method :around_perform do |job| Array(block.call(job)).each do |redis| - redis._client.reconnect + Qless.redis_underline_client(redis).reconnect end super(job) diff --git a/lib/qless/redis_underline_client.rb b/lib/qless/redis_underline_client.rb new file mode 100644 index 00000000..49c0bf3c --- /dev/null +++ b/lib/qless/redis_underline_client.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require 'redis' + +module Qless + module RedisUnderlineClient + # https://github.com/redis/redis-rb/compare/v3.3.5...v4.0.1#diff-06572a96a58dc510037d5efa622f9bec8519bc1beab13c9f251e97e657a9d4edR19-R20 + if ::Redis::VERSION < '4' + def redis_underline_client(redis) + redis.client + end + else + def redis_underline_client(redis) + redis._client + end + end + end +end \ No newline at end of file diff --git a/lib/qless/worker/base.rb b/lib/qless/worker/base.rb index 4d8c3f84..7eada2bf 100644 --- a/lib/qless/worker/base.rb +++ b/lib/qless/worker/base.rb @@ -247,7 +247,7 @@ def current_job=(job) end def reconnect_each_client - uniq_clients.each { |client| client.redis._client.reconnect } + uniq_clients.each { |client| Qless.redis_underline_client(client.redis).reconnect } end end end diff --git a/qless.gemspec b/qless.gemspec index 948f9447..43065fc5 100644 --- a/qless.gemspec +++ b/qless.gemspec @@ -35,7 +35,7 @@ language-specific extension will also remain up to date. s.require_paths = ['lib'] s.add_dependency 'metriks', '>= 0.9' - s.add_dependency 'redis', '>= 4.0' + s.add_dependency 'redis', ['>= 3.0.7'] s.add_dependency 'rusage', '>= 0.2.0' s.add_dependency 'sinatra', '>= 1.3' s.add_dependency 'statsd-ruby', '>= 1.3' diff --git a/spec/integration/job_spec.rb b/spec/integration/job_spec.rb index d10b59fc..2febe523 100644 --- a/spec/integration/job_spec.rb +++ b/spec/integration/job_spec.rb @@ -208,7 +208,7 @@ def self.perform(job) client.config['grace-period'] = 0 client.config['heartbeat'] = 1 - queue.put('JobClass', { redis: redis._client.id }, + queue.put('JobClass', { redis: Qless.redis_underline_client(redis).id }, retries: 0, jid: 'jid') drain_worker_queues(worker) diff --git a/spec/integration/workers/forking_spec.rb b/spec/integration/workers/forking_spec.rb index 1c2b947e..04e74604 100644 --- a/spec/integration/workers/forking_spec.rb +++ b/spec/integration/workers/forking_spec.rb @@ -60,7 +60,7 @@ def self.perform(job) # Make jobs for each word words = %w{foo bar howdy} words.each do |word| - queue.put('JobClass', { redis: redis._client.id, key: key, word: word }) + queue.put('JobClass', { redis: Qless.redis_underline_client(redis).id, key: key, word: word }) end # Wait for the job to complete, and then kill the child process @@ -105,7 +105,7 @@ def self.perform(job) stub_const('JobClass', job_class) # Put a job and run it, making sure it finally succeeds - queue.put('JobClass', { redis: redis._client.id, key: key, word: :foo }, + queue.put('JobClass', { redis: Qless.redis_underline_client(redis).id, key: key, word: :foo }, retries: 5) run_worker_concurrently_with(worker) do client.redis.brpop(key, timeout: 1).should eq([key.to_s, 'foo']) @@ -129,7 +129,7 @@ def self.perform(job); end stub_const('JobClass', job_class) # Put a job in and run it - queue.put('JobClass', { redis: redis._client.id, key: key, word: :foo }) + queue.put('JobClass', { redis: Qless.redis_underline_client(redis).id, key: key, word: :foo }) run_worker_concurrently_with(worker) do client.redis.brpop(key, timeout: 1).should eq([key.to_s, 'foo']) end @@ -146,7 +146,7 @@ def self.perform(job) # Make jobs for each word 3.times do - queue.put('JobClass', { redis: redis._client.id, key: key }) + queue.put('JobClass', { redis: Qless.redis_underline_client(redis).id, key: key }) end # mixin module sends a message to a channel diff --git a/spec/integration/workers/serial_spec.rb b/spec/integration/workers/serial_spec.rb index 5ee4d13e..e870896d 100644 --- a/spec/integration/workers/serial_spec.rb +++ b/spec/integration/workers/serial_spec.rb @@ -35,7 +35,7 @@ def self.perform(job) stub_const('JobClass', job_class) # Put in a single job - queue.put('JobClass', { redis: redis._client.id, key: key, word: 'hello' }) + queue.put('JobClass', { redis: Qless.redis_underline_client(redis).id, key: key, word: 'hello' }) expect do run_jobs(worker, 1) do expect(redis.brpop(key, timeout: 1)).to eq([key.to_s, 'hello']) @@ -55,7 +55,7 @@ def self.perform(job) # Make jobs for each word words = %w{foo bar howdy} words.each do |word| - queue.put('JobClass', { redis: redis._client.id, key: key, word: word }) + queue.put('JobClass', { redis: Qless.redis_underline_client(redis).id, key: key, word: word }) end # Wait for the job to complete, and then kill the child process @@ -110,7 +110,7 @@ def self.perform(job) # Put this job into the queue and then busy-wait for the job to be # running, time it out, then make sure it eventually completes - queue.put('JobClass', { redis: redis._client.id, key: key, word: :foo }, + queue.put('JobClass', { redis: Qless.redis_underline_client(redis).id, key: key, word: :foo }, jid: 'jid') run_jobs(worker, 2) do expect(redis.brpop(key, timeout: 1)).to eq([key.to_s, 'foo']) @@ -179,9 +179,9 @@ def self.perform(job) stub_const('JobClass', job_class) # Put this job into the queue and then have the worker lose its lock - queue.put('JobClass', { redis: redis._client.id, key: key, word: :foo }, + queue.put('JobClass', { redis: Qless.redis_underline_client(redis).id, key: key, word: :foo }, priority: 10, jid: 'jid') - queue.put('JobClass', { redis: redis._client.id, key: key, word: :foo }, + queue.put('JobClass', { redis: Qless.redis_underline_client(redis).id, key: key, word: :foo }, priority: 5, jid: 'other') run_jobs(worker, 1) do @@ -220,7 +220,7 @@ def self.perform(job) stub_const('JobClass', job_class) # Put a job and run it, making sure it gets retried - queue.put('JobClass', { redis: redis._client.id, key: key, word: :foo }, + queue.put('JobClass', { redis: Qless.redis_underline_client(redis).id, key: key, word: :foo }, jid: 'jid', retries: 10) run_jobs(worker, 1) do redis.brpop(key, timeout: 1).should eq([key.to_s, 'foo']) diff --git a/spec/unit/middleware/redis_reconnect_spec.rb b/spec/unit/middleware/redis_reconnect_spec.rb index 5562de06..8e008e2c 100644 --- a/spec/unit/middleware/redis_reconnect_spec.rb +++ b/spec/unit/middleware/redis_reconnect_spec.rb @@ -34,7 +34,12 @@ def create_redis_connections(number, events) number.times.map do |i| client = instance_double('Redis::Client') client.stub(:reconnect) { events << :"reconnect_#{i}" } - instance_double('Redis', _client: client) + + if ::Redis::VERSION < '4' + instance_double('Redis', client: client) + else + instance_double('Redis', _client: client) + end end end From 3107da676990cdf04c02f2e7fc88c42230179620 Mon Sep 17 00:00:00 2001 From: Dmitry Bochkarev Date: Thu, 8 Apr 2021 18:14:40 +0500 Subject: [PATCH 11/15] fix spec --- spec/integration/workers/forking_spec.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/spec/integration/workers/forking_spec.rb b/spec/integration/workers/forking_spec.rb index 04e74604..80c14a38 100644 --- a/spec/integration/workers/forking_spec.rb +++ b/spec/integration/workers/forking_spec.rb @@ -63,12 +63,16 @@ def self.perform(job) queue.put('JobClass', { redis: Qless.redis_underline_client(redis).id, key: key, word: word }) end + worked = [] + # Wait for the job to complete, and then kill the child process run_worker_concurrently_with(worker) do words.each do |word| - client.redis.brpop(key, timeout: 1).should eq([key.to_s, word]) + worked.push(client.redis.brpop(key, timeout: 1)) end end + + expect(worked).to match_array(words.map { |word| [key.to_s, word] }) end it 'can drain its queues and exit' do From 8f3476a16e6ec20abd1e3c8a3cb6c4ed7cb62439 Mon Sep 17 00:00:00 2001 From: Dmitry Bochkarev Date: Mon, 12 Apr 2021 11:33:15 +0500 Subject: [PATCH 12/15] Add Dip for local development --- dip.yml | 26 ++++++++++++++++++++++++++ docker-compose.yml | 22 ++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 dip.yml create mode 100644 docker-compose.yml diff --git a/dip.yml b/dip.yml new file mode 100644 index 00000000..827ac59d --- /dev/null +++ b/dip.yml @@ -0,0 +1,26 @@ +version: "5.0" + +compose: + files: + - ./docker-compose.yml + +interaction: + ruby: + service: ruby + command: /bin/bash + + sh: + service: ruby + command: /bin/bash + + bundle: + service: ruby + command: bundle + + rspec: + service: ruby + command: bundle exec rspec + +provision: + - docker volume create --name bundler_data + - dip bundle install diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..a3b80a7b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,22 @@ +version: "2.1" +services: + ruby: + image: ruby:2.7 + depends_on: + - redis + environment: + - BUNDLE_PATH=/bundle/2.7 + - RACK_ENV=test + - REDIS_HOST=redis + - SKIP_JS_SPEC=true + volumes: + - .:/usr/src/qless + - bundler-data:/bundle + working_dir: /usr/src/qless + redis: + image: redis:6 + +volumes: + bundler-data: + external: + name: bundler_data From 174763bd573a84496c936254dad5c0952d7bb9a4 Mon Sep 17 00:00:00 2001 From: Dmitry Bochkarev Date: Mon, 12 Apr 2021 12:17:02 +0500 Subject: [PATCH 13/15] remove metriks --- lib/qless/middleware/metriks.rb | 45 -------------- qless.gemspec | 1 - spec/unit/middleware/metriks_spec.rb | 91 ---------------------------- 3 files changed, 137 deletions(-) delete mode 100644 lib/qless/middleware/metriks.rb delete mode 100644 spec/unit/middleware/metriks_spec.rb diff --git a/lib/qless/middleware/metriks.rb b/lib/qless/middleware/metriks.rb deleted file mode 100644 index 10d347fc..00000000 --- a/lib/qless/middleware/metriks.rb +++ /dev/null @@ -1,45 +0,0 @@ -# Encoding: utf-8 - -require 'metriks' - -module Qless - module Middleware - module Metriks - - # Tracks the time jobs take, grouping the timings by the job class. - module TimeJobsByClass - def around_perform(job) - ::Metriks.timer("qless.job-times.#{job.klass_name}").time do - super - end - end - end - - # Increments a counter each time an instance of a particular job class - # completes. - # - # Usage: - # - # Qless::Worker.class_eval do - # include Qless::Middleware::CountEvents.new( - # SomeJobClass => "event_name", - # SomeOtherJobClass => "some_other_event" - # ) - # end - class CountEvents < Module - def initialize(class_to_event_map) - module_eval do # eval the block within the module instance - define_method :around_perform do |job| - super(job) - return unless job.state == 'complete' - return unless event_name = class_to_event_map[job.klass] - - counter = ::Metriks.counter("qless.job-events.#{event_name}") - counter.increment - end - end - end - end - end - end -end diff --git a/qless.gemspec b/qless.gemspec index 43065fc5..6e3c3008 100644 --- a/qless.gemspec +++ b/qless.gemspec @@ -34,7 +34,6 @@ language-specific extension will also remain up to date. s.test_files = s.files.grep(/^(test|spec|features)\//) s.require_paths = ['lib'] - s.add_dependency 'metriks', '>= 0.9' s.add_dependency 'redis', ['>= 3.0.7'] s.add_dependency 'rusage', '>= 0.2.0' s.add_dependency 'sinatra', '>= 1.3' diff --git a/spec/unit/middleware/metriks_spec.rb b/spec/unit/middleware/metriks_spec.rb deleted file mode 100644 index d1dd187c..00000000 --- a/spec/unit/middleware/metriks_spec.rb +++ /dev/null @@ -1,91 +0,0 @@ -# Encoding: utf-8 - -require 'spec_helper' -require 'qless/middleware/metriks' -require 'qless/worker' - -module Qless - module Middleware - module Metriks - shared_context 'isolate metriks' do - before { ::Metriks::Registry.default.clear } - end - - describe TimeJobsByClass do - include_context 'isolate metriks' - - it 'tracks the time taken by the job, grouped by the class name' do - stub_const('JobABC', Class.new) - job = Qless::Job.build(double, JobABC, {}) - - base_class = Class.new do - def around_perform(job) - sleep 0.05 - end - end - - worker = Class.new(base_class) do - include TimeJobsByClass - end - - worker.new.around_perform(job) - - timer = ::Metriks.timer('qless.job-times.JobABC') - expect(timer.max).to be_within(10).percent_of(0.05) - end - end - - describe CountEvents do - include_context 'isolate metriks' - - before do - stub_const('Class1', Class.new) - stub_const('Class2', Class.new) - stub_const('Class3', Class.new) - end - - def worker(result = :complete) - base_class = Class.new do - define_method :around_perform do |job| - job.instance_variable_set(:@state, result.to_s) - end - end - - Class.new(base_class) do - include CountEvents.new( - Class1 => 'foo', - Class2 => 'bar' - ) - end - end - - def create_job_and_perform(klass, job_result = :complete) - job = Qless::Job.build(double, klass, {}) - worker(job_result).new.around_perform(job) - end - - it 'increments an event counter when a particular job completes' do - create_job_and_perform(Class1) - - expect(::Metriks.counter('qless.job-events.foo').count).to eq(1) - expect(::Metriks.counter('qless.job-events.bar').count).to eq(0) - - create_job_and_perform(Class2) - - expect(::Metriks.counter('qless.job-events.foo').count).to eq(1) - expect(::Metriks.counter('qless.job-events.bar').count).to eq(1) - end - - it 'does not increment the counter if the job fails' do - create_job_and_perform(Class1, :failed) - expect(::Metriks.counter('qless.job-events.foo').count).to eq(0) - end - - it 'does not increment a counter if it is not in the given map' do - create_job_and_perform(Class3) - expect(::Metriks::Registry.default.each.to_a).to eq([]) - end - end - end - end -end From b74c96fc496e538973871fca14f49b3f7a4df1f4 Mon Sep 17 00:00:00 2001 From: Dmitry Bochkarev Date: Mon, 12 Apr 2021 12:35:52 +0500 Subject: [PATCH 14/15] remove rusage --- benchmarks/current_memory_usage.rb | 28 --- lib/qless/middleware/memory_usage_monitor.rb | 62 ------- qless.gemspec | 1 - .../middleware/memory_usage_monitor_spec.rb | 171 ------------------ 4 files changed, 262 deletions(-) delete mode 100644 benchmarks/current_memory_usage.rb delete mode 100644 lib/qless/middleware/memory_usage_monitor.rb delete mode 100644 spec/integration/middleware/memory_usage_monitor_spec.rb diff --git a/benchmarks/current_memory_usage.rb b/benchmarks/current_memory_usage.rb deleted file mode 100644 index 230f762d..00000000 --- a/benchmarks/current_memory_usage.rb +++ /dev/null @@ -1,28 +0,0 @@ -require 'benchmark' - -n = 1000 - -Benchmark.bmbm do |x| - x.report("shelling out") do - n.times { Integer(`ps -o rss= -p #{Process.pid}`) * 1024 } - end - - x.report("rusage") do - require 'rusage' - n.times { Process.rusage } - end -end - -=begin -On my computer: - -Rehearsal ------------------------------------------------ -shelling out 0.090000 0.550000 3.650000 ( 4.051986) -rusage 0.010000 0.000000 0.010000 ( 0.002501) ---------------------------------------- total: 3.660000sec - - user system total real -shelling out 0.100000 0.550000 3.660000 ( 4.031830) -rusage 0.010000 0.000000 0.010000 ( 0.001669) -=end - diff --git a/lib/qless/middleware/memory_usage_monitor.rb b/lib/qless/middleware/memory_usage_monitor.rb deleted file mode 100644 index ad453fab..00000000 --- a/lib/qless/middleware/memory_usage_monitor.rb +++ /dev/null @@ -1,62 +0,0 @@ - -module Qless - module Middleware - # Monitors the memory usage of the Qless worker, instructing - # it to shutdown when memory exceeds the given :max_memory threshold. - class MemoryUsageMonitor < Module - def initialize(options) - max_memory = options.fetch(:max_memory) - - module_eval do - job_counter = 0 - - define_method :around_perform do |job| - job_counter += 1 - - begin - super(job) - ensure - current_mem = MemoryUsageMonitor.current_usage_in_kb - if current_mem > max_memory - log(:info, "Exiting after job #{job_counter} since current memory " \ - "(#{current_mem} KB) has exceeded max allowed memory " \ - "(#{max_memory} KB).") - shutdown - end - end - end - end - end - - SHELL_OUT_FOR_MEMORY = -> do - # Taken from: - # http://stackoverflow.com/a/4133642/29262 - Integer(`ps -o rss= -p #{Process.pid}`) - end unless defined?(SHELL_OUT_FOR_MEMORY) - - begin - require 'rusage' - rescue LoadError - warn "Could not load `rusage` gem. Falling back to shelling out " - "to get process memory usage, which is several orders of magnitude slower." - - define_singleton_method(:current_usage_in_kb, &SHELL_OUT_FOR_MEMORY) - else - memory_ratio = Process.rusage.maxrss / SHELL_OUT_FOR_MEMORY.().to_f - - if (800...1200).cover?(memory_ratio) - # OS X tends to return maxrss in Bytes. - def self.current_usage_in_kb - Process.rusage.maxrss / 1024 - end - else - # Linux tends to return maxrss in KB. - def self.current_usage_in_kb - Process.rusage.maxrss - end - end - end - end - end -end - diff --git a/qless.gemspec b/qless.gemspec index 6e3c3008..79ba4d39 100644 --- a/qless.gemspec +++ b/qless.gemspec @@ -35,7 +35,6 @@ language-specific extension will also remain up to date. s.require_paths = ['lib'] s.add_dependency 'redis', ['>= 3.0.7'] - s.add_dependency 'rusage', '>= 0.2.0' s.add_dependency 'sinatra', '>= 1.3' s.add_dependency 'statsd-ruby', '>= 1.3' s.add_dependency 'thin', '>= 1.6' diff --git a/spec/integration/middleware/memory_usage_monitor_spec.rb b/spec/integration/middleware/memory_usage_monitor_spec.rb deleted file mode 100644 index c17da224..00000000 --- a/spec/integration/middleware/memory_usage_monitor_spec.rb +++ /dev/null @@ -1,171 +0,0 @@ -require 'spec_helper' -require 'support/forking_worker_context' -require 'qless/middleware/memory_usage_monitor' - -module Qless - module Middleware - describe MemoryUsageMonitor, :not_core do - include_context "forking worker" - - mem_usage_from_other_technique = nil - - shared_examples_for "memory usage monitor" do - it 'can report the amount of memory the process is using' do - mem = MemoryUsageMonitor.current_usage_in_kb - - # We expect mem usage to be at least 10MB, but less than 10GB - expect(mem).to be > 10_000 - expect(mem).to be < 10_000_000 - - if mem_usage_from_other_technique - expect(mem).to be_within(10).percent_of(mem_usage_from_other_technique) - else - mem_usage_from_other_technique = mem - end - end - - it 'does not allow a bloated job to cause a child to permanently retain the memory bloat' do - max_memory = (MemoryUsageMonitor.current_usage_in_kb * 1.5).to_i - - [(max_memory / 2), (max_memory * 1.1).to_i, (max_memory / 2)].each do |target| - queue.put(bloated_job_class, { target: target }) - end - - job_records = [] - - worker.extend(MemoryUsageMonitor.new(max_memory: max_memory)) - - run_worker_concurrently_with(worker) do - 3.times do - _, result = client.redis.brpop('mem_usage', timeout: ENV['TRAVIS'] ? 60 : 20) - job_records << Marshal.load(result) - end - end - - # the second job should increase mem growth but be the same pid. - expect(job_records[1].pid).to eq(job_records[0].pid) - expect(job_records[1].before_mem).to be >= job_records[0].before_mem - - # the third job sould be a new process with cleared out memory - expect(job_records[2].pid).not_to eq(job_records[0].pid) - expect(job_records[2].before_mem).to be <= job_records[1].before_mem - - expect(log_io.string).to match(/Exiting after job 2/) - end - - it 'checks the memory usage even if there was an error in the job' do - failing_job_class = Class.new do - def self.perform(job) - job.client.redis.rpush('pid', Process.pid) - raise "boom" - end - end - - stub_const('FailingJobClass', failing_job_class) - 2.times { queue.put(FailingJobClass, {}) } - - worker.extend(MemoryUsageMonitor.new(max_memory: 1)) # force it to exit after every job - - pids = [] - - run_worker_concurrently_with(worker) do - 2.times do - _, result = client.redis.brpop('pid', timeout: ENV['TRAVIS'] ? 60 : 20) - pids << result - end - end - - expect(pids[1]).not_to eq(pids[0]) - end - - let(:bloated_job_class) do - bloated_job_class = Class.new do - def self.perform(job) - job_record = JobRecord.new(Process.pid, Qless::Middleware::MemoryUsageMonitor.current_usage_in_kb, nil) - job_record.after_mem = bloat_memory(job_record.before_mem, job.data.fetch("target")) - - # publish what the memory usage was before/after - job.client.redis.rpush('mem_usage', Marshal.dump(job_record)) - end - - def self.bloat_memory(original_mem, target) - current_mem = original_mem - - while current_mem < target - SecureRandom.hex( - # The * 100 is based on experimentation, taking into account - # the fact that target/current are in KB - (target - current_mem) * 100 - ).to_sym # symbols are never GC'd. - - print '.' - current_mem = Qless::Middleware::MemoryUsageMonitor.current_usage_in_kb - end - - current_mem - end - - def self.print(msg) - super if ENV['DEBUG'] - end - - def self.puts(msg) - super if ENV['DEBUG'] - end - end - - stub_const("JobRecord", Struct.new(:pid, :before_mem, :after_mem)) - - stub_const('BloatedJobClass', bloated_job_class) - end - end - - context "when the rusage gem is available" do - it_behaves_like "memory usage monitor" do - before do - load "qless/middleware/memory_usage_monitor.rb" - - unless Process.respond_to?(:rusage) - skip "Could not load the rusage gem" - end - end - end - - let(:memory_kb_according_to_ps) { MemoryUsageMonitor::SHELL_OUT_FOR_MEMORY.() } - - context "when rusage returns memory in KB (commonly on Linux)" do - before do - Process.stub(:rusage) { double(maxrss: memory_kb_according_to_ps) } - load "qless/middleware/memory_usage_monitor.rb" - end - - it 'returns the memory in KB' do - expect(MemoryUsageMonitor.current_usage_in_kb).to be_within(1).percent_of(memory_kb_according_to_ps) - end - end - - context "when rusage returns memory in bytes (commonly on OS X)" do - before do - Process.stub(:rusage) { double(maxrss: memory_kb_according_to_ps * 1024) } - load "qless/middleware/memory_usage_monitor.rb" - end - - it 'returns the memory in KB' do - expect(MemoryUsageMonitor.current_usage_in_kb).to be_within(1).percent_of(memory_kb_according_to_ps) - end - end - end - - context "when the rusage gem is not available" do - it_behaves_like "memory usage monitor" do - before do - MemoryUsageMonitor.stub(:warn) - MemoryUsageMonitor.stub(:require).and_raise(LoadError) - load "qless/middleware/memory_usage_monitor.rb" - end - end - end - end - end -end - From de924904f05290c82c4e8f8adb0533863ef625f2 Mon Sep 17 00:00:00 2001 From: Dmitry Bochkarev Date: Wed, 28 Apr 2021 14:00:42 +0500 Subject: [PATCH 15/15] remove support redis-client <4 --- .github/workflows/ruby.yml | 4 ---- Gemfile | 2 -- lib/qless.rb | 3 --- lib/qless/job.rb | 2 +- lib/qless/middleware/redis_reconnect.rb | 4 +--- lib/qless/redis_underline_client.rb | 18 ------------------ lib/qless/worker/base.rb | 2 +- qless.gemspec | 2 +- spec/integration/job_spec.rb | 2 +- spec/integration/workers/forking_spec.rb | 8 ++++---- spec/integration/workers/serial_spec.rb | 12 ++++++------ spec/unit/middleware/redis_reconnect_spec.rb | 6 +----- 12 files changed, 16 insertions(+), 49 deletions(-) delete mode 100644 lib/qless/redis_underline_client.rb diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index cf394f94..a6a69107 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -20,7 +20,6 @@ jobs: matrix: ruby-version: ["2.3", "2.5.5", "2.6", "2.7", "3"] redis-version: ["2.8", "5", "6"] - redis-gem-version: ["< 4", ">= 4"] services: redis: @@ -38,12 +37,9 @@ jobs: with: ruby-version: ${{ matrix.ruby-version }} bundler-cache: true # runs 'bundle install' and caches installed gems automatically - env: - REDIS_GEM_VERSION: ${{ matrix.redis-gem-version }} - name: Run tests env: REDIS_PORT: ${{ job.services.redis.ports['6379'] }} - REDIS_GEM_VERSION: ${{ matrix.redis-gem-version }} SKIP_JS_SPEC: true SKIP_NOT_CORE: true run: bundle exec rspec diff --git a/Gemfile b/Gemfile index dadd622d..270a49b5 100644 --- a/Gemfile +++ b/Gemfile @@ -3,8 +3,6 @@ source "http://rubygems.org" # Specify your gem's dependencies in qless.gemspec gemspec -gem 'redis', ENV['REDIS_GEM_VERSION'] if ENV['REDIS_GEM_VERSION'] - group :extras do gem 'debugger', :platform => :mri_19 end diff --git a/lib/qless.rb b/lib/qless.rb index 95b71867..383ddc5d 100644 --- a/lib/qless.rb +++ b/lib/qless.rb @@ -20,7 +20,6 @@ module Qless USING_LEGACY_REDIS_VERSION = ::Redis::VERSION.to_f < 3.0 end -require 'qless/redis_underline_client' require 'qless/version' require 'qless/config' require 'qless/queue' @@ -32,8 +31,6 @@ module Qless module Qless UnsupportedRedisVersionError = Class.new(Error) - extend RedisUnderlineClient - def generate_jid SecureRandom.uuid.gsub('-', '') end diff --git a/lib/qless/job.rb b/lib/qless/job.rb index de17585a..0437bbbc 100644 --- a/lib/qless/job.rb +++ b/lib/qless/job.rb @@ -172,7 +172,7 @@ def ttl end def reconnect_to_redis - Qless.redis_underline_client(@client.redis).reconnect + @client.redis._client.reconnect end def history diff --git a/lib/qless/middleware/redis_reconnect.rb b/lib/qless/middleware/redis_reconnect.rb index 32e1e0d8..6fb62df3 100644 --- a/lib/qless/middleware/redis_reconnect.rb +++ b/lib/qless/middleware/redis_reconnect.rb @@ -1,7 +1,5 @@ # Encoding: utf-8 -require 'qless/redis_underline_client' - module Qless module Middleware # A module for reconnecting to redis for each job @@ -17,7 +15,7 @@ def self.new(*redis_connections, &block) define_method :around_perform do |job| Array(block.call(job)).each do |redis| - Qless.redis_underline_client(redis).reconnect + redis._client.reconnect end super(job) diff --git a/lib/qless/redis_underline_client.rb b/lib/qless/redis_underline_client.rb deleted file mode 100644 index 49c0bf3c..00000000 --- a/lib/qless/redis_underline_client.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -require 'redis' - -module Qless - module RedisUnderlineClient - # https://github.com/redis/redis-rb/compare/v3.3.5...v4.0.1#diff-06572a96a58dc510037d5efa622f9bec8519bc1beab13c9f251e97e657a9d4edR19-R20 - if ::Redis::VERSION < '4' - def redis_underline_client(redis) - redis.client - end - else - def redis_underline_client(redis) - redis._client - end - end - end -end \ No newline at end of file diff --git a/lib/qless/worker/base.rb b/lib/qless/worker/base.rb index 7eada2bf..4d8c3f84 100644 --- a/lib/qless/worker/base.rb +++ b/lib/qless/worker/base.rb @@ -247,7 +247,7 @@ def current_job=(job) end def reconnect_each_client - uniq_clients.each { |client| Qless.redis_underline_client(client.redis).reconnect } + uniq_clients.each { |client| client.redis._client.reconnect } end end end diff --git a/qless.gemspec b/qless.gemspec index 79ba4d39..b9e69137 100644 --- a/qless.gemspec +++ b/qless.gemspec @@ -34,7 +34,7 @@ language-specific extension will also remain up to date. s.test_files = s.files.grep(/^(test|spec|features)\//) s.require_paths = ['lib'] - s.add_dependency 'redis', ['>= 3.0.7'] + s.add_dependency 'redis', ['>= 4.0'] s.add_dependency 'sinatra', '>= 1.3' s.add_dependency 'statsd-ruby', '>= 1.3' s.add_dependency 'thin', '>= 1.6' diff --git a/spec/integration/job_spec.rb b/spec/integration/job_spec.rb index 2febe523..d10b59fc 100644 --- a/spec/integration/job_spec.rb +++ b/spec/integration/job_spec.rb @@ -208,7 +208,7 @@ def self.perform(job) client.config['grace-period'] = 0 client.config['heartbeat'] = 1 - queue.put('JobClass', { redis: Qless.redis_underline_client(redis).id }, + queue.put('JobClass', { redis: redis._client.id }, retries: 0, jid: 'jid') drain_worker_queues(worker) diff --git a/spec/integration/workers/forking_spec.rb b/spec/integration/workers/forking_spec.rb index 80c14a38..9ea80907 100644 --- a/spec/integration/workers/forking_spec.rb +++ b/spec/integration/workers/forking_spec.rb @@ -60,7 +60,7 @@ def self.perform(job) # Make jobs for each word words = %w{foo bar howdy} words.each do |word| - queue.put('JobClass', { redis: Qless.redis_underline_client(redis).id, key: key, word: word }) + queue.put('JobClass', { redis: redis._client.id, key: key, word: word }) end worked = [] @@ -109,7 +109,7 @@ def self.perform(job) stub_const('JobClass', job_class) # Put a job and run it, making sure it finally succeeds - queue.put('JobClass', { redis: Qless.redis_underline_client(redis).id, key: key, word: :foo }, + queue.put('JobClass', { redis: redis._client.id, key: key, word: :foo }, retries: 5) run_worker_concurrently_with(worker) do client.redis.brpop(key, timeout: 1).should eq([key.to_s, 'foo']) @@ -133,7 +133,7 @@ def self.perform(job); end stub_const('JobClass', job_class) # Put a job in and run it - queue.put('JobClass', { redis: Qless.redis_underline_client(redis).id, key: key, word: :foo }) + queue.put('JobClass', { redis: redis._client.id, key: key, word: :foo }) run_worker_concurrently_with(worker) do client.redis.brpop(key, timeout: 1).should eq([key.to_s, 'foo']) end @@ -150,7 +150,7 @@ def self.perform(job) # Make jobs for each word 3.times do - queue.put('JobClass', { redis: Qless.redis_underline_client(redis).id, key: key }) + queue.put('JobClass', { redis: redis._client.id, key: key }) end # mixin module sends a message to a channel diff --git a/spec/integration/workers/serial_spec.rb b/spec/integration/workers/serial_spec.rb index e870896d..5ee4d13e 100644 --- a/spec/integration/workers/serial_spec.rb +++ b/spec/integration/workers/serial_spec.rb @@ -35,7 +35,7 @@ def self.perform(job) stub_const('JobClass', job_class) # Put in a single job - queue.put('JobClass', { redis: Qless.redis_underline_client(redis).id, key: key, word: 'hello' }) + queue.put('JobClass', { redis: redis._client.id, key: key, word: 'hello' }) expect do run_jobs(worker, 1) do expect(redis.brpop(key, timeout: 1)).to eq([key.to_s, 'hello']) @@ -55,7 +55,7 @@ def self.perform(job) # Make jobs for each word words = %w{foo bar howdy} words.each do |word| - queue.put('JobClass', { redis: Qless.redis_underline_client(redis).id, key: key, word: word }) + queue.put('JobClass', { redis: redis._client.id, key: key, word: word }) end # Wait for the job to complete, and then kill the child process @@ -110,7 +110,7 @@ def self.perform(job) # Put this job into the queue and then busy-wait for the job to be # running, time it out, then make sure it eventually completes - queue.put('JobClass', { redis: Qless.redis_underline_client(redis).id, key: key, word: :foo }, + queue.put('JobClass', { redis: redis._client.id, key: key, word: :foo }, jid: 'jid') run_jobs(worker, 2) do expect(redis.brpop(key, timeout: 1)).to eq([key.to_s, 'foo']) @@ -179,9 +179,9 @@ def self.perform(job) stub_const('JobClass', job_class) # Put this job into the queue and then have the worker lose its lock - queue.put('JobClass', { redis: Qless.redis_underline_client(redis).id, key: key, word: :foo }, + queue.put('JobClass', { redis: redis._client.id, key: key, word: :foo }, priority: 10, jid: 'jid') - queue.put('JobClass', { redis: Qless.redis_underline_client(redis).id, key: key, word: :foo }, + queue.put('JobClass', { redis: redis._client.id, key: key, word: :foo }, priority: 5, jid: 'other') run_jobs(worker, 1) do @@ -220,7 +220,7 @@ def self.perform(job) stub_const('JobClass', job_class) # Put a job and run it, making sure it gets retried - queue.put('JobClass', { redis: Qless.redis_underline_client(redis).id, key: key, word: :foo }, + queue.put('JobClass', { redis: redis._client.id, key: key, word: :foo }, jid: 'jid', retries: 10) run_jobs(worker, 1) do redis.brpop(key, timeout: 1).should eq([key.to_s, 'foo']) diff --git a/spec/unit/middleware/redis_reconnect_spec.rb b/spec/unit/middleware/redis_reconnect_spec.rb index 8e008e2c..9de52b99 100644 --- a/spec/unit/middleware/redis_reconnect_spec.rb +++ b/spec/unit/middleware/redis_reconnect_spec.rb @@ -35,11 +35,7 @@ def create_redis_connections(number, events) client = instance_double('Redis::Client') client.stub(:reconnect) { events << :"reconnect_#{i}" } - if ::Redis::VERSION < '4' - instance_double('Redis', client: client) - else - instance_double('Redis', _client: client) - end + instance_double('Redis', _client: client) end end