diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 72a8b71d..ccb63217 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -1,6 +1,6 @@ # This configuration was generated by # `rubocop --auto-gen-config --auto-gen-only-exclude --exclude-limit 10000` -# on 2022-10-18 21:05:49 UTC using RuboCop version 1.36.0. +# on 2024-07-18 19:32:08 UTC using RuboCop version 1.36.0. # The point is for the user to remove these configuration records # one by one as the offenses are removed from the code base. # Note that changes in the inspected code, or installation of new @@ -28,7 +28,7 @@ Layout/ElseAlignment: Exclude: - 'config/initializers/patch_failure_app.rb' -# Offense count: 49 +# Offense count: 50 # This cop supports safe autocorrection (--autocorrect). Layout/EmptyLineAfterGuardClause: Exclude: @@ -148,7 +148,7 @@ Layout/ExtraSpacing: - 'config/deploy.rb' - 'config/routes.rb' -# Offense count: 93 +# Offense count: 94 # This cop supports safe autocorrection (--autocorrect). # Configuration parameters: AllowMultipleStyles, EnforcedHashRocketStyle, EnforcedColonStyle, EnforcedLastArgumentHashStyle. # SupportedHashRocketStyles: key, separator, table @@ -326,12 +326,6 @@ Lint/MixedRegexpCaptureTypes: Exclude: - 'app/models/deposit.rb' -# Offense count: 1 -# This cop supports unsafe autocorrection (--autocorrect-all). -Lint/NonDeterministicRequireOrder: - Exclude: - - 'spec/rails_helper.rb' - # Offense count: 4 # This cop supports safe autocorrection (--autocorrect). Lint/RedundantCopDisableDirective: @@ -378,7 +372,7 @@ Lint/UselessAssignment: - 'app/models/statistic.rb' - 'lib/tasks/load.rake' -# Offense count: 43 +# Offense count: 44 # Configuration parameters: AllowedMethods, AllowedPatterns, IgnoredMethods, CountRepeatedAttributes, Max. Metrics/AbcSize: Exclude: @@ -395,7 +389,6 @@ Metrics/AbcSize: - 'app/forms/email_author_reports_form.rb' - 'app/forms/request_agreements_form.rb' - 'app/forms/usage_statistics_reports_form.rb' - - 'app/helpers/application_helper.rb' - 'app/helpers/catalog_helper.rb' - 'app/helpers/homepage_helper.rb' - 'app/helpers/use_and_reproduction_helper.rb' @@ -414,6 +407,7 @@ Metrics/AbcSize: - 'lib/academic_commons/indexer.rb' - 'lib/academic_commons/metrics/author_affiliation_report.rb' - 'lib/academic_commons/metrics/output.rb' + - 'lib/tasks/resque.rake' # Offense count: 12 # Configuration parameters: CountComments, Max, CountAsOne, ExcludedMethods, AllowedMethods, AllowedPatterns, IgnoredMethods, inherit_mode. @@ -438,7 +432,7 @@ Metrics/ClassLength: - 'app/models/solr_document.rb' - 'lib/academic_commons/metrics/usage_statistics.rb' -# Offense count: 9 +# Offense count: 10 # Configuration parameters: AllowedMethods, AllowedPatterns, IgnoredMethods, Max. Metrics/CyclomaticComplexity: Exclude: @@ -451,8 +445,9 @@ Metrics/CyclomaticComplexity: - 'app/models/statistic.rb' - 'app/models/user.rb' - 'lib/academic_commons/metrics/author_affiliation_report.rb' + - 'lib/tasks/resque.rake' -# Offense count: 42 +# Offense count: 44 # Configuration parameters: CountComments, Max, CountAsOne, ExcludedMethods, AllowedMethods, AllowedPatterns, IgnoredMethods. Metrics/MethodLength: Exclude: @@ -489,8 +484,9 @@ Metrics/MethodLength: - 'lib/academic_commons/metrics/author_affiliation_report.rb' - 'lib/academic_commons/metrics/output.rb' - 'lib/academic_commons/metrics/usage_statistics.rb' + - 'lib/tasks/resque.rake' -# Offense count: 9 +# Offense count: 10 # Configuration parameters: AllowedMethods, AllowedPatterns, IgnoredMethods, Max. Metrics/PerceivedComplexity: Exclude: @@ -503,6 +499,7 @@ Metrics/PerceivedComplexity: - 'app/models/statistic.rb' - 'app/models/user.rb' - 'lib/academic_commons/metrics/author_affiliation_report.rb' + - 'lib/tasks/resque.rake' # Offense count: 10 # Configuration parameters: EnforcedStyleForLeadingUnderscores. @@ -581,7 +578,7 @@ RSpec/AnyInstance: - 'spec/controllers/users/omniauth_callbacks_controller_spec.rb' - 'spec/support/ldap_mock.rb' -# Offense count: 15 +# Offense count: 16 RSpec/Capybara/VisibilityMatcher: Exclude: - 'spec/features/item_page_spec.rb' @@ -594,7 +591,7 @@ RSpec/ContextMethod: - 'spec/models/email_preference_spec.rb' - 'spec/models/user_spec.rb' -# Offense count: 98 +# Offense count: 99 # Configuration parameters: Prefixes, AllowedPatterns. # Prefixes: when, with, without RSpec/ContextWording: @@ -676,7 +673,7 @@ RSpec/EmptyLineAfterExampleGroup: - 'spec/helpers/use_and_reproduction_display_helper_spec.rb' - 'spec/models/featured_search_spec.rb' -# Offense count: 43 +# Offense count: 45 # This cop supports safe autocorrection (--autocorrect). RSpec/EmptyLineAfterFinalLet: Exclude: @@ -811,7 +808,7 @@ RSpec/MultipleExpectations: - 'spec/models/statistic_spec.rb' - 'spec/support/authorization_for_controller_tests.rb' -# Offense count: 17 +# Offense count: 18 # Configuration parameters: IgnoreSharedExamples. RSpec/NamedSubject: Exclude: @@ -820,7 +817,7 @@ RSpec/NamedSubject: - 'spec/academic_commons/metrics/usage_statistics_spec.rb' - 'spec/controllers/solr_documents_controller_spec.rb' -# Offense count: 13 +# Offense count: 14 # Configuration parameters: Max, AllowedGroups. RSpec/NestedGroups: Exclude: @@ -886,7 +883,7 @@ Rails/DynamicFindBy: - 'app/models/user.rb' - 'lib/academic_commons/metrics/author_affiliation_report.rb' -# Offense count: 16 +# Offense count: 19 # Configuration parameters: EnforcedStyle. # SupportedStyles: slashes, arguments Rails/FilePath: @@ -899,6 +896,7 @@ Rails/FilePath: - 'lib/academic_commons/indexer.rb' - 'lib/tasks/ac/bots.rake' - 'lib/tasks/load.rake' + - 'lib/tasks/resque.rake' - 'spec/factories/deposit.rb' - 'spec/support/test_log.rb' - 'spec/tasks/sitemap_spec.rb' @@ -968,7 +966,7 @@ Rails/Present: Exclude: - 'app/models/notification.rb' -# Offense count: 2 +# Offense count: 1 # This cop supports unsafe autocorrection (--autocorrect-all). # Configuration parameters: Include. # Include: **/Rakefile, **/*.rake @@ -1008,19 +1006,13 @@ Security/Open: - 'lib/tasks/ac/index.rake' - 'lib/tasks/ac/reindex.rake' -# Offense count: 1 -# This cop supports unsafe autocorrection (--autocorrect-all). -Security/YAMLLoad: - Exclude: - - 'config/initializers/resque.rb' - # Offense count: 1 # This cop supports safe autocorrection (--autocorrect). Style/BlockComments: Exclude: - 'spec/spec_helper.rb' -# Offense count: 1 +# Offense count: 2 # This cop supports safe autocorrection (--autocorrect). # Configuration parameters: EnforcedStyle, ProceduralMethods, FunctionalMethods, AllowedMethods, AllowedPatterns, IgnoredMethods, AllowBracesOnProceduralOneLiners, BracesRequiredMethods. # SupportedStyles: line_count_based, semantic, braces_for_chaining, always_braces @@ -1030,6 +1022,7 @@ Style/BlockComments: Style/BlockDelimiters: Exclude: - 'app/presenters/usage_statistics_presenter.rb' + - 'lib/tasks/resque.rake' # Offense count: 1 # This cop supports unsafe autocorrection (--autocorrect-all). @@ -1517,7 +1510,7 @@ Style/StringConcatenation: - 'lib/hyacinth.rb' - 'lib/tasks/load.rake' -# Offense count: 106 +# Offense count: 108 # This cop supports safe autocorrection (--autocorrect). # Configuration parameters: EnforcedStyle, ConsistentQuotesInMultiline. # SupportedStyles: single_quotes, double_quotes @@ -1561,14 +1554,14 @@ Style/StringLiteralsInInterpolation: - 'app/helpers/catalog_helper.rb' - 'lib/academic_commons/indexer.rb' -# Offense count: 13 +# Offense count: 16 # This cop supports unsafe autocorrection (--autocorrect-all). # Configuration parameters: AllowMethodsWithArguments, AllowedMethods, AllowedPatterns, IgnoredMethods, AllowComments. # AllowedMethods: respond_to, define_method Style/SymbolProc: Exclude: - - 'app/api/v1/entities/short_record.rb' - 'app/api/v1/entities/full_record.rb' + - 'app/api/v1/entities/short_record.rb' - 'app/models/statistic.rb' # Offense count: 1 @@ -1610,7 +1603,7 @@ Style/WhileUntilDo: - 'lib/academic_commons/metrics/author_affiliation_report.rb' - 'lib/tasks/ac.rake' -# Offense count: 164 +# Offense count: 174 # This cop supports safe autocorrection (--autocorrect). # Configuration parameters: Max, AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, AllowedPatterns, IgnoredPatterns. # URISchemes: http, https @@ -1647,6 +1640,7 @@ Layout/LineLength: - 'lib/academic_commons/indexer.rb' - 'lib/academic_commons/metrics/usage_statistics.rb' - 'lib/hyacinth.rb' + - 'lib/tasks/resque.rake' - 'spec/academic_commons/featured_searches_spec.rb' - 'spec/academic_commons/metrics/author_affiliation_report_spec.rb' - 'spec/academic_commons/metrics/usage_statistics_spec.rb' diff --git a/Gemfile b/Gemfile index daddd294..b7ef6b41 100644 --- a/Gemfile +++ b/Gemfile @@ -34,7 +34,7 @@ gem 'nokogiri', '~> 1.15.2' gem 'okcomputer' gem 'premailer-rails' gem 'rainbow' -gem 'resque', '~> 1.27' +gem 'resque', '~> 2.0' gem 'rinku' gem 'rsolr-ext' gem 'rubyzip', require: 'zip' @@ -49,8 +49,8 @@ gem 'will_paginate' gem 'wowza-secure_token' # Database -gem 'mysql2' -gem 'sqlite3', '>= 1.3.5' +gem 'mysql2', '>= 0.5.6' +gem 'sqlite3', '~> 1.4' group :development do gem 'listen' diff --git a/Gemfile.lock b/Gemfile.lock index 6690d091..6f1fcaf4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -330,7 +330,7 @@ GEM ruby2_keywords (~> 0.0.1) mustermann-grape (1.0.2) mustermann (>= 1.0.0) - mysql2 (0.5.4) + mysql2 (0.5.6) net-imap (0.4.0) date net-protocol @@ -391,7 +391,7 @@ GEM rack (2.2.4) rack-accept (0.4.5) rack (>= 0.4) - rack-protection (2.2.0) + rack-protection (2.2.1) rack rack-proxy (0.7.2) rack @@ -448,18 +448,17 @@ GEM rdf-xsd (1.99.0) rdf (~> 1.99) redis (4.6.0) - redis-namespace (1.8.2) - redis (>= 3.0.4) + redis-namespace (1.11.0) + redis (>= 4) regexp_parser (2.6.0) responders (3.0.1) actionpack (>= 5.0) railties (>= 5.0) - resque (1.27.4) + resque (2.6.0) mono_logger (~> 1.0) multi_json (~> 1.0) - redis-namespace (~> 1.3) + redis-namespace (~> 1.6) sinatra (>= 0.9.2) - vegas (~> 0.1.2) resque-scheduler (4.10.0) mono_logger (~> 1.0) redis (>= 3.3) @@ -557,10 +556,10 @@ GEM simplecov_json_formatter (~> 0.1) simplecov-html (0.12.3) simplecov_json_formatter (0.1.4) - sinatra (2.2.0) + sinatra (2.2.1) mustermann (~> 1.0) rack (~> 2.2) - rack-protection (= 2.2.0) + rack-protection (= 2.2.1) tilt (~> 2.0) sitemap_generator (6.2.1) builder (~> 3.0) @@ -583,7 +582,9 @@ GEM actionpack (>= 5.2) activesupport (>= 5.2) sprockets (>= 3.0.0) - sqlite3 (1.4.2) + sqlite3 (1.7.3) + mini_portile2 (~> 2.8.0) + sqlite3 (1.7.3-x86_64-darwin) sshkit (1.21.2) net-scp (>= 1.1.2) net-ssh (>= 2.8.0) @@ -606,8 +607,6 @@ GEM unf_ext (0.0.8.2) unicode (0.4.4.4) unicode-display_width (2.3.0) - vegas (0.1.11) - rack (>= 1.0.0) view_component (2.71.0) activesupport (>= 5.0.0, < 8.0) concurrent-ruby (~> 1.0) @@ -680,7 +679,7 @@ DEPENDENCIES listen mail (~> 2.8) multipart-post (~> 2.0.0) - mysql2 + mysql2 (>= 0.5.6) net-scp (~> 4.0.0) net-ssh (~> 7.2.0) nokogiri (~> 1.15.2) @@ -689,7 +688,7 @@ DEPENDENCIES puma (~> 5.2) rails (= 6.0.4) rainbow - resque (~> 1.27) + resque (~> 2.0) rinku rsolr-ext rspec-its @@ -701,7 +700,7 @@ DEPENDENCIES sitemap_generator solr_wrapper (~> 4.0) spring - sqlite3 (>= 1.3.5) + sqlite3 (~> 1.4) turbolinks uglifier unicode diff --git a/app/jobs/sword_deposit_job.rb b/app/jobs/sword_deposit_job.rb index 0114eda1..295207cf 100644 --- a/app/jobs/sword_deposit_job.rb +++ b/app/jobs/sword_deposit_job.rb @@ -11,7 +11,7 @@ def perform(deposit) # Send request to SWORD begin - response = HTTP.timeout(write: 60, connect: 60, read: 60) + response = HTTP.timeout(write: 60, connect: 60, read: 90) .basic_auth(user: credentials['user'], pass: credentials['password']) .headers(content_type: 'application/zip') .post(credentials['url'], body: deposit.sword_zip) diff --git a/config/deploy.rb b/config/deploy.rb index f6c7f122..d5a8b053 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -1,4 +1,4 @@ -lock '3.17.0' +lock '3.17.3' set :instance, 'ac' set :application, 'academiccommons' diff --git a/config/deploy/prod.rb b/config/deploy/prod.rb index 7cfa1b85..8e494fd2 100644 --- a/config/deploy/prod.rb +++ b/config/deploy/prod.rb @@ -2,7 +2,8 @@ role :resque_worker, "ac-rails-#{fetch(:stage)}1.cul.columbia.edu" role :resque_scheduler, "ac-rails-#{fetch(:stage)}1.cul.columbia.edu" -set :workers, { '*' => 6 } +set :workers, + YAML.load_file(File.expand_path('../resque.yml', __dir__), aliases: true)[fetch(:stage).to_s]['workers'] # We default to storing PID files in a tmp/pids folder in your shared path. # set :resque_pid_path, -> { File.join(shared_path, 'tmp', 'pids') } diff --git a/config/deploy/test.rb b/config/deploy/test.rb index d1052393..e5f4db4e 100644 --- a/config/deploy/test.rb +++ b/config/deploy/test.rb @@ -2,7 +2,8 @@ role :resque_worker, "ac-rails-#{fetch(:stage)}1.cul.columbia.edu" role :resque_scheduler, "ac-rails-#{fetch(:stage)}1.cul.columbia.edu" -set :workers, { '*' => 4 } +set :workers, + YAML.load_file(File.expand_path('../resque.yml', __dir__), aliases: true)[fetch(:stage).to_s]['workers'] # We default to storing PID files in a tmp/pids folder in your shared path. # set :resque_pid_path, -> { File.join(shared_path, 'tmp', 'pids') } diff --git a/config/schedule.rb b/config/schedule.rb index e310c6f9..933c2e41 100644 --- a/config/schedule.rb +++ b/config/schedule.rb @@ -27,3 +27,8 @@ every :weekday, at: '6pm' do rake 'sitemap:create', email_subject: 'Sitemap generation' end + +# Restart resque workers daily. +every :day, at: '12am' do + rake 'resque:restart_workers', email_subject: 'Resque workers restart' +end diff --git a/lib/tasks/resque.rake b/lib/tasks/resque.rake new file mode 100644 index 00000000..ab907e2f --- /dev/null +++ b/lib/tasks/resque.rake @@ -0,0 +1,110 @@ +# frozen_string_literal: true + +# Enable resque tasks and ensure that setup and work tasks have access to the environment +require 'open3' +require 'resque/tasks' +task 'resque:setup' => :environment +task 'resque:work' => :environment + +MAX_WAIT_TIME_TO_KILL_WORKERS = 120 +PIDFILE_PATH = 'tmp/pids' + +namespace :resque do + desc 'Stop current workers and start new workers' + task restart_workers: :environment do + Rake::Task['resque:stop_workers'].invoke + Rake::Task['resque:start_workers'].invoke + end + + desc 'Stop running workers' + task stop_workers: :environment do + stop_workers + end + + desc 'Start workers' + task start_workers: :environment do + start_workers(Rails.application.config_for(:resque)) + end + + def pid_files + Dir.glob(File.join(PIDFILE_PATH, 'resque_work*.pid')).select { |path| + File.file?(path) && !File.zero?(path) + } + end + + def clear_pid_files + pid_files.each do |pidfile| + File.delete(pidfile) + end + end + + def read_pids + pid_files.map do |file_path| + File.open(file_path, &:gets).chomp + end + end + + def stop_workers + pids = read_pids + + if pids.empty? + puts 'No known workers to kill.' + else + # First tell workers to stop accepting new work by sending USR2 signal + puts "\nTelling workers to finish current jobs, but not process any new jobs..." + syscmd = "kill -s USR2 #{pids.join(' ')}" + puts "$ #{syscmd}" + `#{syscmd}` + puts "\n" + puts 'Waiting for workers to finish current jobs...' + start_time = Time.zone.now + while (Time.zone.now - start_time) < MAX_WAIT_TIME_TO_KILL_WORKERS + sleep 1 + num_workers_working = Resque.workers.count(&:working?) + puts "#{num_workers_working} workers still working..." + break if num_workers_working.zero? + end + puts "\n" + if Resque.workers.count(&:working?).positive? + puts "Workers are still running, but wait time of #{MAX_WAIT_TIME_TO_KILL_WORKERS} has been exceeded. Sending QUIT signal anyway." + else + puts 'Workers are no longer processing any jobs. Safely sending QUIT signal...' + end + syscmd = "kill -s QUIT #{pids.join(' ')}" + puts "$ #{syscmd}" + `#{syscmd}` + clear_pid_files + puts "\n" + puts 'Workers have been shut down.' + end + + # Unregister old workers + Resque.workers.each(&:unregister_worker) + end + + # Start a worker with proper env vars and output redirection + def start_workers(resque_config) + polling_interval = resque_config[:polling_interval] + worker_config = resque_config.fetch(:workers, {}) + total_workers = 0 + worker_info_string = worker_config.map { |queues, count| + total_workers += count + " [ #{queues} ] => #{count} #{count == 1 ? 'worker' : 'workers'}" + }.join("\n") + interval = polling_interval || '5' + puts "Starting #{total_workers} #{total_workers == 1 ? 'worker' : 'workers'} with a polling interval of #{interval} seconds:\n" + worker_info_string + err = Rails.root.join('log', 'resque.log').to_s + out = Rails.root.join('log', 'resque.log').to_s + rails_env = ENV['RAILS_ENV'] + + worker_config.each do |queues, count| + queues = queues.to_s + count.times do |index| + number = index + 1 + pidfile = Rails.root.join('tmp/pids/', "resque_work_#{number}.pid").to_s + _stdout_str, _stderr_str, status = Open3.capture3("RAILS_ENV=#{rails_env} QUEUE=\"#{queues}\" PIDFILE=#{pidfile} BACKGROUND=yes VERBOSE=1 INTERVAL=#{interval} rake resque:work >> #{out} 2>> #{err}") + puts "Worker #{number} started, status: #{status}" + end + end + end +end