add sidekiq for job processing

.ebextensions/sidekiq.config

@@ -0,0 +1,66 @@
+# Sidekiq interaction and startup script
+commands:
+ create_post_dir:
+ command: "mkdir -p /opt/elasticbeanstalk/hooks/appdeploy/post"
+ ignoreErrors: true
+files:
+ "/opt/elasticbeanstalk/hooks/appdeploy/post/50_restart_sidekiq.sh":
+ mode: "000755"
+ owner: root
+ group: root
+ content: |
+ #!/usr/bin/env bash
+ . /opt/elasticbeanstalk/support/envvars
+
+ EB_APP_DEPLOY_DIR=$(/opt/elasticbeanstalk/bin/get-config container -k app_deploy_dir)
+ EB_APP_PID_DIR=$(/opt/elasticbeanstalk/bin/get-config container -k app_pid_dir)
+ EB_APP_USER=$(/opt/elasticbeanstalk/bin/get-config container -k app_user)
+ EB_SCRIPT_DIR=$(/opt/elasticbeanstalk/bin/get-config container -k script_dir)
+ EB_SUPPORT_DIR=$(/opt/elasticbeanstalk/bin/get-config container -k support_dir)
+
+ . $EB_SUPPORT_DIR/envvars
+ . $EB_SCRIPT_DIR/use-app-ruby.sh
+
+ SIDEKIQ_PID=$EB_APP_PID_DIR/sidekiq.pid
+ SIDEKIQ_CONFIG=$EB_APP_DEPLOY_DIR/config/sidekiq.yml
+ SIDEKIQ_LOG=$EB_APP_DEPLOY_DIR/log/sidekiq.log
+
+ cd $EB_APP_DEPLOY_DIR
+
+ if [ -f $SIDEKIQ_PID ]
+ then
+ su -s /bin/bash -c "kill -TERM `cat $SIDEKIQ_PID`" $EB_APP_USER
+ su -s /bin/bash -c "rm -rf $SIDEKIQ_PID" $EB_APP_USER
+ fi
+
+ . /opt/elasticbeanstalk/support/envvars.d/sysenv
+
+ sleep 10
+
+ su -s /bin/bash -c "bundle exec sidekiq \
+ -e $RACK_ENV \
+ -P $SIDEKIQ_PID \
+ -C $SIDEKIQ_CONFIG \
+ -L $SIDEKIQ_LOG \
+ -d" $EB_APP_USER
+
+ "/opt/elasticbeanstalk/hooks/appdeploy/pre/03_mute_sidekiq.sh":
+ mode: "000755"
+ owner: root
+ group: root
+ content: |
+ #!/usr/bin/env bash
+ . /opt/elasticbeanstalk/support/envvars
+
+ EB_APP_USER=$(/opt/elasticbeanstalk/bin/get-config container -k app_user)
+ EB_SCRIPT_DIR=$(/opt/elasticbeanstalk/bin/get-config container -k script_dir)
+ EB_SUPPORT_DIR=$(/opt/elasticbeanstalk/bin/get-config container -k support_dir)
+
+ . $EB_SUPPORT_DIR/envvars
+ . $EB_SCRIPT_DIR/use-app-ruby.sh
+
+ SIDEKIQ_PID=$EB_APP_PID_DIR/sidekiq.pid
+ if [ -f $SIDEKIQ_PID ]
+ then
+ su -s /bin/bash -c "kill -USR1 `cat $SIDEKIQ_PID`" $EB_APP_USER
+ fi

.ebextensions/sidekiq_permissions.config

@@ -0,0 +1,31 @@
+# Set permissions for files accessed by Sidekiq
+files:
+ "/opt/elasticbeanstalk/hooks/appdeploy/post/51_chmod_logs.sh":
+ mode: "000755"
+ owner: root
+ group: root
+ content: |
+ #!/usr/bin/env bash
+ . /opt/elasticbeanstalk/support/envvars
+
+ EB_APP_DEPLOY_DIR=$(/opt/elasticbeanstalk/bin/get-config container -k app_deploy_dir)
+ APP_LOG_DIR=$EB_APP_DEPLOY_DIR/log
+
+ SIDEKIQ_LOG=$APP_LOG_DIR/sidekiq.log
+
+ case $RACK_ENV in
+ "integration")
+ RAILS_LOG=$APP_LOG_DIR/integration.log
+ ;;
+ "staging")
+ RAILS_LOG=$APP_LOG_DIR/staging.log
+ ;;
+ "production")
+ RAILS_LOG=$APP_LOG_DIR/production.log
+ ;;
+ esac
+
+ touch $SIDEKIQ_LOG
+ chmod 0664 $SIDEKIQ_LOG
+ touch $RAILS_LOG
+ chmod 0664 $RAILS_LOG

Gemfile

@@ -13,6 +13,7 @@ gem 'dotenv-rails'
 gem 'qa_server', '~> 7.2'
 gem 'qa', '~> 5.3'
 gem 'linkeddata'
+gem 'sidekiq', '~> 5.2'
 
 # Other gems
 gem 'coffee-rails', '~> 4.2'

Gemfile.lock

@@ -217,6 +217,8 @@ GEM
  sass-rails (~> 5.0)
  useragent
  rack (2.2.2)
+ rack-protection (2.0.8.1)
+ rack
  rack-test (1.1.0)
  rack (>= 1.0, < 3)
  rails (5.1.7)
@@ -306,6 +308,7 @@ GEM
  rdf (~> 3.1)
  rdf-xsd (3.1.0)
  rdf (~> 3.1)
+ redis (4.1.3)
  request_store (1.5.0)
  rack (>= 1.4)
  rmagick (2.16.0)
@@ -361,6 +364,11 @@ GEM
  rdf-xsd (~> 3.1)
  sparql (~> 3.1)
  sxp (~> 1.1)
+ sidekiq (5.2.7)
+ connection_pool (~> 2.2, >= 2.2.2)
+ rack (>= 1.5.0)
+ rack-protection (>= 1.5.0)
+ redis (>= 3.3.5, < 5)
  simplecov (0.16.1)
  docile (~> 1.1)
  json (>= 1.8, < 3)
@@ -451,6 +459,7 @@ DEPENDENCIES
  rubocop
  rubocop-checkstyle_formatter
  sass-rails (~> 5.0)
+ sidekiq (~> 5.2)
  spring
  sqlite3
  swagger-docs

README.md

@@ -17,9 +17,17 @@ This app can be installed to serve as a Questioning Authority (QA) Server for ac
 
 ### To Start the Server
 
-* run `rails s`
+Start Sidekiq in development for processing background jobs (Needs to run at the root of the rails app.)
+
+```sh
+$ bundle exec sidekiq -d -L log/sidekiq.log -e development
+```
 
-To stop, use `<ctrl><c>` in the terminal window where `rails s` started.
+Start application server
+
+```sh
+$ bundle exec rails s
+```
 
 ## Supported Authorities
 

config/environments/development.rb

@@ -60,6 +60,7 @@
  # routes, locales, etc. This feature depends on the listen gem.
  config.file_watcher = ActiveSupport::EventedFileUpdateChecker
 
- config.active_job.queue_adapter = :async # runs in-memory; a crash will lose the job; change to :inline to run immediately for debugging
+ config.active_job.queue_adapter = :sidekiq
+ # config.active_job.queue_adapter = :async # runs in-memory; a crash will lose the job; change to :inline to run immediately for debugging
  # config.active_job.queue_adapter = :inline # runs when it is called for debugging
 end

config/environments/integration.rb

@@ -62,8 +62,10 @@
  # config.active_job.queue_adapter = :resque
  # config.active_job.queue_name_prefix = "qa_server_#{Rails.env}"
 
+ config.active_job.queue_adapter = :sidekiq
+
  # run jobs asynchronously in background (ok because only background jobs are related to monitoring status)
- config.active_job.queue_adapter = :async # runs in-memory; a crash will lose the job; change to :inline to run immediately for debugging
+ # config.active_job.queue_adapter = :async # runs in-memory; a crash will lose the job; change to :inline to run immediately for debugging
 
  config.action_mailer.perform_caching = false
 

config/environments/production.rb

@@ -56,8 +56,10 @@
  # config.active_job.queue_adapter = :resque
  # config.active_job.queue_name_prefix = "qa_server_#{Rails.env}"
 
+ config.active_job.queue_adapter = :sidekiq
+
  # run jobs asynchronously in background (ok because only background jobs are related to monitoring status)
- config.active_job.queue_adapter = :async # runs in-memory; a crash will lose the job; change to :inline to run immediately for debugging
+ # config.active_job.queue_adapter = :async # runs in-memory; a crash will lose the job; change to :inline to run immediately for debugging
 
  config.action_mailer.perform_caching = false
 

config/environments/staging.rb

@@ -62,8 +62,10 @@
  # config.active_job.queue_adapter = :resque
  # config.active_job.queue_name_prefix = "qa_server_#{Rails.env}"
 
+ config.active_job.queue_adapter = :sidekiq
+
  # run jobs asynchronously in background (ok because only background jobs are related to monitoring status)
- config.active_job.queue_adapter = :async # runs in-memory; a crash will lose the job; change to :inline to run immediately for debugging
+ # config.active_job.queue_adapter = :async # runs in-memory; a crash will lose the job; change to :inline to run immediately for debugging
 
  config.action_mailer.perform_caching = false
 

config/initializers/sidekiq.rb

@@ -0,0 +1,11 @@
+config = YAML.load(ERB.new(IO.read(Rails.root + 'config' + 'redis.yml')).result)[Rails.env].with_indifferent_access
+
+redis_conn = { url: "redis://#{config[:host]}:#{config[:port]}/" }
+
+Sidekiq.configure_server do |s|
+ s.redis = redis_conn
+end
+
+Sidekiq.configure_client do |s|
+ s.redis = redis_conn
+end

config/locales/qa_server.en.yml

@@ -1,7 +1,7 @@
 ---
 en:
  qa_server:
- application_version: "5.3.2"
+ application_version: "5.3.3.rc1"
  application_name: "LD4P Authority Lookup Service"
  footer:
  copyright_html: "<strong>Copyright &copy; 2018-2020 Cornell</strong> Licensed under the Apache License, Version 2.0"

config/redis.yml

@@ -0,0 +1,24 @@
+default:
+ host: <%= ENV['REDIS_HOST'] || "localhost" %>
+ port: 6379
+ db: 0
+development:
+ host: <%= ENV['REDIS_HOST'] || "localhost" %>
+ port: 6379
+ db: 1
+test:
+ host: <%= ENV['REDIS_HOST'] || "localhost" %>
+ port: 6379
+ db: 2
+integration:
+ host: <%= ENV['REDIS_HOST'] || "localhost" %>
+ port: 6379
+ db: 3
+staging:
+ host: <%= ENV['REDIS_HOST'] || "localhost" %>
+ port: 6379
+ db: 4
+production:
+ host: <%= ENV['REDIS_HOST'] || "localhost" %>
+ port: 6379
+ db: 5

config/routes.rb

@@ -1,6 +1,10 @@
 Rails.application.routes.draw do
  mount Qa::Engine => '/authorities'
  mount QaServer::Engine, at: '/'
+
+ require 'sidekiq/web'
+ mount Sidekiq::Web => '/sidekiq'
+
  resources :welcome, only: 'index'
  root 'qa_server/homepage#index'
  # For details on the DSL available within this file, see http://guides.rubyonrails.org/routing.html

config/sidekiq.yml

@@ -0,0 +1,3 @@
+---
+:queues:
+ - default