From bd424556420765363c7dd111378547182b3501ab Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Wed, 10 Jul 2024 14:19:22 -0600 Subject: [PATCH 01/83] SRCH-5358: Add Capistrano Deployment. --- Capfile | 38 +++++++++++++++++++++++++ Gemfile | 7 +++++ Gemfile.lock | 34 +++++++++++++++++++++++ config/deploy.rb | 52 +++++++++++++++++++++++++++++++++++ config/deploy/production.rb | 55 +++++++++++++++++++++++++++++++++++++ config/deploy/staging.rb | 55 +++++++++++++++++++++++++++++++++++++ 6 files changed, 241 insertions(+) create mode 100644 Capfile create mode 100644 config/deploy.rb create mode 100644 config/deploy/production.rb create mode 100644 config/deploy/staging.rb diff --git a/Capfile b/Capfile new file mode 100644 index 0000000000..42dd3a47a1 --- /dev/null +++ b/Capfile @@ -0,0 +1,38 @@ +# Load DSL and set up stages +require "capistrano/setup" + +# Include default deployment tasks +require "capistrano/deploy" + +# Load the SCM plugin appropriate to your project: +# +# require "capistrano/scm/hg" +# install_plugin Capistrano::SCM::Hg +# or +# require "capistrano/scm/svn" +# install_plugin Capistrano::SCM::Svn +# or +require "capistrano/scm/git" +install_plugin Capistrano::SCM::Git + +# Include tasks from other gems included in your Gemfile +# +# For documentation on these, see for example: +# +# https://github.com/capistrano/rvm +# https://github.com/capistrano/rbenv +# https://github.com/capistrano/chruby +# https://github.com/capistrano/bundler +# https://github.com/capistrano/rails +# https://github.com/capistrano/passenger +# +# require "capistrano/rvm" +# require "capistrano/rbenv" +# require "capistrano/chruby" +# require "capistrano/bundler" +# require "capistrano/rails/assets" +# require "capistrano/rails/migrations" +# require "capistrano/passenger" + +# Load custom tasks from `lib/capistrano/tasks` if you have any defined +Dir.glob("lib/capistrano/tasks/*.rake").each { |r| import r } diff --git a/Gemfile b/Gemfile index 041a632c16..f3b92994e1 100644 --- a/Gemfile +++ b/Gemfile @@ -31,6 +31,7 @@ gem 'resque-scheduler', '~> 4.10.2' # Using a third-party fork as an interim measure. gem 'kt-paperclip', '~> 7.1.0' gem 'aws-sdk-s3', '~> 1.102.0' +gem 'aws-sdk-ssm', '~> 1.173' gem 'googlecharts', '~> 1.6.12' gem 'flickraw', '~> 0.9.9' # SRCH-3837: We need this change: https://github.com/activescaffold/active_scaffold/pull/666 @@ -118,6 +119,12 @@ gem 'dogapi', '~> 1.45' # This gem can be removed once we upgrade to Ruby 3.1. gem 'net-http' +# Deployment +gem 'capistrano', '~> 3.19', '>= 3.19.1' +gem 'capistrano-rails', '~> 1.6', '>= 1.6.3' +gem 'capistrano-rbenv', '~> 2.2' +gem 'capistrano3-puma', '~> 5.2' + # Assets-related gems gem 'coffee-rails', '~> 5.0.0' gem 'uglifier', '~> 4.2.0' diff --git a/Gemfile.lock b/Gemfile.lock index 475a366950..76894cd3bb 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -172,6 +172,8 @@ GEM after_commit_action (1.1.0) activerecord (>= 3.0.0) activesupport (>= 3.0.0) + airbrussh (1.5.2) + sshkit (>= 1.6.1, != 1.7.0) amazing_print (1.5.0) american_date (1.1.1) ast (2.4.2) @@ -214,11 +216,29 @@ GEM babel-transpiler (0.7.0) babel-source (>= 4.0, < 6) execjs (~> 2.0) + base64 (0.2.0) bindata (2.5.0) bootsnap (1.16.0) msgpack (~> 1.2) builder (3.2.4) byebug (11.1.3) + capistrano (3.19.1) + airbrussh (>= 1.0.0) + i18n + rake (>= 10.0.0) + sshkit (>= 1.9.0) + capistrano-bundler (2.1.0) + capistrano (~> 3.1) + capistrano-rails (1.6.3) + capistrano (~> 3.1) + capistrano-bundler (>= 1.1, < 3) + capistrano-rbenv (2.2.0) + capistrano (~> 3.1) + sshkit (~> 1.3) + capistrano3-puma (5.2.0) + capistrano (~> 3.7) + capistrano-bundler + puma (>= 4.0, < 6.0) capybara (3.39.2) addressable matrix @@ -504,8 +524,13 @@ GEM net-protocol net-protocol (0.2.2) timeout + net-scp (4.0.0) + net-ssh (>= 2.6.5, < 8.0.0) + net-sftp (4.0.0) + net-ssh (>= 5.0.0, < 8.0.0) net-smtp (0.4.0.1) net-protocol + net-ssh (7.2.3) newrelic_rpm (9.10.2) nio4r (2.7.1) nokogiri (1.16.6) @@ -782,6 +807,11 @@ GEM actionpack (>= 5.2) activesupport (>= 5.2) sprockets (>= 3.0.0) + sshkit (1.23.0) + base64 + net-scp (>= 1.1.2) + net-sftp (>= 2.1.2) + net-ssh (>= 2.8.0) sys-uname (1.2.3) ffi (~> 1.1) temple (0.10.2) @@ -863,6 +893,10 @@ DEPENDENCIES axe-core-capybara axe-core-cucumber bootsnap + capistrano (~> 3.19, >= 3.19.1) + capistrano-rails (~> 1.6, >= 1.6.3) + capistrano-rbenv (~> 2.2) + capistrano3-puma (~> 5.2) capybara (~> 3.26) capybara-screenshot cld3 (~> 3.5.0) diff --git a/config/deploy.rb b/config/deploy.rb new file mode 100644 index 0000000000..7f3a53c9f3 --- /dev/null +++ b/config/deploy.rb @@ -0,0 +1,52 @@ +# config valid for current version and patch releases of Capistrano +lock "~> 3.19.1" + +set :application, 'search-gov' +set :repo_url, "https://github.com/GSA/search-gov" +set :branch, "main" + +# Set the directory to deploy to +set :deploy_to, '/var/www/my_app_name' + +# Use rbenv to manage Ruby versions +set :rbenv_type, :user +set :rbenv_ruby, '3.1.4' + +# Linked files and directories (these will be shared across releases) +append :linked_files, 'config/database.yml', 'config/secrets.yml', '.env' +append :linked_dirs, 'log', 'tmp/pids', 'tmp/cache', 'tmp/sockets', 'public/system', 'vendor', 'storage' + +# Keep only the last 5 releases to save disk space +set :keep_releases, 5 + +# Custom tasks for fetching environment variables from AWS SSM +namespace :deploy do + task :fetch_env_vars do + on roles(:app) do + within release_path do + execute :bundle, :exec, :ruby, '-e', %Q{ + require 'aws-sdk-ssm' + client = Aws::SSM::Client.new(region: 'us-east-1') + path = fetch(:aws_ssm_path) + env_vars = client.get_parameters_by_path({ + path: path, + with_decryption: true + }).parameters + File.open('.env', 'w') do |file| + env_vars.each do |param| + file.puts "#{param.name.split('/').last}=#{param.value}" + end + end + } + end + end + end + + before 'deploy:check:linked_files', 'deploy:fetch_env_vars' + + after :finishing, 'deploy:cleanup' + after :finishing, 'deploy:restart' + after :rollback, 'deploy:restart' + after :finishing, 'datadog:notify' + after :rollback, 'datadog:notify' +end diff --git a/config/deploy/production.rb b/config/deploy/production.rb new file mode 100644 index 0000000000..b1fd182010 --- /dev/null +++ b/config/deploy/production.rb @@ -0,0 +1,55 @@ +# config/deploy/production.rb + +# Server-based syntax +# ====================== +# Defines a single server with a list of roles and multiple properties. +# You can define all roles on a single server, or split them: + +server "production.server.com", user: "deploy", roles: %w{app db web} + +# Role-based syntax +# ================== +# Defines a role with one or multiple servers. +# The primary server in each group is considered to be the first unless any hosts have the primary property set. +# Specify the username and a domain or IP for the server. + +# role :app, %w{deploy@production.server.com} +# role :web, %w{user1@production.server.com user2@production.server.com} +# role :db, %w{deploy@production.server.com} + +# Configuration +# ============= +# You can set any configuration variable like in config/deploy.rb. +# These variables are then only loaded and set in this stage. +# For available Capistrano configuration variables see the documentation page. +# http://capistranorb.com/documentation/getting-started/configuration/ +# Feel free to add new variables to customize your setup. + +set :rails_env, 'production' +set :aws_ssm_path, '/your/application/env/production/' + +# Custom SSH Options +# ================== +# You may pass any option but keep in mind that net/ssh understands a limited set of options, consult the Net::SSH documentation. +# http://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start + +# Global options +# -------------- +# set :ssh_options, { +# keys: %w(/home/deploy/.ssh/id_rsa), +# forward_agent: false, +# auth_methods: %w(publickey) +# } + +# The server-based syntax can be used to override options: +# ------------------------------------ +# server "production.server.com", +# user: "deploy", +# roles: %w{web app}, +# ssh_options: { +# user: "deploy", # overrides user setting above +# keys: %w(/home/deploy/.ssh/id_rsa), +# forward_agent: false, +# auth_methods: %w(publickey password) +# # password: "please use keys" +# } diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb new file mode 100644 index 0000000000..decbe13d15 --- /dev/null +++ b/config/deploy/staging.rb @@ -0,0 +1,55 @@ +# config/deploy/staging.rb + +# Server-based syntax +# ====================== +# Defines a single server with a list of roles and multiple properties. +# You can define all roles on a single server, or split them: + +server "staging.server.com", user: "deploy", roles: %w{app db web} + +# Role-based syntax +# ================== +# Defines a role with one or multiple servers. +# The primary server in each group is considered to be the first unless any hosts have the primary property set. +# Specify the username and a domain or IP for the server. + +# role :app, %w{deploy@staging.server.com} +# role :web, %w{user1@staging.server.com user2@staging.server.com} +# role :db, %w{deploy@staging.server.com} + +# Configuration +# ============= +# You can set any configuration variable like in config/deploy.rb. +# These variables are then only loaded and set in this stage. +# For available Capistrano configuration variables see the documentation page. +# http://capistranorb.com/documentation/getting-started/configuration/ +# Feel free to add new variables to customize your setup. + +set :rails_env, 'staging' +set :aws_ssm_path, '/your/application/env/staging/' + +# Custom SSH Options +# ================== +# You may pass any option but keep in mind that net/ssh understands a limited set of options, consult the Net::SSH documentation. +# http://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start + +# Global options +# -------------- +# set :ssh_options, { +# keys: %w(/home/deploy/.ssh/id_rsa), +# forward_agent: false, +# auth_methods: %w(publickey) +# } + +# The server-based syntax can be used to override options: +# ------------------------------------ +# server "staging.server.com", +# user: "deploy", +# roles: %w{web app}, +# ssh_options: { +# user: "deploy", # overrides user setting above +# keys: %w(/home/deploy/.ssh/id_rsa), +# forward_agent: false, +# auth_methods: %w(publickey password) +# # password: "please use keys" +# } From b7580380912e90b2f31eab3f8eb579a4d4f7c140 Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Fri, 12 Jul 2024 12:59:46 -0600 Subject: [PATCH 02/83] SRCH-5358: Update Capistrano script to deploy. --- Gemfile.lock | 14 +++++++++----- config/deploy.rb | 12 ++++++------ config/deploy/staging.rb | 28 +++++++++------------------- 3 files changed, 24 insertions(+), 30 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 76894cd3bb..46810f289b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -182,12 +182,12 @@ GEM activerecord (>= 5.2, < 7.1) activesupport (>= 5.2, < 7.1) request_store (~> 1.0) - aws-eventstream (1.2.0) + aws-eventstream (1.3.0) aws-partitions (1.810.0) - aws-sdk-core (3.181.0) - aws-eventstream (~> 1, >= 1.0.2) + aws-sdk-core (3.201.1) + aws-eventstream (~> 1, >= 1.3.0) aws-partitions (~> 1, >= 1.651.0) - aws-sigv4 (~> 1.5) + aws-sigv4 (~> 1.8) jmespath (~> 1, >= 1.6.1) aws-sdk-kms (1.71.0) aws-sdk-core (~> 3, >= 3.177.0) @@ -196,7 +196,10 @@ GEM aws-sdk-core (~> 3, >= 3.120.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.4) - aws-sigv4 (1.6.0) + aws-sdk-ssm (1.173.0) + aws-sdk-core (~> 3, >= 3.201.0) + aws-sigv4 (~> 1.5) + aws-sigv4 (1.8.0) aws-eventstream (~> 1, >= 1.0.2) axe-core-api (4.7.0) dumb_delegator @@ -890,6 +893,7 @@ DEPENDENCIES american_date (~> 1.1.1) authlogic (~> 6.4.1) aws-sdk-s3 (~> 1.102.0) + aws-sdk-ssm (~> 1.173) axe-core-capybara axe-core-cucumber bootsnap diff --git a/config/deploy.rb b/config/deploy.rb index 7f3a53c9f3..efdb271c86 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -6,7 +6,7 @@ set :branch, "main" # Set the directory to deploy to -set :deploy_to, '/var/www/my_app_name' +set :deploy_to, ENV['DEPLOYMENT_PATH'] # Use rbenv to manage Ruby versions set :rbenv_type, :user @@ -26,13 +26,14 @@ within release_path do execute :bundle, :exec, :ruby, '-e', %Q{ require 'aws-sdk-ssm' - client = Aws::SSM::Client.new(region: 'us-east-1') - path = fetch(:aws_ssm_path) + require 'rails' + client = Aws::SSM::Client.new(region: "#{ENV['AWS_REGION']}") + path = "#{ENV['AWS_SSM_PATH']}" env_vars = client.get_parameters_by_path({ path: path, with_decryption: true }).parameters - File.open('.env', 'w') do |file| + File.open(Rails.root.join('.env'), 'w') do |file| env_vars.each do |param| file.puts "#{param.name.split('/').last}=#{param.value}" end @@ -47,6 +48,5 @@ after :finishing, 'deploy:cleanup' after :finishing, 'deploy:restart' after :rollback, 'deploy:restart' - after :finishing, 'datadog:notify' - after :rollback, 'datadog:notify' end + diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index decbe13d15..8619fba98a 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -5,17 +5,7 @@ # Defines a single server with a list of roles and multiple properties. # You can define all roles on a single server, or split them: -server "staging.server.com", user: "deploy", roles: %w{app db web} - -# Role-based syntax -# ================== -# Defines a role with one or multiple servers. -# The primary server in each group is considered to be the first unless any hosts have the primary property set. -# Specify the username and a domain or IP for the server. - -# role :app, %w{deploy@staging.server.com} -# role :web, %w{user1@staging.server.com user2@staging.server.com} -# role :db, %w{deploy@staging.server.com} +server ENV['SERVER_ADDRESS'], user: ENV['SERVER_DEPLOYMENT_USER'], roles: %w{app db web} # Configuration # ============= @@ -26,20 +16,20 @@ # Feel free to add new variables to customize your setup. set :rails_env, 'staging' -set :aws_ssm_path, '/your/application/env/staging/' +set :aws_ssm_path, ENV['AWS_SSM_PATH'] # Custom SSH Options # ================== -# You may pass any option but keep in mind that net/ssh understands a limited set of options, consult the Net::SSH documentation. +# You may pass any option but keep in mind that net/ssh understands a limited set of options, consult the Net/SSH documentation. # http://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start # Global options # -------------- -# set :ssh_options, { -# keys: %w(/home/deploy/.ssh/id_rsa), -# forward_agent: false, -# auth_methods: %w(publickey) -# } +set :ssh_options, { + keys: [ENV['SSH_KEY_PATH']], + forward_agent: false, + auth_methods: %w(publickey) +} # The server-based syntax can be used to override options: # ------------------------------------ @@ -48,7 +38,7 @@ # roles: %w{web app}, # ssh_options: { # user: "deploy", # overrides user setting above -# keys: %w(/home/deploy/.ssh/id_rsa), +# keys: [fetch(:ssh_key_path, '/default/path/to/staging_ec2_keypair.pem')], # forward_agent: false, # auth_methods: %w(publickey password) # # password: "please use keys" From a678093a86f92839a7ef907e2e96ec5a68b58e3d Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Fri, 12 Jul 2024 13:24:16 -0600 Subject: [PATCH 03/83] SRCH-5358: Linter and skip failing test. --- Capfile | 10 ++++++---- spec/system/bing_v7_searches_spec.rb | 2 ++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Capfile b/Capfile index 42dd3a47a1..60c47d72d4 100644 --- a/Capfile +++ b/Capfile @@ -1,8 +1,10 @@ +# frozen_string_literal: true + # Load DSL and set up stages -require "capistrano/setup" +require 'capistrano/setup' # Include default deployment tasks -require "capistrano/deploy" +require 'capistrano/deploy' # Load the SCM plugin appropriate to your project: # @@ -12,7 +14,7 @@ require "capistrano/deploy" # require "capistrano/scm/svn" # install_plugin Capistrano::SCM::Svn # or -require "capistrano/scm/git" +require 'capistrano/scm/git' install_plugin Capistrano::SCM::Git # Include tasks from other gems included in your Gemfile @@ -35,4 +37,4 @@ install_plugin Capistrano::SCM::Git # require "capistrano/passenger" # Load custom tasks from `lib/capistrano/tasks` if you have any defined -Dir.glob("lib/capistrano/tasks/*.rake").each { |r| import r } +Dir.glob('lib/capistrano/tasks/*.rake').each { |r| import r } diff --git a/spec/system/bing_v7_searches_spec.rb b/spec/system/bing_v7_searches_spec.rb index 2069045cb9..d74c75950b 100644 --- a/spec/system/bing_v7_searches_spec.rb +++ b/spec/system/bing_v7_searches_spec.rb @@ -12,6 +12,8 @@ end it 'uses the web search key and end point' do + skip 'Bing is failing and Jim has approved skipping of Bing tests.' + expect(WebMock).to have_requested(:get, /#{web_search_host}#{web_search_path}/). with(headers: { 'Ocp-Apim-Subscription-Key' => web_subscription_id }) end From 67595f110b3e59dd61b8dd0ecab9e54536468888 Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Wed, 17 Jul 2024 12:02:52 -0600 Subject: [PATCH 04/83] latest updates to capistrano. --- Capfile | 8 ++++-- Gemfile | 12 ++++----- config/deploy.rb | 56 +++++++++++++++++++++++----------------- config/deploy/staging.rb | 9 ++++++- 4 files changed, 53 insertions(+), 32 deletions(-) diff --git a/Capfile b/Capfile index 60c47d72d4..0d2f7db08f 100644 --- a/Capfile +++ b/Capfile @@ -29,12 +29,16 @@ install_plugin Capistrano::SCM::Git # https://github.com/capistrano/passenger # # require "capistrano/rvm" -# require "capistrano/rbenv" +require "capistrano/rbenv" # require "capistrano/chruby" -# require "capistrano/bundler" +require "capistrano/bundler" # require "capistrano/rails/assets" # require "capistrano/rails/migrations" # require "capistrano/passenger" +require 'capistrano/rails' +require 'capistrano/puma' + +install_plugin Capistrano::Puma # Default puma tasks # Load custom tasks from `lib/capistrano/tasks` if you have any defined Dir.glob('lib/capistrano/tasks/*.rake').each { |r| import r } diff --git a/Gemfile b/Gemfile index f3b92994e1..0cdb7a2867 100644 --- a/Gemfile +++ b/Gemfile @@ -119,12 +119,6 @@ gem 'dogapi', '~> 1.45' # This gem can be removed once we upgrade to Ruby 3.1. gem 'net-http' -# Deployment -gem 'capistrano', '~> 3.19', '>= 3.19.1' -gem 'capistrano-rails', '~> 1.6', '>= 1.6.3' -gem 'capistrano-rbenv', '~> 2.2' -gem 'capistrano3-puma', '~> 5.2' - # Assets-related gems gem 'coffee-rails', '~> 5.0.0' gem 'uglifier', '~> 4.2.0' @@ -178,6 +172,12 @@ group :development do # gem 'web-console' # Add speed badges [https://github.com/MiniProfiler/rack-mini-profiler] # gem "rack-mini-profiler" + + # Deployment + gem 'capistrano', '~> 3.19', '>= 3.19.1' + gem 'capistrano-rails', '~> 1.6', '>= 1.6.3' + gem 'capistrano-rbenv', '~> 2.2' + gem 'capistrano3-puma', '~> 5.2' end group :development, :test do diff --git a/config/deploy.rb b/config/deploy.rb index efdb271c86..020df9a084 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -1,52 +1,62 @@ # config valid for current version and patch releases of Capistrano -lock "~> 3.19.1" +lock '~> 3.19.1' set :application, 'search-gov' -set :repo_url, "https://github.com/GSA/search-gov" -set :branch, "main" +set :repo_url, 'https://github.com/GSA/search-gov' +set :branch, 'main' # Set the directory to deploy to set :deploy_to, ENV['DEPLOYMENT_PATH'] +set :releases_directory, File.join(ENV['DEPLOYMENT_PATH'], 'releases') +set :current_directory, File.join(ENV['DEPLOYMENT_PATH'], 'current') +set :bundle_without, %w{development test}.join(' ') # Use rbenv to manage Ruby versions set :rbenv_type, :user set :rbenv_ruby, '3.1.4' # Linked files and directories (these will be shared across releases) -append :linked_files, 'config/database.yml', 'config/secrets.yml', '.env' -append :linked_dirs, 'log', 'tmp/pids', 'tmp/cache', 'tmp/sockets', 'public/system', 'vendor', 'storage' +append :linked_files, '.env' +append :linked_dirs, 'log', 'tmp/pids', 'tmp/cache', 'tmp/sockets', 'public/system', 'vendor', 'storage', '.bundle', 'public/assets' # Keep only the last 5 releases to save disk space set :keep_releases, 5 -# Custom tasks for fetching environment variables from AWS SSM namespace :deploy do - task :fetch_env_vars do + task :use_node_version do on roles(:app) do within release_path do - execute :bundle, :exec, :ruby, '-e', %Q{ - require 'aws-sdk-ssm' - require 'rails' - client = Aws::SSM::Client.new(region: "#{ENV['AWS_REGION']}") - path = "#{ENV['AWS_SSM_PATH']}" - env_vars = client.get_parameters_by_path({ - path: path, - with_decryption: true - }).parameters - File.open(Rails.root.join('.env'), 'w') do |file| - env_vars.each do |param| - file.puts "#{param.name.split('/').last}=#{param.value}" - end - end - } + execute 'source ~/.nvm/nvm.sh && nvm install 16.20.2' + execute 'source ~/.nvm/nvm.sh && nvm use 16.20.2' end end end - before 'deploy:check:linked_files', 'deploy:fetch_env_vars' + task :ensure_yarn do + on roles(:app) do + within release_path do + execute 'source ~/.nvm/nvm.sh && yarn install' + end + end + end + + task :precompile_assets do + on roles(:app) do + within release_path do + with rails_env: fetch(:rails_env), SECRET_KEY_BASE: '1', SKIP_CSS_BUILD: '1' do + execute :rake, 'assets:precompile' + end + end + end + end + + before 'deploy:assets:precompile', 'deploy:use_node_version' + before 'deploy:assets:precompile', 'deploy:ensure_yarn' + before 'deploy:assets:precompile', 'deploy:precompile_assets' after :finishing, 'deploy:cleanup' after :finishing, 'deploy:restart' after :rollback, 'deploy:restart' end + diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index 8619fba98a..82e062b80c 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -15,8 +15,15 @@ # http://capistranorb.com/documentation/getting-started/configuration/ # Feel free to add new variables to customize your setup. -set :rails_env, 'staging' +set :rails_env, 'production' set :aws_ssm_path, ENV['AWS_SSM_PATH'] +set :bundle_without, %w{development test}.join(' ') + + +# Fetch and set environment variables +# fetch(:default_env).merge!( +# 'SECRET_KEY_BASE' => '1' +# ) # Custom SSH Options # ================== From 8e337dd182058b4158c444bd06dec7999717aab1 Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Wed, 17 Jul 2024 12:08:25 -0600 Subject: [PATCH 05/83] SRCH-5358: Add comment. --- config/deploy.rb | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/config/deploy.rb b/config/deploy.rb index 020df9a084..de7152c307 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -23,6 +23,33 @@ set :keep_releases, 5 namespace :deploy do + + task :fetch_env_vars do + on roles(:app) do + within release_path do + info "Fetching environment variables from AWS SSM..." + require 'aws-sdk-ssm' + require 'fileutils' + + client = Aws::SSM::Client.new(region: ENV['AWS_REGION']) + path = ENV['AWS_SSM_PATH'] + env_vars = client.get_parameters_by_path({ + path: path, + with_decryption: true + }).parameters + + # parse env_vars + # and set to default_envs using + # the following pattern: + # source: https://capistranorb.com/documentation/getting-started/configuration/ + # fetch(:default_env).merge!( + # 'SECRET_KEY_BASE' => '1' + # ) + end + end + end + + task :use_node_version do on roles(:app) do within release_path do @@ -60,3 +87,4 @@ end + From 5d60f31230eaba9e2cd807e33921ff5dd9a63926 Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Fri, 19 Jul 2024 20:48:04 -0600 Subject: [PATCH 06/83] SRCH-5358: Update dpeloyment configurations. --- Capfile | 6 ++- Gemfile | 4 +- app/assets/builds/.keep | 0 config/deploy.rb | 89 +++++++++++++++------------------------- config/deploy/staging.rb | 7 ++-- 5 files changed, 46 insertions(+), 60 deletions(-) delete mode 100644 app/assets/builds/.keep diff --git a/Capfile b/Capfile index 0d2f7db08f..82b5b0cd50 100644 --- a/Capfile +++ b/Capfile @@ -35,8 +35,12 @@ require "capistrano/bundler" # require "capistrano/rails/assets" # require "capistrano/rails/migrations" # require "capistrano/passenger" -require 'capistrano/rails' +# require 'capistrano/rails' + +# Web server plugins require 'capistrano/puma' +require 'capistrano/puma/workers' + install_plugin Capistrano::Puma # Default puma tasks diff --git a/Gemfile b/Gemfile index 0cdb7a2867..4bc08111ab 100644 --- a/Gemfile +++ b/Gemfile @@ -173,6 +173,9 @@ group :development do # Add speed badges [https://github.com/MiniProfiler/rack-mini-profiler] # gem "rack-mini-profiler" + # web server + gem 'puma', '~> 5.6' + # Deployment gem 'capistrano', '~> 3.19', '>= 3.19.1' gem 'capistrano-rails', '~> 1.6', '>= 1.6.3' @@ -194,7 +197,6 @@ group :development, :test do # For improved console readability: # https://github.com/amazing-print/amazing_print gem 'amazing_print', '~> 1.4' - gem 'puma', '~> 5.6' gem 'debug' end diff --git a/app/assets/builds/.keep b/app/assets/builds/.keep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/config/deploy.rb b/config/deploy.rb index de7152c307..1aaae20524 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -7,84 +7,63 @@ # Set the directory to deploy to set :deploy_to, ENV['DEPLOYMENT_PATH'] -set :releases_directory, File.join(ENV['DEPLOYMENT_PATH'], 'releases') -set :current_directory, File.join(ENV['DEPLOYMENT_PATH'], 'current') -set :bundle_without, %w{development test}.join(' ') # Use rbenv to manage Ruby versions set :rbenv_type, :user set :rbenv_ruby, '3.1.4' # Linked files and directories (these will be shared across releases) -append :linked_files, '.env' -append :linked_dirs, 'log', 'tmp/pids', 'tmp/cache', 'tmp/sockets', 'public/system', 'vendor', 'storage', '.bundle', 'public/assets' +# set :linked_files, %w{ +# config/database.yml +# } -# Keep only the last 5 releases to save disk space -set :keep_releases, 5 - -namespace :deploy do +set :optional_linked_files, %w{ + config/secrets.yml +} - task :fetch_env_vars do - on roles(:app) do - within release_path do - info "Fetching environment variables from AWS SSM..." - require 'aws-sdk-ssm' - require 'fileutils' +set :linked_dirs, %w{ + log + tmp +} - client = Aws::SSM::Client.new(region: ENV['AWS_REGION']) - path = ENV['AWS_SSM_PATH'] - env_vars = client.get_parameters_by_path({ - path: path, - with_decryption: true - }).parameters +# Keep only the last 5 releases to save disk space +set :keep_releases, 5 - # parse env_vars - # and set to default_envs using - # the following pattern: - # source: https://capistranorb.com/documentation/getting-started/configuration/ - # fetch(:default_env).merge!( - # 'SECRET_KEY_BASE' => '1' - # ) - end - end - end +# Configurations for Capistrano and Puma +set :puma_threads, [4, 16] +set :puma_workers, 0 +set :puma_bind, "tcp://0.0.0.0:3000" # or "unix://#{shared_path}/tmp/sockets/puma.sock" +set :puma_state, "#{shared_path}/tmp/pids/puma.state" +set :puma_pid, "#{shared_path}/tmp/pids/puma.pid" +set :puma_access_log, "#{release_path}/log/puma.error.log" +set :puma_error_log, "#{release_path}/log/puma.access.log" +set :puma_preload_app, true +set :puma_worker_timeout, nil +set :puma_init_active_record, true # Change to false when not using ActiveRecord - task :use_node_version do - on roles(:app) do - within release_path do - execute 'source ~/.nvm/nvm.sh && nvm install 16.20.2' - execute 'source ~/.nvm/nvm.sh && nvm use 16.20.2' - end - end - end - - task :ensure_yarn do +namespace :deploy do + task :skip_assets_precompile do on roles(:app) do - within release_path do - execute 'source ~/.nvm/nvm.sh && yarn install' - end + info "Skipping assets precompile step" end end - task :precompile_assets do + task :ensure_precompiled_assets do on roles(:app) do within release_path do - with rails_env: fetch(:rails_env), SECRET_KEY_BASE: '1', SKIP_CSS_BUILD: '1' do - execute :rake, 'assets:precompile' + unless test("[ -d #{release_path}/app/assets/builds ]") + error "Precompiled assets not found in #{release_path}/app/assets/builds" + exit 1 end end end end - before 'deploy:assets:precompile', 'deploy:use_node_version' - before 'deploy:assets:precompile', 'deploy:ensure_yarn' - before 'deploy:assets:precompile', 'deploy:precompile_assets' + before 'deploy:symlink:release', 'deploy:skip_assets_precompile' + before 'deploy:symlink:release', 'deploy:ensure_precompiled_assets' after :finishing, 'deploy:cleanup' - after :finishing, 'deploy:restart' - after :rollback, 'deploy:restart' + after :finishing, 'puma:restart' + after :rollback, 'puma:restart' end - - - diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index 82e062b80c..79e31f488b 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -21,9 +21,10 @@ # Fetch and set environment variables -# fetch(:default_env).merge!( -# 'SECRET_KEY_BASE' => '1' -# ) +fetch(:default_env).merge!( + 'SECRET_KEY_BASE' => '1', + 'SKIP_CSS_BUILD' => '1' +) # Custom SSH Options # ================== From ea514f03779e5f805040610df060c467dabaebf1 Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Fri, 19 Jul 2024 21:55:29 -0600 Subject: [PATCH 07/83] SRCH-5358: Update Gemfile. --- Gemfile | 6 +++--- config/deploy.rb | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Gemfile b/Gemfile index 4bc08111ab..b6835980c4 100644 --- a/Gemfile +++ b/Gemfile @@ -158,6 +158,9 @@ gem 'bootsnap', require: false gem 'rails_semantic_logger', '~> 4.14' +# web server +gem 'puma', '~> 5.6' + # Bundle gems for the local environment. Make sure to # put test-only gems in this group so their generators # and rake tasks are available in development mode: @@ -173,9 +176,6 @@ group :development do # Add speed badges [https://github.com/MiniProfiler/rack-mini-profiler] # gem "rack-mini-profiler" - # web server - gem 'puma', '~> 5.6' - # Deployment gem 'capistrano', '~> 3.19', '>= 3.19.1' gem 'capistrano-rails', '~> 1.6', '>= 1.6.3' diff --git a/config/deploy.rb b/config/deploy.rb index 1aaae20524..39149e1a37 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -53,7 +53,6 @@ on roles(:app) do within release_path do unless test("[ -d #{release_path}/app/assets/builds ]") - error "Precompiled assets not found in #{release_path}/app/assets/builds" exit 1 end end @@ -67,3 +66,6 @@ after :finishing, 'puma:restart' after :rollback, 'puma:restart' end + + + From d41fd320f4498e8091324fcb26de06b9d552abad Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Sat, 20 Jul 2024 10:31:51 -0600 Subject: [PATCH 08/83] SRCH-5358: Update Capistrano configurations. --- config/deploy.rb | 44 ---------------------------------------- config/deploy/staging.rb | 17 ++++++++++++++++ 2 files changed, 17 insertions(+), 44 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index 39149e1a37..0f9d64971d 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -25,47 +25,3 @@ log tmp } - -# Keep only the last 5 releases to save disk space -set :keep_releases, 5 - -# Configurations for Capistrano and Puma -set :puma_threads, [4, 16] -set :puma_workers, 0 - -set :puma_bind, "tcp://0.0.0.0:3000" # or "unix://#{shared_path}/tmp/sockets/puma.sock" -set :puma_state, "#{shared_path}/tmp/pids/puma.state" -set :puma_pid, "#{shared_path}/tmp/pids/puma.pid" -set :puma_access_log, "#{release_path}/log/puma.error.log" -set :puma_error_log, "#{release_path}/log/puma.access.log" -set :puma_preload_app, true -set :puma_worker_timeout, nil -set :puma_init_active_record, true # Change to false when not using ActiveRecord - -namespace :deploy do - task :skip_assets_precompile do - on roles(:app) do - info "Skipping assets precompile step" - end - end - - task :ensure_precompiled_assets do - on roles(:app) do - within release_path do - unless test("[ -d #{release_path}/app/assets/builds ]") - exit 1 - end - end - end - end - - before 'deploy:symlink:release', 'deploy:skip_assets_precompile' - before 'deploy:symlink:release', 'deploy:ensure_precompiled_assets' - - after :finishing, 'deploy:cleanup' - after :finishing, 'puma:restart' - after :rollback, 'puma:restart' -end - - - diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index 79e31f488b..6192477728 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -51,3 +51,20 @@ # auth_methods: %w(publickey password) # # password: "please use keys" # } + + +# Keep only the last 5 releases to save disk space +set :keep_releases, 5 + +# Configurations for Capistrano and Puma +set :puma_threads, [4, 16] +set :puma_workers, 0 + +set :puma_bind, "tcp://0.0.0.0:3000" # or "unix://#{shared_path}/tmp/sockets/puma.sock" +set :puma_state, "#{shared_path}/tmp/pids/puma.state" +set :puma_pid, "#{shared_path}/tmp/pids/puma.pid" +set :puma_access_log, "#{release_path}/log/puma.error.log" +set :puma_error_log, "#{release_path}/log/puma.access.log" +set :puma_preload_app, true +set :puma_worker_timeout, nil +set :puma_init_active_record, true # Change to false when not using ActiveRecord \ No newline at end of file From 5dd405224f887a1722bd81ab406ff0260d69485a Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Mon, 22 Jul 2024 11:07:24 -0600 Subject: [PATCH 09/83] SRCH-5358: Add explicit binding to puma config. --- config/puma.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/puma.rb b/config/puma.rb index daaf036999..185d097671 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -41,3 +41,5 @@ # Allow puma to be restarted by `bin/rails restart` command. plugin :tmp_restart + +bind "tcp://0.0.0.0:3000" # Ensure Puma listens on all interfaces \ No newline at end of file From 940d12c4f928ec32689b50971ba10d21414e415d Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Mon, 22 Jul 2024 12:06:55 -0600 Subject: [PATCH 10/83] SRCH-5358: Remove explicit bind. --- config/puma.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/puma.rb b/config/puma.rb index 185d097671..daaf036999 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -41,5 +41,3 @@ # Allow puma to be restarted by `bin/rails restart` command. plugin :tmp_restart - -bind "tcp://0.0.0.0:3000" # Ensure Puma listens on all interfaces \ No newline at end of file From adc3c4544e6baa310dc67a27d52465cb24d9b2ef Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Mon, 22 Jul 2024 12:58:31 -0600 Subject: [PATCH 11/83] SRCH-5358: Revert commit for binding on puma server. --- config/puma.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/puma.rb b/config/puma.rb index daaf036999..185d097671 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -41,3 +41,5 @@ # Allow puma to be restarted by `bin/rails restart` command. plugin :tmp_restart + +bind "tcp://0.0.0.0:3000" # Ensure Puma listens on all interfaces \ No newline at end of file From 13ed0445cb041395ac345993ddfdfac2c56ce19d Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Mon, 22 Jul 2024 16:02:14 -0600 Subject: [PATCH 12/83] Add missing puma plugin. --- Capfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Capfile b/Capfile index 82b5b0cd50..607225d990 100644 --- a/Capfile +++ b/Capfile @@ -43,6 +43,7 @@ require 'capistrano/puma/workers' install_plugin Capistrano::Puma # Default puma tasks +install_plugin Capistrano::Puma::Systemd # Load custom tasks from `lib/capistrano/tasks` if you have any defined Dir.glob('lib/capistrano/tasks/*.rake').each { |r| import r } From 3c831d8c48060bdf792dc6d37c56d2a77c24498b Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Mon, 22 Jul 2024 16:40:20 -0600 Subject: [PATCH 13/83] SRCH-5358: Add config for puma. --- config/deploy/staging.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index 6192477728..ef805b2227 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -67,4 +67,5 @@ set :puma_error_log, "#{release_path}/log/puma.access.log" set :puma_preload_app, true set :puma_worker_timeout, nil -set :puma_init_active_record, true # Change to false when not using ActiveRecord \ No newline at end of file +set :puma_init_active_record, true # Change to false when not using ActiveRecord +set :puma_enable_socket_service, true \ No newline at end of file From 276e621939eed8d6282b077e0adc6163ca43dbdb Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Mon, 22 Jul 2024 22:06:38 -0600 Subject: [PATCH 14/83] SRCH-5358: Update configs. --- config/deploy/staging.rb | 21 --------------------- config/puma.rb | 18 ------------------ 2 files changed, 39 deletions(-) diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index ef805b2227..c43ba19de4 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -39,33 +39,12 @@ auth_methods: %w(publickey) } -# The server-based syntax can be used to override options: -# ------------------------------------ -# server "staging.server.com", -# user: "deploy", -# roles: %w{web app}, -# ssh_options: { -# user: "deploy", # overrides user setting above -# keys: [fetch(:ssh_key_path, '/default/path/to/staging_ec2_keypair.pem')], -# forward_agent: false, -# auth_methods: %w(publickey password) -# # password: "please use keys" -# } - - # Keep only the last 5 releases to save disk space set :keep_releases, 5 -# Configurations for Capistrano and Puma -set :puma_threads, [4, 16] -set :puma_workers, 0 - -set :puma_bind, "tcp://0.0.0.0:3000" # or "unix://#{shared_path}/tmp/sockets/puma.sock" set :puma_state, "#{shared_path}/tmp/pids/puma.state" set :puma_pid, "#{shared_path}/tmp/pids/puma.pid" set :puma_access_log, "#{release_path}/log/puma.error.log" set :puma_error_log, "#{release_path}/log/puma.access.log" set :puma_preload_app, true -set :puma_worker_timeout, nil -set :puma_init_active_record, true # Change to false when not using ActiveRecord set :puma_enable_socket_service, true \ No newline at end of file diff --git a/config/puma.rb b/config/puma.rb index 185d097671..6023eac9e8 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -10,35 +10,17 @@ # Specifies the `worker_timeout` threshold that Puma will use to wait before # terminating a worker in development environments. -# worker_timeout 3600 if ENV.fetch("RAILS_ENV", "development") == "development" # Specifies the `port` that Puma will listen on to receive requests; default is 3000. -# port ENV.fetch("PORT") { 3000 } # Specifies the `environment` that Puma will run in. -# environment ENV.fetch("RAILS_ENV") { "development" } # Specifies the `pidfile` that Puma will use. pidfile ENV.fetch("PIDFILE") { "tmp/pids/server.pid" } -# Specifies the number of `workers` to boot in clustered mode. -# Workers are forked web server processes. If using threads and workers together -# the concurrency of the application would be max `threads` * `workers`. -# Workers do not work on JRuby or Windows (both of which do not support -# processes). -# -# workers ENV.fetch("WEB_CONCURRENCY") { 2 } - -# Use the `preload_app!` method when specifying a `workers` number. -# This directive tells Puma to first boot the application and load code -# before forking the application. This takes advantage of Copy On Write -# process behavior so workers use less memory. -# -# preload_app! - # Allow puma to be restarted by `bin/rails restart` command. plugin :tmp_restart From 84b85007fb77e1a367d2bf08a1c9a5b0834756ce Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Mon, 22 Jul 2024 22:17:39 -0600 Subject: [PATCH 15/83] SRCH-5358: Add new rails script to capistrano. --- config/deploy/staging.rb | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index c43ba19de4..eba78709ca 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -47,4 +47,21 @@ set :puma_access_log, "#{release_path}/log/puma.error.log" set :puma_error_log, "#{release_path}/log/puma.access.log" set :puma_preload_app, true -set :puma_enable_socket_service, true \ No newline at end of file +set :puma_enable_socket_service, true + + +namespace :deploy do + task :start_rails_server do + on roles(:app) do + within release_path do + with rails_env: fetch(:rails_env) do + execute :bundle, 'exec rails server -e production' + end + end + end + end + + after :finishing, 'deploy:cleanup' + after :finishing, 'deploy:start_rails_server' + after :rollback, 'deploy:start_rails_server' +end \ No newline at end of file From c9f954e42c85bdb690f69423179a1689b2e18ecc Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Mon, 22 Jul 2024 22:19:47 -0600 Subject: [PATCH 16/83] SRCH-5358: Fix spacing. --- config/deploy.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/deploy.rb b/config/deploy.rb index 0f9d64971d..714c9ce013 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -25,3 +25,5 @@ log tmp } + + From 214dda9e531e7f629a8a921ec5505c276f873bcd Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Tue, 23 Jul 2024 18:11:43 -0600 Subject: [PATCH 17/83] Update environment variable. --- config/database.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/database.yml b/config/database.yml index 265f1dc2f8..075ee9782d 100644 --- a/config/database.yml +++ b/config/database.yml @@ -27,7 +27,7 @@ development: production: <<: *DEFAULT database: <%= ENV['DB_NAME'] %> - username: <%= ENV['DB_USERNAME'] %> + username: <%= ENV['DB_USER'] %> password: <%= ENV['DB_PASSWORD'] %> host: <%= ENV['DB_HOST'] %> port: <%= ENV['DB_PORT'] %> From 48db1907aaf2c6c3c2a870bdc90d55494e1480bd Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Tue, 23 Jul 2024 19:11:21 -0600 Subject: [PATCH 18/83] Revert binding. --- config/puma.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/config/puma.rb b/config/puma.rb index 6023eac9e8..705eff8465 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -24,4 +24,3 @@ # Allow puma to be restarted by `bin/rails restart` command. plugin :tmp_restart -bind "tcp://0.0.0.0:3000" # Ensure Puma listens on all interfaces \ No newline at end of file From 1a4118f16a65ac4e94bc6fc50bf2523ca90a0635 Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Tue, 23 Jul 2024 21:45:11 -0600 Subject: [PATCH 19/83] Add bind and logs. --- config/puma.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/puma.rb b/config/puma.rb index 705eff8465..af320cb324 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -21,6 +21,10 @@ # Specifies the `pidfile` that Puma will use. pidfile ENV.fetch("PIDFILE") { "tmp/pids/server.pid" } +bind "tcp://0.0.0.0:3000" + +stdout_redirect "#{shared_path}/log/puma.access.log", "#{shared_path}/log/puma.error.log", true + # Allow puma to be restarted by `bin/rails restart` command. plugin :tmp_restart From a5546b22e818db66fc396139f78d203474c67e67 Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Tue, 23 Jul 2024 22:52:06 -0600 Subject: [PATCH 20/83] Remove log files --- config/puma.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/puma.rb b/config/puma.rb index af320cb324..60fc8786ca 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -23,8 +23,6 @@ bind "tcp://0.0.0.0:3000" -stdout_redirect "#{shared_path}/log/puma.access.log", "#{shared_path}/log/puma.error.log", true - # Allow puma to be restarted by `bin/rails restart` command. plugin :tmp_restart From c3660b3d4227cf5294d8615ca6f6543ce01a779a Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Wed, 24 Jul 2024 22:01:35 -0600 Subject: [PATCH 21/83] Update puma config file. --- config/puma.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/puma.rb b/config/puma.rb index 60fc8786ca..3d26a2ecf7 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -12,8 +12,9 @@ # terminating a worker in development environments. worker_timeout 3600 if ENV.fetch("RAILS_ENV", "development") == "development" +# Comment out the port setting if you are using bind # Specifies the `port` that Puma will listen on to receive requests; default is 3000. -port ENV.fetch("PORT") { 3000 } +# port ENV.fetch("PORT") { 3000 } # Specifies the `environment` that Puma will run in. environment ENV.fetch("RAILS_ENV") { "development" } @@ -21,8 +22,8 @@ # Specifies the `pidfile` that Puma will use. pidfile ENV.fetch("PIDFILE") { "tmp/pids/server.pid" } +# Use the bind directive to specify the address and port to listen on bind "tcp://0.0.0.0:3000" # Allow puma to be restarted by `bin/rails restart` command. plugin :tmp_restart - From b17eed266974027bd88c48ec8d1461bbc56b530b Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Thu, 25 Jul 2024 12:16:08 -0600 Subject: [PATCH 22/83] SRCH-5358: Update deploy script. --- config/deploy.rb | 2 -- config/deploy/staging.rb | 37 +++++++++---------------------------- 2 files changed, 9 insertions(+), 30 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index 714c9ce013..0f9d64971d 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -25,5 +25,3 @@ log tmp } - - diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index eba78709ca..3a6bf303a7 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -19,13 +19,6 @@ set :aws_ssm_path, ENV['AWS_SSM_PATH'] set :bundle_without, %w{development test}.join(' ') - -# Fetch and set environment variables -fetch(:default_env).merge!( - 'SECRET_KEY_BASE' => '1', - 'SKIP_CSS_BUILD' => '1' -) - # Custom SSH Options # ================== # You may pass any option but keep in mind that net/ssh understands a limited set of options, consult the Net/SSH documentation. @@ -42,26 +35,14 @@ # Keep only the last 5 releases to save disk space set :keep_releases, 5 +set :puma_rackup, -> { File.join(current_path, 'config.ru') } set :puma_state, "#{shared_path}/tmp/pids/puma.state" set :puma_pid, "#{shared_path}/tmp/pids/puma.pid" -set :puma_access_log, "#{release_path}/log/puma.error.log" -set :puma_error_log, "#{release_path}/log/puma.access.log" -set :puma_preload_app, true -set :puma_enable_socket_service, true - - -namespace :deploy do - task :start_rails_server do - on roles(:app) do - within release_path do - with rails_env: fetch(:rails_env) do - execute :bundle, 'exec rails server -e production' - end - end - end - end - - after :finishing, 'deploy:cleanup' - after :finishing, 'deploy:start_rails_server' - after :rollback, 'deploy:start_rails_server' -end \ No newline at end of file +set :puma_access_log, "#{release_path}/log/puma.access.log" +set :puma_error_log, "#{release_path}/log/puma.error.log" +set :puma_threads, [0, 8] +set :puma_workers, 0 +set :puma_worker_timeout, nil +set :puma_init_active_record, true +set :puma_preload_app, false +set :puma_bind, "tcp://0.0.0.0:3000" From 3ec9a31c4aa16d624b01bc2e47ad1b20390a95d1 Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Thu, 25 Jul 2024 14:52:42 -0600 Subject: [PATCH 23/83] Update ES configs. --- lib/es.rb | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/es.rb b/lib/es.rb index 6bfbdab3ac..452a61c7aa 100644 --- a/lib/es.rb +++ b/lib/es.rb @@ -38,19 +38,33 @@ def initialize_client(config) module ELK extend Es + private def self.client_config(mode) - Rails.application.secrets.dig(:analytics, :elasticsearch, mode).freeze + config = { + hosts: ENV['ES_HOSTS'] ? JSON.parse(ENV['ES_HOSTS']) : Rails.application.secrets.dig(:analytics, :elasticsearch, mode, :hosts), + user: ENV['ES_USER'] || Rails.application.secrets.dig(:analytics, :elasticsearch, mode, :user), + password: ENV['ES_PASSWORD'] || Rails.application.secrets.dig(:analytics, :elasticsearch, mode, :password) + }.compact + + config.freeze end end module CustomIndices extend Es + private def self.client_config(mode) - Rails.application.secrets.dig(:custom_indices, :elasticsearch, mode).freeze + config = { + hosts: ENV['ES_HOSTS'] ? JSON.parse(ENV['ES_HOSTS']) : Rails.application.secrets.dig(:custom_indices, :elasticsearch, mode, :hosts), + user: ENV['ES_USER'] || Rails.application.secrets.dig(:custom_indices, :elasticsearch, mode, :user), + password: ENV['ES_PASSWORD'] || Rails.application.secrets.dig(:custom_indices, :elasticsearch, mode, :password) + }.compact + + config.freeze end end end From dac358f4a0f74dab1f6f21d2224f72adf94e5748 Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Mon, 29 Jul 2024 13:25:40 -0600 Subject: [PATCH 24/83] SRCH-5358: Set SSL to false. --- config/environments/production.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/environments/production.rb b/config/environments/production.rb index bf78b066ba..dc9ec89916 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -49,7 +49,7 @@ # config.action_cable.allowed_request_origins = [ "http://example.com", /http:\/\/example.*/ ] # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. - config.force_ssl = true + config.force_ssl = false config.ssl_options[:secure_cookies] = true # Include generic and useful information about system operation, but avoid logging too much From 288a6309cf86efb5dc40d8628ae5f12010b89490 Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Mon, 29 Jul 2024 13:51:55 -0600 Subject: [PATCH 25/83] Rever SSL force change for production. --- config/environments/production.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/environments/production.rb b/config/environments/production.rb index dc9ec89916..bf78b066ba 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -49,7 +49,7 @@ # config.action_cable.allowed_request_origins = [ "http://example.com", /http:\/\/example.*/ ] # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. - config.force_ssl = false + config.force_ssl = true config.ssl_options[:secure_cookies] = true # Include generic and useful information about system operation, but avoid logging too much From 32628728f4bb9c176ef190da5d8b7dea9ed17ace Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Mon, 29 Jul 2024 14:16:23 -0600 Subject: [PATCH 26/83] SRCH-5358: Add SSL config to bind in puma config. --- config/puma.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/config/puma.rb b/config/puma.rb index 3d26a2ecf7..07278880c9 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -23,7 +23,14 @@ pidfile ENV.fetch("PIDFILE") { "tmp/pids/server.pid" } # Use the bind directive to specify the address and port to listen on -bind "tcp://0.0.0.0:3000" +# bind "tcp://0.0.0.0:3000" + +ssl_bind '0.0.0.0', 3000, { + key: File.read('/etc/ssl/private/apache-selfsigned.key'), + cert: File.read('/etc/ssl/certs/apache-selfsigned.crt'), + verify_mode: 'none' +} + # Allow puma to be restarted by `bin/rails restart` command. plugin :tmp_restart From e8d8c10607b7bac4023ea031af34ea35c45f4d22 Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Mon, 29 Jul 2024 14:22:13 -0600 Subject: [PATCH 27/83] Update port for SSL bind. --- config/puma.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/puma.rb b/config/puma.rb index 07278880c9..b8d2358a17 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -25,7 +25,7 @@ # Use the bind directive to specify the address and port to listen on # bind "tcp://0.0.0.0:3000" -ssl_bind '0.0.0.0', 3000, { +ssl_bind '0.0.0.0', 3001, { key: File.read('/etc/ssl/private/apache-selfsigned.key'), cert: File.read('/etc/ssl/certs/apache-selfsigned.crt'), verify_mode: 'none' From 0b9cd2c8f2c7acc5a5f68e3638868725a19afe10 Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Mon, 29 Jul 2024 14:30:21 -0600 Subject: [PATCH 28/83] Revert binding back. --- config/puma.rb | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/config/puma.rb b/config/puma.rb index b8d2358a17..3d26a2ecf7 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -23,14 +23,7 @@ pidfile ENV.fetch("PIDFILE") { "tmp/pids/server.pid" } # Use the bind directive to specify the address and port to listen on -# bind "tcp://0.0.0.0:3000" - -ssl_bind '0.0.0.0', 3001, { - key: File.read('/etc/ssl/private/apache-selfsigned.key'), - cert: File.read('/etc/ssl/certs/apache-selfsigned.crt'), - verify_mode: 'none' -} - +bind "tcp://0.0.0.0:3000" # Allow puma to be restarted by `bin/rails restart` command. plugin :tmp_restart From df3834b9ad579485105fb976748877672979ca6f Mon Sep 17 00:00:00 2001 From: Luis Metzger Date: Mon, 29 Jul 2024 14:31:04 -0600 Subject: [PATCH 29/83] Reset force ssl false. --- config/environments/production.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/environments/production.rb b/config/environments/production.rb index bf78b066ba..dc9ec89916 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -49,7 +49,7 @@ # config.action_cable.allowed_request_origins = [ "http://example.com", /http:\/\/example.*/ ] # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. - config.force_ssl = true + config.force_ssl = false config.ssl_options[:secure_cookies] = true # Include generic and useful information about system operation, but avoid logging too much From 8b166b8e13ad99df9124390207798c9753cf0a4e Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Tue, 30 Jul 2024 14:49:18 -0700 Subject: [PATCH 30/83] Setup branch as staging --- config/deploy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy.rb b/config/deploy.rb index 0f9d64971d..6de3856166 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -3,7 +3,7 @@ set :application, 'search-gov' set :repo_url, 'https://github.com/GSA/search-gov' -set :branch, 'main' +set :branch, 'staging' # Set the directory to deploy to set :deploy_to, ENV['DEPLOYMENT_PATH'] From fe2637c922b1270fe06abfd5615426319798b6ce Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Tue, 30 Jul 2024 14:53:48 -0700 Subject: [PATCH 31/83] Enable assets compilation --- Capfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Capfile b/Capfile index 607225d990..c1df71f6d5 100644 --- a/Capfile +++ b/Capfile @@ -32,7 +32,7 @@ install_plugin Capistrano::SCM::Git require "capistrano/rbenv" # require "capistrano/chruby" require "capistrano/bundler" -# require "capistrano/rails/assets" +require "capistrano/rails/assets" # require "capistrano/rails/migrations" # require "capistrano/passenger" # require 'capistrano/rails' From d8078ac7552a53a946587aa571d80d62e197c998 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Wed, 31 Jul 2024 13:50:37 -0700 Subject: [PATCH 32/83] Hardcode SECRET_KEY_BASE for assets compilation --- config/deploy.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/deploy.rb b/config/deploy.rb index 6de3856166..be793a2f1c 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -25,3 +25,7 @@ log tmp } + +set :default_env, { + 'SECRET_KEY_BASE' => '1', +} From 47332493efb73f19710dc28d77603265ab08c995 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Wed, 31 Jul 2024 14:35:24 -0700 Subject: [PATCH 33/83] Setup RAILS_SERVE_STATIC_FILES --- config/deploy.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/deploy.rb b/config/deploy.rb index be793a2f1c..50f5b44365 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -27,5 +27,6 @@ } set :default_env, { - 'SECRET_KEY_BASE' => '1', + 'SECRET_KEY_BASE' => '1', + 'RAILS_SERVE_STATIC_FILES' => 'true' } From 57cfd3b100db5c9388262022db07b87c70d9d3e9 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Wed, 31 Jul 2024 16:19:34 -0700 Subject: [PATCH 34/83] Setup json logs for production --- config/environments/production.rb | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/config/environments/production.rb b/config/environments/production.rb index dc9ec89916..402c79ea8a 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -84,12 +84,7 @@ # require "syslog/logger" # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new "app-name") - if ENV["RAILS_LOG_TO_STDOUT"].present? - $stdout.sync = true - config.rails_semantic_logger.add_file_appender = false - config.rails_semantic_logger.format = :json - config.semantic_logger.add_appender(io: $stdout, formatter: config.rails_semantic_logger.format) - end + config.rails_semantic_logger.format = :json # Do not dump schema after migrations. config.active_record.dump_schema_after_migration = false From 3a2a32c775fb4e9cc0fb678351acbab7ec505766 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Thu, 1 Aug 2024 12:36:40 -0700 Subject: [PATCH 35/83] SRCH-5389 setup dotenv gem --- Gemfile | 4 +++- Gemfile.lock | 2 ++ config/deploy.rb | 32 +++++++++----------------------- 3 files changed, 14 insertions(+), 24 deletions(-) diff --git a/Gemfile b/Gemfile index b6835980c4..a3870361c4 100644 --- a/Gemfile +++ b/Gemfile @@ -149,7 +149,6 @@ gem 'react-rails', '~> 3.0.0' # See https://github.com/shakacode/shakapacker#upgrading gem 'shakapacker', '~> 6.5.4' gem 'cssbundling-rails', '~> 1.2' # Management of css (Less) files conversion - # Temporarily locking the 'mail' version until the next version of Rails is released # https://github.com/rails/rails/pull/46650 gem 'mail', '~> 2.7.1' @@ -158,6 +157,8 @@ gem 'bootsnap', require: false gem 'rails_semantic_logger', '~> 4.14' +gem 'dotenv', '~> 3.1' + # web server gem 'puma', '~> 5.6' @@ -218,3 +219,4 @@ group :test do gem 'vcr', '~> 6.0' gem 'webmock', '~> 3.8' end + diff --git a/Gemfile.lock b/Gemfile.lock index 46810f289b..207d89f0d9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -356,6 +356,7 @@ GEM dogstatsd-ruby (3.2.0) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) + dotenv (3.1.2) dumb_delegator (1.0.0) elasticsearch-xpack (7.4.0) elasticsearch-api (>= 6) @@ -917,6 +918,7 @@ DEPENDENCIES debug dogapi (~> 1.45) dogstatsd-ruby (~> 3.2.0) + dotenv (~> 3.1) elasticsearch! elasticsearch-xpack (~> 7.4.0) email_spec (~> 2.2) diff --git a/config/deploy.rb b/config/deploy.rb index 50f5b44365..45f7b98108 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -2,31 +2,17 @@ lock '~> 3.19.1' set :application, 'search-gov' -set :repo_url, 'https://github.com/GSA/search-gov' -set :branch, 'staging' - -# Set the directory to deploy to -set :deploy_to, ENV['DEPLOYMENT_PATH'] +set :branch, :staging +set :deploy_to, ENV['DEPLOYMENT_PATH'] +set :repo_url, 'https://github.com/GSA/search-gov' +set :format, :pretty # Use rbenv to manage Ruby versions -set :rbenv_type, :user set :rbenv_ruby, '3.1.4' +set :rbenv_type, :user -# Linked files and directories (these will be shared across releases) -# set :linked_files, %w{ -# config/database.yml -# } - -set :optional_linked_files, %w{ - config/secrets.yml -} - -set :linked_dirs, %w{ - log - tmp -} +append :linked_dirs, 'log', 'tmp', 'node_modules' +append :linked_files, '.env' -set :default_env, { - 'SECRET_KEY_BASE' => '1', - 'RAILS_SERVE_STATIC_FILES' => 'true' -} +set :rails_env, :production +set :default_env, { SECRET_KEY_BASE: '1' } From a004a4b840ec544d7f5014ad5c8f569cad4cf0de Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Thu, 1 Aug 2024 14:58:51 -0700 Subject: [PATCH 36/83] Use .env.production because .env is a directory already --- config/deploy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy.rb b/config/deploy.rb index 45f7b98108..741d717f62 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -12,7 +12,7 @@ set :rbenv_type, :user append :linked_dirs, 'log', 'tmp', 'node_modules' -append :linked_files, '.env' +append :linked_files, '.env.production' set :rails_env, :production set :default_env, { SECRET_KEY_BASE: '1' } From 42f2040118c192d2ff5b8c08ca65269b8c0e3d6c Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Thu, 1 Aug 2024 15:04:24 -0700 Subject: [PATCH 37/83] Remove .env folder --- .env/development/app | 5 ----- .env/development/resque | 2 -- config/deploy.rb | 2 +- 3 files changed, 1 insertion(+), 8 deletions(-) delete mode 100644 .env/development/app delete mode 100644 .env/development/resque diff --git a/.env/development/app b/.env/development/app deleted file mode 100644 index b51abc514b..0000000000 --- a/.env/development/app +++ /dev/null @@ -1,5 +0,0 @@ -DB_HOST=mysql -ES_HOSTS=http://elasticsearch7:9200 -REDIS_HOST=redis -I14Y_DOCKER_HOST=http://i14y:3200 -ASIS_DOCKER_HOST=http://asis:3300 diff --git a/.env/development/resque b/.env/development/resque deleted file mode 100644 index a3da6088b9..0000000000 --- a/.env/development/resque +++ /dev/null @@ -1,2 +0,0 @@ -QUEUE=* -VERBOSE=true diff --git a/config/deploy.rb b/config/deploy.rb index 741d717f62..45f7b98108 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -12,7 +12,7 @@ set :rbenv_type, :user append :linked_dirs, 'log', 'tmp', 'node_modules' -append :linked_files, '.env.production' +append :linked_files, '.env' set :rails_env, :production set :default_env, { SECRET_KEY_BASE: '1' } From 4e1fc8ee21ea923bf09d2baa54a90d7c0b70d894 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Thu, 1 Aug 2024 15:28:14 -0700 Subject: [PATCH 38/83] Set user=search and environment cleanup --- config/deploy.rb | 1 + config/deploy/staging.rb | 7 ------- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index 45f7b98108..8cd201a92c 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -6,6 +6,7 @@ set :deploy_to, ENV['DEPLOYMENT_PATH'] set :repo_url, 'https://github.com/GSA/search-gov' set :format, :pretty +set :user, :search # Use rbenv to manage Ruby versions set :rbenv_ruby, '3.1.4' diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index 3a6bf303a7..526d064e47 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -15,10 +15,6 @@ # http://capistranorb.com/documentation/getting-started/configuration/ # Feel free to add new variables to customize your setup. -set :rails_env, 'production' -set :aws_ssm_path, ENV['AWS_SSM_PATH'] -set :bundle_without, %w{development test}.join(' ') - # Custom SSH Options # ================== # You may pass any option but keep in mind that net/ssh understands a limited set of options, consult the Net/SSH documentation. @@ -32,9 +28,6 @@ auth_methods: %w(publickey) } -# Keep only the last 5 releases to save disk space -set :keep_releases, 5 - set :puma_rackup, -> { File.join(current_path, 'config.ru') } set :puma_state, "#{shared_path}/tmp/pids/puma.state" set :puma_pid, "#{shared_path}/tmp/pids/puma.pid" From 6bf5c60f4b13f0cfbd24053b6a836dd1b831db3d Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Wed, 7 Aug 2024 13:32:19 -0700 Subject: [PATCH 39/83] Setup capistrano-resque --- Capfile | 2 ++ Gemfile | 10 ++++++---- Gemfile.lock | 5 +++++ config/deploy.rb | 4 ++-- config/deploy/staging.rb | 5 +++++ 5 files changed, 20 insertions(+), 6 deletions(-) diff --git a/Capfile b/Capfile index c1df71f6d5..a53f8d4949 100644 --- a/Capfile +++ b/Capfile @@ -45,5 +45,7 @@ require 'capistrano/puma/workers' install_plugin Capistrano::Puma # Default puma tasks install_plugin Capistrano::Puma::Systemd +require 'capistrano-resque' + # Load custom tasks from `lib/capistrano/tasks` if you have any defined Dir.glob('lib/capistrano/tasks/*.rake').each { |r| import r } diff --git a/Gemfile b/Gemfile index a3870361c4..7b489fc54b 100644 --- a/Gemfile +++ b/Gemfile @@ -178,10 +178,11 @@ group :development do # gem "rack-mini-profiler" # Deployment - gem 'capistrano', '~> 3.19', '>= 3.19.1' - gem 'capistrano-rails', '~> 1.6', '>= 1.6.3' - gem 'capistrano-rbenv', '~> 2.2' - gem 'capistrano3-puma', '~> 5.2' + gem "capistrano-resque", "~> 0.2.3", require: false + gem 'capistrano', '~> 3.19', '>= 3.19.1', require: false + gem 'capistrano-rails', '~> 1.6', '>= 1.6.3', require: false + gem 'capistrano-rbenv', '~> 2.2', require: false + gem 'capistrano3-puma', '~> 5.2', require: false end group :development, :test do @@ -220,3 +221,4 @@ group :test do gem 'webmock', '~> 3.8' end + diff --git a/Gemfile.lock b/Gemfile.lock index 207d89f0d9..b197066e96 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -238,6 +238,10 @@ GEM capistrano-rbenv (2.2.0) capistrano (~> 3.1) sshkit (~> 1.3) + capistrano-resque (0.2.3) + capistrano + resque + resque-scheduler capistrano3-puma (5.2.0) capistrano (~> 3.7) capistrano-bundler @@ -901,6 +905,7 @@ DEPENDENCIES capistrano (~> 3.19, >= 3.19.1) capistrano-rails (~> 1.6, >= 1.6.3) capistrano-rbenv (~> 2.2) + capistrano-resque (~> 0.2.3) capistrano3-puma (~> 5.2) capybara (~> 3.26) capybara-screenshot diff --git a/config/deploy.rb b/config/deploy.rb index 8cd201a92c..b053cd8b4e 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -12,8 +12,8 @@ set :rbenv_ruby, '3.1.4' set :rbenv_type, :user -append :linked_dirs, 'log', 'tmp', 'node_modules' +append :linked_dirs, 'log', 'tmp', 'node_modules', 'public' append :linked_files, '.env' -set :rails_env, :production +set :rails_env, 'production' set :default_env, { SECRET_KEY_BASE: '1' } diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index 526d064e47..176fd20d64 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -39,3 +39,8 @@ set :puma_init_active_record, true set :puma_preload_app, false set :puma_bind, "tcp://0.0.0.0:3000" + + +role :resque, [ENV['RESQUE_SERVER']] + +set :workers, { "searchgov" => 1 } From 882ceff135e92fb8b6a08c99378ab43e1afddb13 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Wed, 7 Aug 2024 13:55:12 -0700 Subject: [PATCH 40/83] Fix resque worker --- Capfile | 3 +++ config/deploy/staging.rb | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Capfile b/Capfile index a53f8d4949..6eefc92723 100644 --- a/Capfile +++ b/Capfile @@ -30,6 +30,9 @@ install_plugin Capistrano::SCM::Git # # require "capistrano/rvm" require "capistrano/rbenv" + +set :rbenv_prefix, "env RBENV_ROOT=#{fetch(:rbenv_path)} RBENV_VERSION=#{fetch(:rbenv_ruby)} #{fetch(:rbenv_path)}/bin/rbenv exec" + # require "capistrano/chruby" require "capistrano/bundler" require "capistrano/rails/assets" diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index 176fd20d64..fe48d1b0d8 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -41,6 +41,6 @@ set :puma_bind, "tcp://0.0.0.0:3000" -role :resque, [ENV['RESQUE_SERVER']] +role :resque_worker, [ENV['RESQUE_SERVER']] set :workers, { "searchgov" => 1 } From fdaa85c63584a87facc9c356db44fb86ca8623b4 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Wed, 7 Aug 2024 15:08:33 -0700 Subject: [PATCH 41/83] Setup workers --- Capfile | 2 +- config/deploy/staging.rb | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Capfile b/Capfile index 6eefc92723..8a2231a941 100644 --- a/Capfile +++ b/Capfile @@ -31,7 +31,7 @@ install_plugin Capistrano::SCM::Git # require "capistrano/rvm" require "capistrano/rbenv" -set :rbenv_prefix, "env RBENV_ROOT=#{fetch(:rbenv_path)} RBENV_VERSION=#{fetch(:rbenv_ruby)} #{fetch(:rbenv_path)}/bin/rbenv exec" +# set :rbenv_prefix, "env RBENV_ROOT=#{fetch(:rbenv_path)} RBENV_VERSION=#{fetch(:rbenv_ruby)} #{fetch(:rbenv_path)}/bin/rbenv exec" # require "capistrano/chruby" require "capistrano/bundler" diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index fe48d1b0d8..89f65d5b1f 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -40,7 +40,6 @@ set :puma_preload_app, false set :puma_bind, "tcp://0.0.0.0:3000" +role :resque_worker, [ENV['RESQUE_ADDRESS']], user: ENV['SERVER_DEPLOYMENT_USER'] -role :resque_worker, [ENV['RESQUE_SERVER']] - -set :workers, { "searchgov" => 1 } +set :workers, { searchgov: 1, sitemap: 1 } From 2b14da594b7907610eceab5a07a474409d16f9dc Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Thu, 8 Aug 2024 13:18:41 -0700 Subject: [PATCH 42/83] Update resque worker to take an array of addresses * Setup primary and general queue as well --- config/deploy.rb | 2 ++ config/deploy/staging.rb | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index b053cd8b4e..4f6aa378a0 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -17,3 +17,5 @@ set :rails_env, 'production' set :default_env, { SECRET_KEY_BASE: '1' } + +set :resque_environment_task, true diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index 89f65d5b1f..c7471b72ed 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -40,6 +40,6 @@ set :puma_preload_app, false set :puma_bind, "tcp://0.0.0.0:3000" -role :resque_worker, [ENV['RESQUE_ADDRESS']], user: ENV['SERVER_DEPLOYMENT_USER'] +role :resque_worker, JSON.parse(ENV.fetch('RESQUE_SERVER_ADDRESSES', '')), user: ENV['SERVER_DEPLOYMENT_USER'] -set :workers, { searchgov: 1, sitemap: 1 } +set :workers, { searchgov: 1, sitemap: 1, primary: 1, '*' => 1 } From ab3fcc57c93269cdddd7a11b1f8db947b75ea773 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Thu, 8 Aug 2024 13:21:19 -0700 Subject: [PATCH 43/83] Add .env to gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 583fb71701..c3c91a4f54 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,5 @@ yarn-debug.log* /app/assets/builds/* !/app/assets/builds/.keep + +.env From 2ae79c610b24734a2f4de69f7357d11215d9d8c6 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Thu, 8 Aug 2024 13:24:43 -0700 Subject: [PATCH 44/83] Setup workers for all environments --- config/deploy.rb | 2 ++ config/deploy/staging.rb | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index 4f6aa378a0..d0152d27cd 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -19,3 +19,5 @@ set :default_env, { SECRET_KEY_BASE: '1' } set :resque_environment_task, true + +role :resque_worker, JSON.parse(ENV.fetch('RESQUE_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index c7471b72ed..2fd57f5e80 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -40,6 +40,4 @@ set :puma_preload_app, false set :puma_bind, "tcp://0.0.0.0:3000" -role :resque_worker, JSON.parse(ENV.fetch('RESQUE_SERVER_ADDRESSES', '')), user: ENV['SERVER_DEPLOYMENT_USER'] - set :workers, { searchgov: 1, sitemap: 1, primary: 1, '*' => 1 } From acb951339cf9d3465473f58140f0e719c01c60c8 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Wed, 14 Aug 2024 16:08:22 -0700 Subject: [PATCH 45/83] Setup deployment to cron servers --- config/deploy.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/deploy.rb b/config/deploy.rb index d0152d27cd..b19c2e9e4c 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -20,4 +20,5 @@ set :resque_environment_task, true -role :resque_worker, JSON.parse(ENV.fetch('RESQUE_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] +role :resque_worker, JSON.parse(ENV.fetch('RESQUE_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] +role :resque_scheduler, JSON.parse(ENV.fetch('CRON_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] From 4fb7c4bddd1930f17bc4b1935422077420b989a8 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Thu, 15 Aug 2024 11:52:53 -0700 Subject: [PATCH 46/83] Automate the creation of .env EMPTY --- config/deploy.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/deploy.rb b/config/deploy.rb index b19c2e9e4c..9a7e6e62bc 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -22,3 +22,9 @@ role :resque_worker, JSON.parse(ENV.fetch('RESQUE_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] role :resque_scheduler, JSON.parse(ENV.fetch('CRON_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] + +namespace :deploy do + task :check do + run "touch #{ shared_path }/.env" + end +end From c9c101c6cb5316d6ca30dea177f5478e59f64ad2 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Thu, 15 Aug 2024 14:05:38 -0700 Subject: [PATCH 47/83] Setup cron task --- config/deploy.rb | 6 ------ config/resque_schedule.yml | 16 ++++++++++++---- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index 9a7e6e62bc..b19c2e9e4c 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -22,9 +22,3 @@ role :resque_worker, JSON.parse(ENV.fetch('RESQUE_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] role :resque_scheduler, JSON.parse(ENV.fetch('CRON_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] - -namespace :deploy do - task :check do - run "touch #{ shared_path }/.env" - end -end diff --git a/config/resque_schedule.yml b/config/resque_schedule.yml index 9fe37a2648..f8afe29e3d 100644 --- a/config/resque_schedule.yml +++ b/config/resque_schedule.yml @@ -1,5 +1,13 @@ production: - SitemapMonitorJob: - cron: "0 9 * * *" - ClickMonitorJob: - cron: "0 7 * * 0" + usesearch:reports:email_monthly_reports: + cron: "0 0 1 * *" + class: "Resque::Job" + usasearch:reports:email_yearly_reports: + cron: "35 21 18 12 *" + class: "Resque::Job" + usasearch:federal_register:import_agencies + cron: "18 9 * * 1-5" + class: "Resque::Job" + usasearch:federal_register:import_documents: + cron: "18 9 * * 1-5" + class: "Resque::Job" From 30da2642016b2c17688a748ef0978c0aca00515a Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Wed, 21 Aug 2024 11:45:59 -0700 Subject: [PATCH 48/83] SRCH-5407 Setup cron tasks --- Capfile | 1 + Gemfile | 13 ++++++------- Gemfile.lock | 8 ++++++-- config/deploy.rb | 3 +-- config/deploy/staging.rb | 4 ++++ config/schedule.rb | 15 +++++++++++++++ 6 files changed, 33 insertions(+), 11 deletions(-) create mode 100644 config/schedule.rb diff --git a/Capfile b/Capfile index 8a2231a941..d59002aaaf 100644 --- a/Capfile +++ b/Capfile @@ -49,6 +49,7 @@ install_plugin Capistrano::Puma # Default puma tasks install_plugin Capistrano::Puma::Systemd require 'capistrano-resque' +require 'whenever/capistrano' # Load custom tasks from `lib/capistrano/tasks` if you have any defined Dir.glob('lib/capistrano/tasks/*.rake').each { |r| import r } diff --git a/Gemfile b/Gemfile index 7b489fc54b..79383c2aca 100644 --- a/Gemfile +++ b/Gemfile @@ -158,6 +158,7 @@ gem 'bootsnap', require: false gem 'rails_semantic_logger', '~> 4.14' gem 'dotenv', '~> 3.1' +gem 'whenever', '~> 1.0', require: false # web server gem 'puma', '~> 5.6' @@ -178,11 +179,11 @@ group :development do # gem "rack-mini-profiler" # Deployment - gem "capistrano-resque", "~> 0.2.3", require: false - gem 'capistrano', '~> 3.19', '>= 3.19.1', require: false - gem 'capistrano-rails', '~> 1.6', '>= 1.6.3', require: false - gem 'capistrano-rbenv', '~> 2.2', require: false - gem 'capistrano3-puma', '~> 5.2', require: false + gem 'capistrano-resque', '~> 0.2.3', require: false + gem 'capistrano', '~> 3.19', require: false + gem 'capistrano-rails', '~> 1.6', require: false + gem 'capistrano-rbenv', '~> 2.2', require: false + gem 'capistrano3-puma', '~> 5.2', require: false end group :development, :test do @@ -220,5 +221,3 @@ group :test do gem 'vcr', '~> 6.0' gem 'webmock', '~> 3.8' end - - diff --git a/Gemfile.lock b/Gemfile.lock index b197066e96..3d9692a761 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -258,6 +258,7 @@ GEM capybara-screenshot (1.0.26) capybara (>= 1.0, < 4) launchy + chronic (0.10.2) chunky_png (1.4.0) cld3 (3.5.3) climate_control (0.2.0) @@ -874,6 +875,8 @@ GEM websocket-driver (0.7.6) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) + whenever (1.0.0) + chronic (>= 0.6.3) will_paginate (3.3.1) will_paginate-bootstrap (1.0.2) will_paginate (>= 3.0.3) @@ -902,8 +905,8 @@ DEPENDENCIES axe-core-capybara axe-core-cucumber bootsnap - capistrano (~> 3.19, >= 3.19.1) - capistrano-rails (~> 1.6, >= 1.6.3) + capistrano (~> 3.19) + capistrano-rails (~> 1.6) capistrano-rbenv (~> 2.2) capistrano-resque (~> 0.2.3) capistrano3-puma (~> 5.2) @@ -1016,6 +1019,7 @@ DEPENDENCIES virtus (~> 1.0.5) webdrivers (~> 5.0) webmock (~> 3.8) + whenever (~> 1.0) will_paginate (~> 3.3.0) will_paginate-bootstrap (~> 1.0.1) diff --git a/config/deploy.rb b/config/deploy.rb index b19c2e9e4c..e1f4fd04f0 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -13,7 +13,7 @@ set :rbenv_type, :user append :linked_dirs, 'log', 'tmp', 'node_modules', 'public' -append :linked_files, '.env' +append :linked_files, '.env', 'config/logindotgov.pem' set :rails_env, 'production' set :default_env, { SECRET_KEY_BASE: '1' } @@ -21,4 +21,3 @@ set :resque_environment_task, true role :resque_worker, JSON.parse(ENV.fetch('RESQUE_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] -role :resque_scheduler, JSON.parse(ENV.fetch('CRON_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index 2fd57f5e80..c86976ac9d 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -6,6 +6,7 @@ # You can define all roles on a single server, or split them: server ENV['SERVER_ADDRESS'], user: ENV['SERVER_DEPLOYMENT_USER'], roles: %w{app db web} +role :cron, JSON.parse(ENV.fetch('CRON_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] # Configuration # ============= @@ -23,6 +24,7 @@ # Global options # -------------- set :ssh_options, { + user: ENV['SERVER_DEPLOYMENT_USER'], keys: [ENV['SSH_KEY_PATH']], forward_agent: false, auth_methods: %w(publickey) @@ -41,3 +43,5 @@ set :puma_bind, "tcp://0.0.0.0:3000" set :workers, { searchgov: 1, sitemap: 1, primary: 1, '*' => 1 } + +set :whenever_roles, :cron diff --git a/config/schedule.rb b/config/schedule.rb new file mode 100644 index 0000000000..4896dd0a97 --- /dev/null +++ b/config/schedule.rb @@ -0,0 +1,15 @@ +every 1.month, roles: [:cron] do + rake 'search:reports:email_monthly_reports' +end + +every '35 21 18 12 *', roles: [:cron] do + rake 'search:reports:email_yearly_reports' +end + +every '18 9 * * 1-5', roles: [:cron] do + rake 'search:federal_register:import_agencies' +end + +every '18 9 * * 1-5', roles: [:cron] do + rake 'search:federal_register:import_documents' +end From c9790ff2fb878def22b9350927ec03ce2a48c2e5 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Tue, 27 Aug 2024 17:55:37 -0700 Subject: [PATCH 49/83] Remove .env directory --- .env/development/app | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .env/development/app diff --git a/.env/development/app b/.env/development/app deleted file mode 100644 index b758f16a23..0000000000 --- a/.env/development/app +++ /dev/null @@ -1,6 +0,0 @@ -DB_HOST=mysql -ES_HOSTS=http://elasticsearch7:9200 -REDIS_HOST=redis -I14Y_DOCKER_HOST=http://i14y:3200 -ASIS_DOCKER_HOST=http://asis:3300 -ENABLE_VOTE_ORG_LINK=false From 0ba34600f3da494d6123c6a2195983ff6962ae07 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Thu, 29 Aug 2024 10:43:51 -0700 Subject: [PATCH 50/83] SRCH-5479 Add /up route for health endpoint --- config/routes.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/config/routes.rb b/config/routes.rb index 59be3a0cbd..ce468626a6 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -32,6 +32,7 @@ get '/sayt' => 'sayt#index' post '/clicked' => 'clicked#create' get '/healthcheck' => 'health_checks#new' + get '/up' => 'health_checks#new' get '/login' => 'user_sessions#security_notification', as: :login get '/signup' => 'user_sessions#security_notification', as: :signup get '/dcv/:affiliate.txt' => 'statuses#domain_control_validation', From 6fa51d461aeaa34c5ca02ddcfc576d911f7ca058 Mon Sep 17 00:00:00 2001 From: sb-ebukaanene <98557321+sb-ebukaanene@users.noreply.github.com> Date: Thu, 29 Aug 2024 15:58:35 -0700 Subject: [PATCH 51/83] Srch-5391 codepipeline (#1638) * Create appspec.yml * Create buildspec_searchgov.yml * Update buildspec_searchgov.yml * Create BeforeInstall.sh * Update buildspec_searchgov.yml * Update buildspec_searchgov.yml * Update buildspec_searchgov.yml * Update appspec.yml * Update and rename BeforeInstall.sh to fetch_env_vars.sh * Update appspec.yml * Create deploy_with_capistrano.sh * Update deploy_with_capistrano.sh * Update appspec.yml * Update fetch_env_vars.sh * Update deploy_with_capistrano.sh * Update deploy_with_capistrano.sh * Update appspec.yml * Update fetch_env_vars.sh * Update fetch_env_vars.sh * Update fetch_env_vars.sh * Update fetch_env_vars.sh * Update fetch_env_vars.sh * Update fetch_env_vars.sh * Update buildspec_searchgov.yml * Update buildspec_searchgov.yml * Update fetch_env_vars.sh * Update fetch_env_vars.sh * Update fetch_env_vars.sh * Update fetch_env_vars.sh * Update fetch_env_vars.sh * Update fetch_env_vars.sh * Update fetch_env_vars.sh * Update fetch_env_vars.sh * Update buildspec_searchgov.yml * Update buildspec_searchgov.yml * Update buildspec_searchgov.yml * Update buildspec_searchgov.yml * Update fetch_env_vars.sh * Update fetch_env_vars.sh * Update fetch_env_vars.sh * Update fetch_env_vars.sh * removing region in the fetch param command * Update appspec.yml * Update deploy_with_capistrano.sh * added appsepc and buildspec files * added appsepc and buildspec files * modified fetch env * modified fetch env * modified fetch env * modified fetch env * modified fetch env * buildspec --- appspec.yml | 15 ++++++++ buildspec_searchgov.yml | 49 +++++++++++++++++++++++++ cicd-scripts/fetch_env_vars.sh | 66 ++++++++++++++++++++++++++++++++++ 3 files changed, 130 insertions(+) create mode 100644 appspec.yml create mode 100644 buildspec_searchgov.yml create mode 100644 cicd-scripts/fetch_env_vars.sh diff --git a/appspec.yml b/appspec.yml new file mode 100644 index 0000000000..6a90d0c3cc --- /dev/null +++ b/appspec.yml @@ -0,0 +1,15 @@ + +version: 0.0 +os: linux +# files: +# - source: / +# destination: /home/search/cicd_temp + +hooks: + + BeforeInstall: + - location: cicd-scripts/fetch_env_vars.sh + timeout: 300 + runas: search + + diff --git a/buildspec_searchgov.yml b/buildspec_searchgov.yml new file mode 100644 index 0000000000..c6312972d1 --- /dev/null +++ b/buildspec_searchgov.yml @@ -0,0 +1,49 @@ + +version: 0.2 +env: + parameter-store: + # env_used_in_app: "env_in_param_store" + SERVER_ADDRESS: "DEPLOY_SEARCHGOV_SERVER_ADDRESS " + RESQUE_SERVER_ADDRESSES: "DEPLOY_RESQUE_SERVER_ADDRESSES" + DEPLOYMENT_PATH: "DEPLOY_SEARCHGOV_DEPLOYMENT_PATH" + SERVER_DEPLOYMENT_USER: "DEPLOY_SERVER_DEPLOYMENT_USER" + # SSH_KEY_PATH: "DEPLOY_SSH_KEY_PATH" - defined below + + # shared deployment variables with subsequent stages - might not to export as this is the final stage +exported-variables: + - SERVER_ADDRESS + - RESQUE_SERVER_ADDRESSES + - DEPLOYMENT_PATH + - SERVER_DEPLOYMENT_USER + - SSH_KEY_PATH + +phases: + install: + runtime-versions: + python: 3.x + commands: + - echo "test" + # - rbenv install 3.1.4 + # - rbenv global 3.1.4 + # - rbenv rehash + - ruby -v + + pre_build: + commands: + - aws secretsmanager get-secret-value --secret-id $SEARCH_SECRETSMANAGER_KEY_SECRET_NAME --region $SEARCH_AWS_REGION --query 'SecretString' --output text > $SEARCH_ENV_EC2_KEY + build: + commands: + - CURRENT_LOCATION=$(pwd) # would look something like this - /codebuild/output/src559980389/src - a temp dir created by codebuild + - SSH_KEY_PATH="${CURRENT_LOCATION}/${SEARCH_ENV_EC2_KEY}" + - echo $SSH_KEY_PATH + - echo "deploying searchgov app with capistrano" + - bundle install + - cap $SEARCH_ENV puma:config puma:systemd:config puma:systemd:enable + - cap $SEARCH_ENV deploy + - cap $SEARCH_ENV --tasks + - cap $SEARCH_ENV resque:start + - cap $SEARCH_ENV puma:restart + +artifacts: + files: + - '**/*' diff --git a/cicd-scripts/fetch_env_vars.sh b/cicd-scripts/fetch_env_vars.sh new file mode 100644 index 0000000000..48f87c4148 --- /dev/null +++ b/cicd-scripts/fetch_env_vars.sh @@ -0,0 +1,66 @@ +#!/bin/bash +set -x +# Move to a writable location +cd /home/search/cicd_temp + +# Leave PARAM_PATH empty to fetch all parameters in the region +PARAM_PATH="" + +# Clear the .env file if it exists +> .env + +echo "Starting the script" +# Fetch all parameter names in the region +echo $AWS_REGION +if [ -n "$PARAM_PATH" ]; then + PARAM_KEYS=$(aws ssm get-parameters-by-path --path "$PARAM_PATH" --recursive --query "Parameters[*].Name" --output text) +else + PARAM_KEYS=$(aws ssm describe-parameters --query "Parameters[*].Name" --output text) +fi +echo "Fetched parameter keys: $PARAM_KEYS" + +# Loop through each parameter key +for PARAM in $PARAM_KEYS; do + # Exclude parameters that start with "DEPLOY_" or match "*_EC2_PEM_KEY" or match LOGIN_DOT_GOV_PEM + if [[ $PARAM != DEPLOY_* && ! $PARAM =~ .*_EC2_PEM_KEY$ && $PARAM != "LOGIN_DOT_GOV_PEM" ]]; then + # Fetch the parameter value from SSM + VALUE=$(aws ssm get-parameter --name "$PARAM" --with-decryption --query "Parameter.Value" --output text) + + # Rename parameters that start with "SEARCH_AWS_" to "AWS_" + if [[ $PARAM == SEARCH_AWS_* ]]; then + PARAM=${PARAM/SEARCH_AWS_/AWS_} + fi + + # Write the key=value pair to the .env file + echo "$PARAM=$VALUE" >> .env + fi +done + +# Output the result +echo ".env file created with the following content:" +cat .env +cp /home/search/cicd_temp/.env /home/search/searchgov/shared + +# Fetch a specific parameter and save it to a file +aws ssm get-parameter --name "LOGIN_DOT_GOV_PEM" --region us-east-2 --with-decryption --query "Parameter.Value" --output text > /home/search/searchgov/logindotgov.pem + +# create puma folders and files + +# Create directories if they do not already exist +[ ! -d /home/search/searchgov/shared/tmp/pids/ ] && mkdir -p /home/search/searchgov/shared/tmp/pids/ +[ ! -d /home/search/searchgov/shared/log ] && mkdir -p /home/search/searchgov/shared/log + +# Create log files if they do not already exist +[ ! -f /home/search/searchgov/shared/log/puma_access.log ] && touch /home/search/searchgov/shared/log/puma_access.log +[ ! -f /home/search/searchgov/shared/log/puma_error.log ] && touch /home/search/searchgov/shared/log/puma_error.log + + +# Set ownership and permissions +chown -R search:search /home/search/searchgov/ +chmod -R 777 /home/search/searchgov/ + +find /home/search/searchgov/ -type d -exec chmod 2777 {} \; + +umask 000 + +sudo rm -rf /home/search/cicd_temp/* From 4409bd67204ad499395699d1170769981e1293de Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Thu, 5 Sep 2024 11:06:21 -0700 Subject: [PATCH 52/83] Refactor capistrano setup * Migrate staging setup to deploy file to be applied in production as well * Deploy webserver to multiple servers --- config/deploy.rb | 39 ++++++++++++++++++++++++--------------- config/deploy/staging.rb | 12 ------------ 2 files changed, 24 insertions(+), 27 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index e1f4fd04f0..ace8ff56ea 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -1,23 +1,32 @@ # config valid for current version and patch releases of Capistrano lock '~> 3.19.1' -set :application, 'search-gov' -set :branch, :staging -set :deploy_to, ENV['DEPLOYMENT_PATH'] -set :repo_url, 'https://github.com/GSA/search-gov' -set :format, :pretty -set :user, :search - -# Use rbenv to manage Ruby versions -set :rbenv_ruby, '3.1.4' -set :rbenv_type, :user +set :application, 'search-gov' +set :branch, :staging +set :default_env, { SECRET_KEY_BASE: '1' } +set :deploy_to, ENV['DEPLOYMENT_PATH'] +set :format, :pretty +set :puma_bind, "tcp://0.0.0.0:3000" +set :rails_env, 'production' +set :rbenv_ruby, '3.1.4' +set :rbenv_type, :user +set :repo_url, 'https://github.com/GSA/search-gov' +set :resque_environment_task, true +set :user, ENV['SERVER_DEPLOYMENT_USER'] +set :whenever_roles, :cron +set :workers, { searchgov: 1, sitemap: 1, primary: 1, '*' => ENV.fetch('RESQUE_WORKERS_COUNT', '5') } append :linked_dirs, 'log', 'tmp', 'node_modules', 'public' append :linked_files, '.env', 'config/logindotgov.pem' -set :rails_env, 'production' -set :default_env, { SECRET_KEY_BASE: '1' } - -set :resque_environment_task, true +role :app, JSON.parse(ENV.fetch('APP_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] +role :cron, JSON.parse(ENV.fetch('CRON_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] +role :db, JSON.parse(ENV.fetch('APP_SERVER_ADDRESSES', '[]')).first, user: ENV['SERVER_DEPLOYMENT_USER'] +role :resque_worker, JSON.parse(ENV.fetch('RESQUE_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] -role :resque_worker, JSON.parse(ENV.fetch('RESQUE_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] +set :ssh_options, { + auth_methods: %w(publickey), + forward_agent: false, + keys: [ENV['SSH_KEY_PATH']], + user: ENV['SERVER_DEPLOYMENT_USER'], +} diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index c86976ac9d..bc89f9f015 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -6,7 +6,6 @@ # You can define all roles on a single server, or split them: server ENV['SERVER_ADDRESS'], user: ENV['SERVER_DEPLOYMENT_USER'], roles: %w{app db web} -role :cron, JSON.parse(ENV.fetch('CRON_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] # Configuration # ============= @@ -23,12 +22,6 @@ # Global options # -------------- -set :ssh_options, { - user: ENV['SERVER_DEPLOYMENT_USER'], - keys: [ENV['SSH_KEY_PATH']], - forward_agent: false, - auth_methods: %w(publickey) -} set :puma_rackup, -> { File.join(current_path, 'config.ru') } set :puma_state, "#{shared_path}/tmp/pids/puma.state" @@ -40,8 +33,3 @@ set :puma_worker_timeout, nil set :puma_init_active_record, true set :puma_preload_app, false -set :puma_bind, "tcp://0.0.0.0:3000" - -set :workers, { searchgov: 1, sitemap: 1, primary: 1, '*' => 1 } - -set :whenever_roles, :cron From 1a13078c333bedb06be02f4646627a08dd030a5d Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Thu, 5 Sep 2024 11:40:43 -0700 Subject: [PATCH 53/83] Setup assets compilation on multiple servers --- config/deploy.rb | 1 + config/deploy/staging.rb | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index ace8ff56ea..cb8e9982c5 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -23,6 +23,7 @@ role :cron, JSON.parse(ENV.fetch('CRON_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] role :db, JSON.parse(ENV.fetch('APP_SERVER_ADDRESSES', '[]')).first, user: ENV['SERVER_DEPLOYMENT_USER'] role :resque_worker, JSON.parse(ENV.fetch('RESQUE_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] +role :web, JSON.parse(ENV.fetch('APP_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] set :ssh_options, { auth_methods: %w(publickey), diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index bc89f9f015..3873cd4529 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -5,8 +5,6 @@ # Defines a single server with a list of roles and multiple properties. # You can define all roles on a single server, or split them: -server ENV['SERVER_ADDRESS'], user: ENV['SERVER_DEPLOYMENT_USER'], roles: %w{app db web} - # Configuration # ============= # You can set any configuration variable like in config/deploy.rb. From 7f0e7b3f16095da197691345746be487663f2ff9 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Fri, 6 Sep 2024 11:40:49 -0700 Subject: [PATCH 54/83] Run migrations during deployment. --- Capfile | 2 +- config/deploy.rb | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Capfile b/Capfile index d59002aaaf..c766020a37 100644 --- a/Capfile +++ b/Capfile @@ -36,7 +36,7 @@ require "capistrano/rbenv" # require "capistrano/chruby" require "capistrano/bundler" require "capistrano/rails/assets" -# require "capistrano/rails/migrations" +require "capistrano/rails/migrations" # require "capistrano/passenger" # require 'capistrano/rails' diff --git a/config/deploy.rb b/config/deploy.rb index cb8e9982c5..756dbb4d4e 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -19,11 +19,11 @@ append :linked_dirs, 'log', 'tmp', 'node_modules', 'public' append :linked_files, '.env', 'config/logindotgov.pem' -role :app, JSON.parse(ENV.fetch('APP_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] -role :cron, JSON.parse(ENV.fetch('CRON_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] -role :db, JSON.parse(ENV.fetch('APP_SERVER_ADDRESSES', '[]')).first, user: ENV['SERVER_DEPLOYMENT_USER'] -role :resque_worker, JSON.parse(ENV.fetch('RESQUE_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] -role :web, JSON.parse(ENV.fetch('APP_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] +role :app, JSON.parse(ENV.fetch('APP_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] +role :cron, JSON.parse(ENV.fetch('CRON_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] +role :db, JSON.parse(ENV.fetch('APP_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] +role :resque_worker, JSON.parse(ENV.fetch('RESQUE_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] +role :web, JSON.parse(ENV.fetch('APP_SERVER_ADDRESSES', '[]')), user: ENV['SERVER_DEPLOYMENT_USER'] set :ssh_options, { auth_methods: %w(publickey), From e1095c36470f7132683ef5c193c59a0910232579 Mon Sep 17 00:00:00 2001 From: sb-ebukaanene <98557321+sb-ebukaanene@users.noreply.github.com> Date: Fri, 6 Sep 2024 15:57:43 -0700 Subject: [PATCH 55/83] Update buildspec_searchgov.yml (#1640) * Update buildspec_searchgov.yml * Update buildspec_searchgov.yml --- buildspec_searchgov.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildspec_searchgov.yml b/buildspec_searchgov.yml index c6312972d1..ce5d4ae338 100644 --- a/buildspec_searchgov.yml +++ b/buildspec_searchgov.yml @@ -3,7 +3,7 @@ version: 0.2 env: parameter-store: # env_used_in_app: "env_in_param_store" - SERVER_ADDRESS: "DEPLOY_SEARCHGOV_SERVER_ADDRESS " + APP_SERVER_ADDRESSES: "DEPLOY_SEARCHGOV_SERVER_ADDRESS " RESQUE_SERVER_ADDRESSES: "DEPLOY_RESQUE_SERVER_ADDRESSES" DEPLOYMENT_PATH: "DEPLOY_SEARCHGOV_DEPLOYMENT_PATH" SERVER_DEPLOYMENT_USER: "DEPLOY_SERVER_DEPLOYMENT_USER" @@ -11,7 +11,7 @@ env: # shared deployment variables with subsequent stages - might not to export as this is the final stage exported-variables: - - SERVER_ADDRESS + - APP_SERVER_ADDRESSES - RESQUE_SERVER_ADDRESSES - DEPLOYMENT_PATH - SERVER_DEPLOYMENT_USER From f60fad81feb9bd582b4ed8be328e0d05f6222985 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Tue, 10 Sep 2024 08:42:20 -0700 Subject: [PATCH 56/83] Remove extra configurations --- config/deploy.rb | 2 +- config/deploy/staging.rb | 11 ----------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/config/deploy.rb b/config/deploy.rb index 756dbb4d4e..9616ac597e 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -14,7 +14,7 @@ set :resque_environment_task, true set :user, ENV['SERVER_DEPLOYMENT_USER'] set :whenever_roles, :cron -set :workers, { searchgov: 1, sitemap: 1, primary: 1, '*' => ENV.fetch('RESQUE_WORKERS_COUNT', '5') } +set :workers, { '*' => ENV.fetch('RESQUE_WORKERS_COUNT', '5') } append :linked_dirs, 'log', 'tmp', 'node_modules', 'public' append :linked_files, '.env', 'config/logindotgov.pem' diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index 3873cd4529..af74c8d2fa 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -20,14 +20,3 @@ # Global options # -------------- - -set :puma_rackup, -> { File.join(current_path, 'config.ru') } -set :puma_state, "#{shared_path}/tmp/pids/puma.state" -set :puma_pid, "#{shared_path}/tmp/pids/puma.pid" -set :puma_access_log, "#{release_path}/log/puma.access.log" -set :puma_error_log, "#{release_path}/log/puma.error.log" -set :puma_threads, [0, 8] -set :puma_workers, 0 -set :puma_worker_timeout, nil -set :puma_init_active_record, true -set :puma_preload_app, false From 761348c6e88a9655aae623da61a256b905bc3e0f Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Tue, 10 Sep 2024 09:10:54 -0700 Subject: [PATCH 57/83] Setup puma logs * as they where before --- config/deploy.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/deploy.rb b/config/deploy.rb index 9616ac597e..643c79011e 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -6,7 +6,9 @@ set :default_env, { SECRET_KEY_BASE: '1' } set :deploy_to, ENV['DEPLOYMENT_PATH'] set :format, :pretty -set :puma_bind, "tcp://0.0.0.0:3000" +set :puma_access_log, "#{release_path}/log/puma.access.log" +set :puma_bind, 'tcp://0.0.0.0:3300' +set :puma_error_log, "#{release_path}/log/puma.error.log" set :rails_env, 'production' set :rbenv_ruby, '3.1.4' set :rbenv_type, :user From 853bad023c71c87d81c7dfa1c08198160fe7aab2 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Tue, 10 Sep 2024 09:36:44 -0700 Subject: [PATCH 58/83] Fix rubocop offences --- Capfile | 11 +++++------ Gemfile | 2 ++ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Capfile b/Capfile index c766020a37..81161c683a 100644 --- a/Capfile +++ b/Capfile @@ -29,14 +29,14 @@ install_plugin Capistrano::SCM::Git # https://github.com/capistrano/passenger # # require "capistrano/rvm" -require "capistrano/rbenv" +require 'capistrano/rbenv' # set :rbenv_prefix, "env RBENV_ROOT=#{fetch(:rbenv_path)} RBENV_VERSION=#{fetch(:rbenv_ruby)} #{fetch(:rbenv_path)}/bin/rbenv exec" # require "capistrano/chruby" -require "capistrano/bundler" -require "capistrano/rails/assets" -require "capistrano/rails/migrations" +require 'capistrano/bundler' +require 'capistrano/rails/assets' +require 'capistrano/rails/migrations' # require "capistrano/passenger" # require 'capistrano/rails' @@ -44,8 +44,7 @@ require "capistrano/rails/migrations" require 'capistrano/puma' require 'capistrano/puma/workers' - -install_plugin Capistrano::Puma # Default puma tasks +install_plugin Capistrano::Puma # Default puma tasks install_plugin Capistrano::Puma::Systemd require 'capistrano-resque' diff --git a/Gemfile b/Gemfile index 79383c2aca..731ce5e42c 100644 --- a/Gemfile +++ b/Gemfile @@ -1,3 +1,5 @@ +# frozen_string_literal: true + source 'https://rubygems.org' gem 'rails', '~> 7.0.8' From 98e64f1ce606dd133b0e01639b5781e3e3b60754 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Tue, 10 Sep 2024 12:38:18 -0700 Subject: [PATCH 59/83] Use the same environment variable acrosss apps --- config/database.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/database.yml b/config/database.yml index 075ee9782d..265f1dc2f8 100644 --- a/config/database.yml +++ b/config/database.yml @@ -27,7 +27,7 @@ development: production: <<: *DEFAULT database: <%= ENV['DB_NAME'] %> - username: <%= ENV['DB_USER'] %> + username: <%= ENV['DB_USERNAME'] %> password: <%= ENV['DB_PASSWORD'] %> host: <%= ENV['DB_HOST'] %> port: <%= ENV['DB_PORT'] %> From d69156ad3820d4dadbcaadc99649ff159a6f357d Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Tue, 10 Sep 2024 13:04:08 -0700 Subject: [PATCH 60/83] Refactor lib/es.rb --- lib/es.rb | 38 +++----------------------------------- 1 file changed, 3 insertions(+), 35 deletions(-) diff --git a/lib/es.rb b/lib/es.rb index 452a61c7aa..9f280188e6 100644 --- a/lib/es.rb +++ b/lib/es.rb @@ -10,24 +10,16 @@ module Es ).deep_symbolize_keys.freeze def client_reader - @client_reader ||= initialize_client(reader_config) + @client_reader ||= initialize_client end def client_writers - @client_writers ||= writer_config.map { |config| initialize_client(config) } + @client_writers ||= [initialize_client] end private - def reader_config - client_config(:reader) || {} - end - - def writer_config - client_config(:writers) || [{}] - end - - def initialize_client(config) + def initialize_client(config = {}) Elasticsearch::Client.new(config.merge(CLIENT_CONFIG)).tap do |client| client.transport.logger = Rails.logger.clone client.transport.logger.formatter = proc do |severity, time, _progname, msg| @@ -38,33 +30,9 @@ def initialize_client(config) module ELK extend Es - - private - - def self.client_config(mode) - config = { - hosts: ENV['ES_HOSTS'] ? JSON.parse(ENV['ES_HOSTS']) : Rails.application.secrets.dig(:analytics, :elasticsearch, mode, :hosts), - user: ENV['ES_USER'] || Rails.application.secrets.dig(:analytics, :elasticsearch, mode, :user), - password: ENV['ES_PASSWORD'] || Rails.application.secrets.dig(:analytics, :elasticsearch, mode, :password) - }.compact - - config.freeze - end end module CustomIndices extend Es - - private - - def self.client_config(mode) - config = { - hosts: ENV['ES_HOSTS'] ? JSON.parse(ENV['ES_HOSTS']) : Rails.application.secrets.dig(:custom_indices, :elasticsearch, mode, :hosts), - user: ENV['ES_USER'] || Rails.application.secrets.dig(:custom_indices, :elasticsearch, mode, :user), - password: ENV['ES_PASSWORD'] || Rails.application.secrets.dig(:custom_indices, :elasticsearch, mode, :password) - }.compact - - config.freeze - end end end From 8fd4499a9cccb53dd64e1cf297492f0f3dab5f46 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Tue, 10 Sep 2024 13:21:48 -0700 Subject: [PATCH 61/83] Remove aws-sdk-ssm gem --- Gemfile | 2 -- Gemfile.lock | 6 +----- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/Gemfile b/Gemfile index bc0e53f220..f06a454997 100644 --- a/Gemfile +++ b/Gemfile @@ -3,7 +3,6 @@ source 'https://rubygems.org' gem 'rails', '~> 7.0.8' -gem 'dotenv', '~> 3.1', '>= 3.1.2' gem 'base64', '~> 0.2.0' @@ -36,7 +35,6 @@ gem 'resque-scheduler', '~> 4.10.2' # Using a third-party fork as an interim measure. gem 'kt-paperclip', '~> 7.1.0' gem 'aws-sdk-s3', '~> 1.102.0' -gem 'aws-sdk-ssm', '~> 1.173' gem 'googlecharts', '~> 1.6.12' gem 'flickraw', '~> 0.9.9' gem 'mutex_m', '~> 0.2.0' diff --git a/Gemfile.lock b/Gemfile.lock index dc07f5261b..2130973eca 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -196,9 +196,6 @@ GEM aws-sdk-core (~> 3, >= 3.120.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.4) - aws-sdk-ssm (1.173.0) - aws-sdk-core (~> 3, >= 3.201.0) - aws-sigv4 (~> 1.5) aws-sigv4 (1.8.0) aws-eventstream (~> 1, >= 1.0.2) axe-core-api (4.7.0) @@ -900,7 +897,6 @@ DEPENDENCIES american_date (~> 1.1.1) authlogic (~> 6.4.1) aws-sdk-s3 (~> 1.102.0) - aws-sdk-ssm (~> 1.173) axe-core-capybara axe-core-cucumber base64 (~> 0.2.0) @@ -927,7 +923,7 @@ DEPENDENCIES debug dogapi (~> 1.45) dogstatsd-ruby (~> 3.2.0) - dotenv (~> 3.1, >= 3.1.2) + dotenv (~> 3.1) elasticsearch! elasticsearch-xpack (~> 7.4.0) email_spec (~> 2.2) From 88c61735961d218457bc012740b43f80643082f3 Mon Sep 17 00:00:00 2001 From: sb-ebukaanene <98557321+sb-ebukaanene@users.noreply.github.com> Date: Tue, 10 Sep 2024 15:03:26 -0700 Subject: [PATCH 62/83] Update buildspec_searchgov.yml (#1643) * Update buildspec_searchgov.yml * Update fetch_env_vars.sh --- buildspec_searchgov.yml | 4 ++-- cicd-scripts/fetch_env_vars.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/buildspec_searchgov.yml b/buildspec_searchgov.yml index ce5d4ae338..9a29e0cea9 100644 --- a/buildspec_searchgov.yml +++ b/buildspec_searchgov.yml @@ -22,11 +22,11 @@ phases: runtime-versions: python: 3.x commands: - - echo "test" # - rbenv install 3.1.4 # - rbenv global 3.1.4 # - rbenv rehash - - ruby -v + # - ruby -v + - echo "checking runtime" pre_build: commands: diff --git a/cicd-scripts/fetch_env_vars.sh b/cicd-scripts/fetch_env_vars.sh index 48f87c4148..7ccd53c9cd 100644 --- a/cicd-scripts/fetch_env_vars.sh +++ b/cicd-scripts/fetch_env_vars.sh @@ -39,7 +39,7 @@ done # Output the result echo ".env file created with the following content:" cat .env -cp /home/search/cicd_temp/.env /home/search/searchgov/shared +cp /home/search/cicd_temp/.env /home/search/searchgov/shared/ # Fetch a specific parameter and save it to a file aws ssm get-parameter --name "LOGIN_DOT_GOV_PEM" --region us-east-2 --with-decryption --query "Parameter.Value" --output text > /home/search/searchgov/logindotgov.pem From 7ce3a85c7d7e4d3a8d0e17a11099e9e72821d91a Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Tue, 10 Sep 2024 13:26:57 -0700 Subject: [PATCH 63/83] Deploy branch from environment variable --- config/deploy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy.rb b/config/deploy.rb index 643c79011e..32260393c9 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -2,7 +2,7 @@ lock '~> 3.19.1' set :application, 'search-gov' -set :branch, :staging +set :branch, ENV.fetch('SEARCH_ENV', 'staging') set :default_env, { SECRET_KEY_BASE: '1' } set :deploy_to, ENV['DEPLOYMENT_PATH'] set :format, :pretty From 11e89027733dc983a57ef2ffbc7b4580c68796a7 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Wed, 11 Sep 2024 10:42:28 -0700 Subject: [PATCH 64/83] Make sure number of resque workers is an integer --- config/deploy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy.rb b/config/deploy.rb index 32260393c9..b98d84322f 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -16,7 +16,7 @@ set :resque_environment_task, true set :user, ENV['SERVER_DEPLOYMENT_USER'] set :whenever_roles, :cron -set :workers, { '*' => ENV.fetch('RESQUE_WORKERS_COUNT', '5') } +set :workers, { '*' => ENV.fetch('RESQUE_WORKERS_COUNT', '5').to_i } append :linked_dirs, 'log', 'tmp', 'node_modules', 'public' append :linked_files, '.env', 'config/logindotgov.pem' From 7bfddf6c8da4dd75d2094dec3cc62ab484a50ebb Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Wed, 11 Sep 2024 10:50:11 -0700 Subject: [PATCH 65/83] Use DB_USER instead of DB_USERNAME --- config/database.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/database.yml b/config/database.yml index 265f1dc2f8..075ee9782d 100644 --- a/config/database.yml +++ b/config/database.yml @@ -27,7 +27,7 @@ development: production: <<: *DEFAULT database: <%= ENV['DB_NAME'] %> - username: <%= ENV['DB_USERNAME'] %> + username: <%= ENV['DB_USER'] %> password: <%= ENV['DB_PASSWORD'] %> host: <%= ENV['DB_HOST'] %> port: <%= ENV['DB_PORT'] %> From 075e00ad6f35c55fd9cce819ac7661f664a45a71 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Wed, 11 Sep 2024 11:34:50 -0700 Subject: [PATCH 66/83] Setup consider_all_requests_local for staging --- config/environments/production.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/environments/production.rb b/config/environments/production.rb index dbc186d049..b5d2b8f60d 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -109,6 +109,8 @@ # config.active_record.database_selector = { delay: 2.seconds } # config.active_record.database_resolver = ActiveRecord::Middleware::DatabaseSelector::Resolver # config.active_record.database_resolver_context = ActiveRecord::Middleware::DatabaseSelector::Resolver::Session + + config.consider_all_requests_local = ENV['LOCAL_REQUEST'].present? end ADDITIONAL_BING_PARAMS = {} From f25e18c481f583263f7be34d40ca1401fcb52e8a Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Wed, 11 Sep 2024 12:13:39 -0700 Subject: [PATCH 67/83] Capfile comments cleanup --- Capfile | 30 +----------------------------- 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/Capfile b/Capfile index 81161c683a..00b4c4af92 100644 --- a/Capfile +++ b/Capfile @@ -1,50 +1,22 @@ # frozen_string_literal: true -# Load DSL and set up stages require 'capistrano/setup' -# Include default deployment tasks require 'capistrano/deploy' -# Load the SCM plugin appropriate to your project: -# -# require "capistrano/scm/hg" -# install_plugin Capistrano::SCM::Hg -# or -# require "capistrano/scm/svn" -# install_plugin Capistrano::SCM::Svn -# or require 'capistrano/scm/git' install_plugin Capistrano::SCM::Git -# Include tasks from other gems included in your Gemfile -# -# For documentation on these, see for example: -# -# https://github.com/capistrano/rvm -# https://github.com/capistrano/rbenv -# https://github.com/capistrano/chruby -# https://github.com/capistrano/bundler -# https://github.com/capistrano/rails -# https://github.com/capistrano/passenger -# -# require "capistrano/rvm" require 'capistrano/rbenv' -# set :rbenv_prefix, "env RBENV_ROOT=#{fetch(:rbenv_path)} RBENV_VERSION=#{fetch(:rbenv_ruby)} #{fetch(:rbenv_path)}/bin/rbenv exec" - -# require "capistrano/chruby" require 'capistrano/bundler' require 'capistrano/rails/assets' require 'capistrano/rails/migrations' -# require "capistrano/passenger" -# require 'capistrano/rails' -# Web server plugins require 'capistrano/puma' require 'capistrano/puma/workers' -install_plugin Capistrano::Puma # Default puma tasks +install_plugin Capistrano::Puma install_plugin Capistrano::Puma::Systemd require 'capistrano-resque' From 2b8a58869588bec840910b23c39c3837e6b70ac9 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Wed, 11 Sep 2024 12:14:58 -0700 Subject: [PATCH 68/83] Remove specific production capistrano config --- config/deploy/production.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/deploy/production.rb b/config/deploy/production.rb index b1fd182010..9ac5b511fd 100644 --- a/config/deploy/production.rb +++ b/config/deploy/production.rb @@ -5,7 +5,7 @@ # Defines a single server with a list of roles and multiple properties. # You can define all roles on a single server, or split them: -server "production.server.com", user: "deploy", roles: %w{app db web} +# server "production.server.com", user: "deploy", roles: %w{app db web} # Role-based syntax # ================== @@ -25,8 +25,8 @@ # http://capistranorb.com/documentation/getting-started/configuration/ # Feel free to add new variables to customize your setup. -set :rails_env, 'production' -set :aws_ssm_path, '/your/application/env/production/' +# set :rails_env, 'production' +# set :aws_ssm_path, '/your/application/env/production/' # Custom SSH Options # ================== From c6dbe69d25222b72fa2e3c66371d599495b315b4 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Wed, 11 Sep 2024 12:30:57 -0700 Subject: [PATCH 69/83] Put back resque_scheduler --- config/resque_schedule.yml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/config/resque_schedule.yml b/config/resque_schedule.yml index f8afe29e3d..9fe37a2648 100644 --- a/config/resque_schedule.yml +++ b/config/resque_schedule.yml @@ -1,13 +1,5 @@ production: - usesearch:reports:email_monthly_reports: - cron: "0 0 1 * *" - class: "Resque::Job" - usasearch:reports:email_yearly_reports: - cron: "35 21 18 12 *" - class: "Resque::Job" - usasearch:federal_register:import_agencies - cron: "18 9 * * 1-5" - class: "Resque::Job" - usasearch:federal_register:import_documents: - cron: "18 9 * * 1-5" - class: "Resque::Job" + SitemapMonitorJob: + cron: "0 9 * * *" + ClickMonitorJob: + cron: "0 7 * * 0" From b8b521c5a79623cac184b9e0796fa7829cc3973d Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Wed, 11 Sep 2024 13:02:16 -0700 Subject: [PATCH 70/83] Put back /app/assets/builds/.keep file --- app/assets/builds/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 app/assets/builds/.keep diff --git a/app/assets/builds/.keep b/app/assets/builds/.keep new file mode 100644 index 0000000000..e69de29bb2 From 3ae3a9c18fb9bebadc41d7314cdf01bf3a575607 Mon Sep 17 00:00:00 2001 From: sb-ebukaanene <98557321+sb-ebukaanene@users.noreply.github.com> Date: Wed, 11 Sep 2024 13:02:49 -0700 Subject: [PATCH 71/83] Cicd fix bundler (#1645) * Update buildspec_searchgov.yml * Update buildspec_searchgov.yml * Update buildspec_searchgov.yml * Update buildspec_searchgov.yml * Update buildspec_searchgov.yml --- buildspec_searchgov.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/buildspec_searchgov.yml b/buildspec_searchgov.yml index 9a29e0cea9..f7775e4eb2 100644 --- a/buildspec_searchgov.yml +++ b/buildspec_searchgov.yml @@ -3,7 +3,7 @@ version: 0.2 env: parameter-store: # env_used_in_app: "env_in_param_store" - APP_SERVER_ADDRESSES: "DEPLOY_SEARCHGOV_SERVER_ADDRESS " + APP_SERVER_ADDRESSES: "DEPLOY_SEARCHGOV_SERVER_ADDRESS" RESQUE_SERVER_ADDRESSES: "DEPLOY_RESQUE_SERVER_ADDRESSES" DEPLOYMENT_PATH: "DEPLOY_SEARCHGOV_DEPLOYMENT_PATH" SERVER_DEPLOYMENT_USER: "DEPLOY_SERVER_DEPLOYMENT_USER" @@ -22,11 +22,9 @@ phases: runtime-versions: python: 3.x commands: - # - rbenv install 3.1.4 - # - rbenv global 3.1.4 - # - rbenv rehash - # - ruby -v - - echo "checking runtime" + - export PATH="$HOME/.rbenv/bin:$PATH" + - eval "$(rbenv init -)" + pre_build: commands: From bf3a691fa39ab758eeec24ef6bfbd304544b456c Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Wed, 11 Sep 2024 13:29:37 -0700 Subject: [PATCH 72/83] Fix capistrano puma port --- config/deploy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/deploy.rb b/config/deploy.rb index b98d84322f..e909fb75de 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -7,7 +7,7 @@ set :deploy_to, ENV['DEPLOYMENT_PATH'] set :format, :pretty set :puma_access_log, "#{release_path}/log/puma.access.log" -set :puma_bind, 'tcp://0.0.0.0:3300' +set :puma_bind, 'tcp://0.0.0.0:3000' set :puma_error_log, "#{release_path}/log/puma.error.log" set :rails_env, 'production' set :rbenv_ruby, '3.1.4' From 67e82e02e208b17f55ab717ad862d11bdd1c6c8b Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Wed, 11 Sep 2024 15:16:37 -0700 Subject: [PATCH 73/83] Fix session redis host environment variables --- config/initializers/session_store.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/initializers/session_store.rb b/config/initializers/session_store.rb index 0cd3a5df02..d62d1011c1 100644 --- a/config/initializers/session_store.rb +++ b/config/initializers/session_store.rb @@ -3,8 +3,8 @@ secure: Rails.application.config.ssl_options[:secure_cookies], key: '_usasearch_session', servers: { - host: ENV['REDIS_HOST'], - port: ENV['REDIS_PORT'], + host: ENV['REDIS_SESSION_HOST'], + port: ENV['REDIS_SESSION_PORT'], db: 2, key_prefix: 'usasearch:session' } From 79a628eeed7abaed3d6cabc0035a9eb11a8fe5cf Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Thu, 12 Sep 2024 14:27:28 -0700 Subject: [PATCH 74/83] Make sure resque gets restart after deploy --- config/deploy.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/deploy.rb b/config/deploy.rb index e909fb75de..b9d990266a 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -33,3 +33,5 @@ keys: [ENV['SSH_KEY_PATH']], user: ENV['SERVER_DEPLOYMENT_USER'], } + +after 'deploy:finished', 'resque:restart' From fa2afe3210f78d51b1c1355938c3c4be4e0bfa00 Mon Sep 17 00:00:00 2001 From: sb-ebukaanene <98557321+sb-ebukaanene@users.noreply.github.com> Date: Thu, 12 Sep 2024 15:05:52 -0700 Subject: [PATCH 75/83] Update buildspec_searchgov.yml (#1642) --- buildspec_searchgov.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/buildspec_searchgov.yml b/buildspec_searchgov.yml index f7775e4eb2..87b0abdafc 100644 --- a/buildspec_searchgov.yml +++ b/buildspec_searchgov.yml @@ -7,6 +7,7 @@ env: RESQUE_SERVER_ADDRESSES: "DEPLOY_RESQUE_SERVER_ADDRESSES" DEPLOYMENT_PATH: "DEPLOY_SEARCHGOV_DEPLOYMENT_PATH" SERVER_DEPLOYMENT_USER: "DEPLOY_SERVER_DEPLOYMENT_USER" + RESQUE_WORKERS_COUNT: "DEPLOY_RESQUE_WORKERS_COUNT" # SSH_KEY_PATH: "DEPLOY_SSH_KEY_PATH" - defined below # shared deployment variables with subsequent stages - might not to export as this is the final stage From 2845cbf27d6dc1c1b0d37c9a52fce4041539848e Mon Sep 17 00:00:00 2001 From: sb-ebukaanene <98557321+sb-ebukaanene@users.noreply.github.com> Date: Fri, 13 Sep 2024 14:22:27 -0700 Subject: [PATCH 76/83] Update fetch_env_vars.sh --- cicd-scripts/fetch_env_vars.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cicd-scripts/fetch_env_vars.sh b/cicd-scripts/fetch_env_vars.sh index 7ccd53c9cd..40e14c35f4 100644 --- a/cicd-scripts/fetch_env_vars.sh +++ b/cicd-scripts/fetch_env_vars.sh @@ -11,11 +11,12 @@ PARAM_PATH="" echo "Starting the script" # Fetch all parameter names in the region -echo $AWS_REGION +REGION=$(curl -s http://169.254.169.254/latest/meta-data/placement/region) +echo $REGION if [ -n "$PARAM_PATH" ]; then - PARAM_KEYS=$(aws ssm get-parameters-by-path --path "$PARAM_PATH" --recursive --query "Parameters[*].Name" --output text) + PARAM_KEYS=$(aws ssm get-parameters-by-path --path "$PARAM_PATH" --recursive --query "Parameters[*].Name" --output text --region $REGION) else - PARAM_KEYS=$(aws ssm describe-parameters --query "Parameters[*].Name" --output text) + PARAM_KEYS=$(aws ssm describe-parameters --query "Parameters[*].Name" --output text --region $REGION) fi echo "Fetched parameter keys: $PARAM_KEYS" @@ -24,7 +25,7 @@ for PARAM in $PARAM_KEYS; do # Exclude parameters that start with "DEPLOY_" or match "*_EC2_PEM_KEY" or match LOGIN_DOT_GOV_PEM if [[ $PARAM != DEPLOY_* && ! $PARAM =~ .*_EC2_PEM_KEY$ && $PARAM != "LOGIN_DOT_GOV_PEM" ]]; then # Fetch the parameter value from SSM - VALUE=$(aws ssm get-parameter --name "$PARAM" --with-decryption --query "Parameter.Value" --output text) + VALUE=$(aws ssm get-parameter --name "$PARAM" --with-decryption --query "Parameter.Value" --output text --region $REGION) # Rename parameters that start with "SEARCH_AWS_" to "AWS_" if [[ $PARAM == SEARCH_AWS_* ]]; then From 94692329025894850b9796d646c332b55b8592e8 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Fri, 13 Sep 2024 15:50:55 -0700 Subject: [PATCH 77/83] Setup host and port for sidekiq --- config/initializers/sidekiq.rb | 4 ++-- config/sidekiq.yml | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index e0606f71c2..0203f127ce 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -3,9 +3,9 @@ sidekiq = Rails.application.config_for(:sidekiq) Sidekiq.configure_server do |config| - config.redis = { url: sidekiq['url'] } + config.redis = { host: sidekiq['host'], port: sidekiq['port'] } end Sidekiq.configure_client do |config| - config.redis = { url: sidekiq['url'] } + config.redis = { host: sidekiq['host'], port: sidekiq['port'] } end diff --git a/config/sidekiq.yml b/config/sidekiq.yml index c02cf92d34..ec382d59ed 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -1,6 +1,7 @@ default: &DEFAULT namespace: oasis - url: <%= ENV['REDIS_HOST'] || 'redis://localhost:6379' %> + host: <%= ENV.fetch('REDIS_HOST', 'localhost') %> + port: <%= ENV.fetch('REDIS_PORT', '6379') %> development: <<: *DEFAULT From 76c7f2af46721881f46da54a8bebff1fdc650fb3 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Mon, 16 Sep 2024 10:45:50 -0700 Subject: [PATCH 78/83] Log search impression data on its own key --- app/models/search_impression.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/search_impression.rb b/app/models/search_impression.rb index a4f08facf0..040312e751 100644 --- a/app/models/search_impression.rb +++ b/app/models/search_impression.rb @@ -16,7 +16,7 @@ def self.log(search, vertical, params, request) modules: search.modules.join('|'), params: clean_params(params)) - Rails.logger.info("[Search Impression] #{hash.to_json}") + Rails.logger.info('[Search Impression]', search_data: hash) end def self.clean_params(params) From 321df4d7f6e3e9a1772587362c6449c6aff7afdb Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Mon, 16 Sep 2024 10:58:45 -0700 Subject: [PATCH 79/83] Log clicks data on its own key, same as impressions --- app/models/click.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/click.rb b/app/models/click.rb index 0eba046283..39c35217c8 100644 --- a/app/models/click.rb +++ b/app/models/click.rb @@ -34,7 +34,7 @@ def initialize(params) end def log - Rails.logger.info("[Click] #{click_hash.to_json}") + Rails.logger.info('[Click]', search_data: click_hash) end private From 5981d178fdbde007f556a599b02143e882aafbfc Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Mon, 16 Sep 2024 12:20:01 -0700 Subject: [PATCH 80/83] Log impression and clicks to its own file --- app/models/click.rb | 5 +++++ app/models/search_impression.rb | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/app/models/click.rb b/app/models/click.rb index 39c35217c8..ad882a2eec 100644 --- a/app/models/click.rb +++ b/app/models/click.rb @@ -35,10 +35,15 @@ def initialize(params) def log Rails.logger.info('[Click]', search_data: click_hash) + clicks_logger.info(click_hash) end private + def clicks_logger + @@logger ||= Logger.new('log/clicks.log') + end + # prevent validation from choking on "invalid byte sequence in UTF-8" def validate_url_encoding return unless url diff --git a/app/models/search_impression.rb b/app/models/search_impression.rb index 040312e751..fbed6a6024 100644 --- a/app/models/search_impression.rb +++ b/app/models/search_impression.rb @@ -17,6 +17,7 @@ def self.log(search, vertical, params, request) params: clean_params(params)) Rails.logger.info('[Search Impression]', search_data: hash) + impression_logger.info(hash) end def self.clean_params(params) @@ -37,4 +38,8 @@ def self.get_url_from_request(request) def self.flatten_diagnostics_hash(diagnostics_hash) diagnostics_hash.keys.sort.map { |k| diagnostics_hash[k].merge(module: k) } end + + def impression_logger + @@logger ||= Logger.new('log/impressions.log') + end end From 7ebc2b8927fc78edcc49bd94008c75af0a873f8d Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Mon, 16 Sep 2024 12:25:28 -0700 Subject: [PATCH 81/83] Fix search impression --- app/models/search_impression.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/search_impression.rb b/app/models/search_impression.rb index fbed6a6024..c3ac78aee4 100644 --- a/app/models/search_impression.rb +++ b/app/models/search_impression.rb @@ -39,7 +39,7 @@ def self.flatten_diagnostics_hash(diagnostics_hash) diagnostics_hash.keys.sort.map { |k| diagnostics_hash[k].merge(module: k) } end - def impression_logger + def self.impression_logger @@logger ||= Logger.new('log/impressions.log') end end From 21b789f1ba65f2d14305773b8a8bc31045c5b408 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Mon, 16 Sep 2024 12:50:23 -0700 Subject: [PATCH 82/83] Log click and search impressions the same way as before --- app/models/click.rb | 2 +- app/models/search_impression.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/click.rb b/app/models/click.rb index ad882a2eec..92b5464892 100644 --- a/app/models/click.rb +++ b/app/models/click.rb @@ -34,8 +34,8 @@ def initialize(params) end def log + clicks_logger.info("[Click] #{click_hash.to_json}") Rails.logger.info('[Click]', search_data: click_hash) - clicks_logger.info(click_hash) end private diff --git a/app/models/search_impression.rb b/app/models/search_impression.rb index c3ac78aee4..204f518873 100644 --- a/app/models/search_impression.rb +++ b/app/models/search_impression.rb @@ -16,8 +16,8 @@ def self.log(search, vertical, params, request) modules: search.modules.join('|'), params: clean_params(params)) + impression_logger.info("[Search Impression] #{hash.to_json}") Rails.logger.info('[Search Impression]', search_data: hash) - impression_logger.info(hash) end def self.clean_params(params) From 7b2dca4f584d0d0402f8bd315f4ea52a6e218b77 Mon Sep 17 00:00:00 2001 From: Steven Barragan Naranjo Date: Tue, 17 Sep 2024 12:10:42 -0700 Subject: [PATCH 83/83] Use REDIS_SYSTEM_URL instead of REDIS_HOST and REDIS_PORT --- app/models/rtu_dashboard.rb | 2 +- app/models/trending_url.rb | 2 +- config/initializers/load_resque.rb | 5 +---- config/sidekiq.yml | 3 +-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/app/models/rtu_dashboard.rb b/app/models/rtu_dashboard.rb index c0ea7003b2..2328765d21 100644 --- a/app/models/rtu_dashboard.rb +++ b/app/models/rtu_dashboard.rb @@ -29,7 +29,7 @@ def top_urls end def trending_urls - redis = Redis.new(:host => REDIS_HOST, :port => REDIS_PORT) + redis = Redis.new(url: ENV.fetch('REDIS_SYSTEM_URL')) trending_urls_key = ['TrendingUrls', @site.name].join(':') redis.smembers(trending_urls_key) end diff --git a/app/models/trending_url.rb b/app/models/trending_url.rb index bd1d37075d..000b7e732e 100644 --- a/app/models/trending_url.rb +++ b/app/models/trending_url.rb @@ -6,7 +6,7 @@ class TrendingUrl attr_accessor :affiliate, :url cattr_reader :redis - @@redis = Redis.new(:host => REDIS_HOST, :port => REDIS_PORT) + @@redis = Redis.new(url: ENV.fetch('REDIS_SYSTEM_URL')) def self.all sorted_trending_affiliate_keys = @@redis.keys("TrendingUrls:*").sort diff --git a/config/initializers/load_resque.rb b/config/initializers/load_resque.rb index e50217cae1..e070d5b34c 100644 --- a/config/initializers/load_resque.rb +++ b/config/initializers/load_resque.rb @@ -7,10 +7,7 @@ require 'resque/server' require 'resque/job_timeout' -host = ENV['REDIS_HOST'] -port = ENV['REDIS_PORT'] - -Resque.redis = "#{host}:#{port}" +Resque.redis = ENV.fetch('REDIS_SYSTEM_URL') Resque::Failure::Multiple.classes = [Resque::Failure::Redis] Resque::Failure.backend = Resque::Failure::Multiple diff --git a/config/sidekiq.yml b/config/sidekiq.yml index ec382d59ed..cd2456e415 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -1,7 +1,6 @@ default: &DEFAULT namespace: oasis - host: <%= ENV.fetch('REDIS_HOST', 'localhost') %> - port: <%= ENV.fetch('REDIS_PORT', '6379') %> + url: <%= ENV.fetch('REDIS_SYSTEM_URL', 'localhost:6379') %> development: <<: *DEFAULT