Merge pull request #15762 from CartoDB/feature/ch91595/set-up-cartodb…

…-to-start-using-the-new-carto

Install new Carto::Logger

Gemfile

@@ -69,7 +69,7 @@ gem 'google-cloud-pubsub', '1.2.0'
 
 # Service components (/services)
 gem 'virtus',                   '1.0.5'
-gem 'cartodb-common',           git: 'https://github.com/cartodb/cartodb-common.git', tag: 'v0.2.1'
+gem 'cartodb-common', git: 'https://github.com/cartodb/cartodb-common.git', tag: 'v0.3.2'
 gem 'email_address',            '~> 0.1.11'
 
 # Markdown

Gemfile.lock

@@ -8,10 +8,11 @@ GIT
 
 GIT
   remote: https://github.com/cartodb/cartodb-common.git
-  revision: 90e50effa1c49b5eb35aa27279bd7b74e7e0314c
-  tag: v0.2.1
+  revision: 5fb23da8a3fe048b742e23b02362b25209e02e81
+  tag: v0.3.2
   specs:
-    cartodb-common (0.2.1)
+    cartodb-common (0.3.0)
+      activesupport (~> 4.2.11.3)
       argon2 (~> 2.0)
 
 GIT
@@ -563,4 +564,4 @@ DEPENDENCIES
   zeus
 
 BUNDLED WITH
-   1.17.3
+   1.17.3

NEWS.md

@@ -16,6 +16,7 @@ Development
 - Fix imports from query that contain `(sql_expression)::cast` ([#15765](https://github.com/CartoDB/cartodb/pull/15765))
 - Fix wrong popup position, via new internal carto.js version 4.2.2-1 ([CARTO.js#2254](https://github.com/CartoDB/carto.js/pull/2254))
 - Modify .gitignore
+- Install Carto::Common::Logger with JSON support ([#15762](https://github.com/CartoDB/cartodb/pull/15762))
 
 4.39.0 (2020-07-20)
 -------------------

config/application.rb

@@ -218,6 +218,10 @@ class Application < Rails::Application
 
     # Put sequel db tasks into its own namespace
     config.sequel.load_database_tasks = :sequel
+
+    ## Logging
+    config.log_level = :info
+    config.logger = Carto::Common::Logger.new("log/#{Rails.env}.log")
   end
 end
 

config/environments/development.rb

@@ -23,15 +23,13 @@
   # If you have no front-end server that supports something like X-Sendfile,
   # just comment this out and Rails will serve the files
 
-  # Use a different logger for distributed setups
-  config.logger = Logger.new(STDOUT)
-  # config.logger = ActiveSupport::Logger.new(Carto::Conf.new.log_file_path('development.log'))
+  ## Logging
+  logger_stdout = STDOUT.dup
+  logger_stdout.sync = true
 
-  # Adjust the log level. Note that assigning to `config.log_levl` would
-  # have no effect here, since we have set the logger explicitly.
-  config.logger.level = Logger::DEBUG
-
-  config.log_level = :debug
+  config.log_level = :info
+  config.logger = Carto::Common::Logger.new(logger_stdout)
+  ## ./ Logging
 
   # Use a different cache store in production
   # config.cache_store = :mem_cache_store

config/environments/production.rb

@@ -20,22 +20,6 @@
   # If you have no front-end server that supports something like X-Sendfile,
   # just comment this out and Rails will serve the files
 
-  # Use a different logger for distributed setups
-  # config.logger = SyslogLogger.new
-
-  # Note that we pass the desired log level to the logger's constructor;
-  # assigning to `config.log_level` would have no effect here, since we have set the logger explicitly.
-  config.logger = ActiveSupport::TaggedLogging.new(
-    ActiveSupport::Logger.new(Carto::Conf.new.log_file_path('production.log'))
-  )
-
-  config.log_level = :info
-
-  # # Adjust the log level. Note that assigning to `config.log_level` would
-  # # have no effect here, since we have set the logger explicitly.
-  # # We'll set the level to :info, which omits SQL statements in the log.
-  # config.logger.level = 1 # :info
-
   # Use a different cache store in production
   # config.cache_store = :mem_cache_store
 

config/environments/staging.rb

@@ -20,20 +20,9 @@
   # If you have no front-end server that supports something like X-Sendfile,
   # just comment this out and Rails will serve the files
 
-  # Note that we pass the desired log level to the logger's constructor;
-  # assigning to `config.log_level` would have no effect here, since we have set the logger explicitly.
-  config.logger = ActiveSupport::TaggedLogging.new(
-    ActiveSupport::Logger.new(Carto::Conf.new.log_file_path('staging.log'))
-  )
-
-  config.log_level = :info
-
   # Setting this to true will enable ActiveController's enforcement of SSL.
   config.ssl_required = true
 
-  # Use a different logger for distributed setups
-  # config.logger = SyslogLogger.new
-
   # Use a different cache store in production
   # config.cache_store = :mem_cache_store
 

config/initializers/04_install_carto_logger.rb

@@ -0,0 +1 @@
+Carto::Common::Logger.install

config/initializers/load_resque.rb

@@ -11,3 +11,10 @@
 
 Resque::Failure::Multiple.classes = [Resque::Failure::Redis, CartoDB::Logger::RollbarLogger]
 Resque::Failure.backend = Resque::Failure::Multiple
+
+## Logging
+logger_stdout = STDOUT.dup
+logger_stdout.sync = Rails.env.development?
+
+Resque.logger.level = :info
+Resque.logger = Carto::Common::Logger.new(logger_stdout)

lib/resque/base_job.rb

@@ -4,6 +4,7 @@
 module Resque
   class BaseJob
     extend ::Resque::Metrics
+    include Carto::Common::JobLogger
     MAX_RETRIES = 3
 
     @@queue = ''

lib/resque/reporter_jobs.rb

@@ -4,6 +4,8 @@ module Resque
   module Reporter
     module Mail
       module TrendingMapsReport
+        include Carto::Common::JobLogger
+
         @queue = :users
 
         def self.perform(mail_to, trending_visualizations)

lib/resque/tracking_jobs.rb

@@ -4,6 +4,8 @@
 module Resque
   module TrackingJobs
     module SendPubSubEvent
+      include Carto::Common::JobLogger
+
       @queue = :tracker
 
       def self.perform(user_id, name, properties)
@@ -12,6 +14,8 @@ def self.perform(user_id, name, properties)
     end
 
     module SendSegmentEvent
+      include Carto::Common::JobLogger
+
       ANONYMOUS_SEGMENT_USER_ID = '00000000-0000-0000-0000-000000000000'.freeze
 
       @queue = :tracker
@@ -31,6 +35,8 @@ def self.perform(user_id, name, properties)
     end
 
     module SendHubspotEvent
+      include Carto::Common::JobLogger
+
       @queue = :tracker
 
       def self.perform(id, params)

lib/resque/user_db_jobs.rb

@@ -7,6 +7,8 @@ module UserDBJobs
     module UserDBMaintenance
       module LinkGhostTables
         extend ::Resque::Metrics
+        include Carto::Common::JobLogger
+
         @queue = :user_dbs
 
         def self.perform(user_id)
@@ -19,6 +21,8 @@ def self.perform(user_id)
 
       module LinkGhostTablesByUsername
         extend ::Resque::Metrics
+        include Carto::Common::JobLogger
+
         @queue = :user_dbs
 
         def self.perform(username)
@@ -31,6 +35,8 @@ def self.perform(username)
       end
 
       module AutoIndexTable
+        include Carto::Common::JobLogger
+
         @queue = :user_dbs
 
         def self.perform(user_table_id)
@@ -44,6 +50,8 @@ def self.perform(user_table_id)
 
     module CommonData
       module LoadCommonData
+        include Carto::Common::JobLogger
+
         @queue = :user_dbs
 
         def self.perform(user_id, visualizations_api_url)

lib/resque/user_jobs.rb

@@ -7,6 +7,8 @@ module OrganizationJobs
     module Mail
       module DiskQuotaLimitReached
         extend ::Resque::Metrics
+        include Carto::Common::JobLogger
+
         @queue = :users
 
         def self.perform(organization_id)
@@ -15,6 +17,8 @@ def self.perform(organization_id)
       end
 
       module Invitation
+        include Carto::Common::JobLogger
+
         @queue = :users
 
         def self.perform(invitation_id)
@@ -27,6 +31,8 @@ def self.perform(invitation_id)
 
       module SeatLimitReached
         extend ::Resque::Metrics
+        include Carto::Common::JobLogger
+
         @queue = :users
 
         def self.perform(organization_id)
@@ -39,6 +45,8 @@ def self.perform(organization_id)
   module UserJobs
     module Signup
       module NewUser
+        include Carto::Common::JobLogger
+
         @queue = :users
 
         def self.perform(user_creation_id, common_data_url = nil, organization_owner_promotion = false)
@@ -53,6 +61,8 @@ def self.perform(user_creation_id, common_data_url = nil, organization_owner_pro
     module RateLimitsJobs
       module SyncRedis
         extend ::Resque::Metrics
+        include Carto::Common::JobLogger
+
         @queue = :batch_updates
 
         def self.perform(account_type)
@@ -70,6 +80,8 @@ def self.perform(account_type)
     module Notifications
       module Send
         extend ::Resque::Metrics
+        include Carto::Common::JobLogger
+
         @queue = :users
 
         def self.perform(user_ids, notification_id)
@@ -84,6 +96,8 @@ def self.perform(user_ids, notification_id)
     module Mail
       module NewOrganizationUser
         extend ::Resque::Metrics
+        include Carto::Common::JobLogger
+
         @queue = :users
 
         def self.perform(user_id)
@@ -94,6 +108,8 @@ def self.perform(user_id)
 
       module ShareVisualization
         extend ::Resque::Metrics
+        include Carto::Common::JobLogger
+
         @queue = :users
 
         def self.perform(visualization_id, user_id)
@@ -105,6 +121,8 @@ def self.perform(visualization_id, user_id)
 
       module ShareTable
         extend ::Resque::Metrics
+        include Carto::Common::JobLogger
+
         @queue = :users
 
         def self.perform(table_id, user_id)
@@ -116,6 +134,8 @@ def self.perform(table_id, user_id)
 
       module UnshareVisualization
         extend ::Resque::Metrics
+        include Carto::Common::JobLogger
+
         @queue = :users
 
         def self.perform(visualization_name, visualization_owner_name, user_id)
@@ -126,6 +146,8 @@ def self.perform(visualization_name, visualization_owner_name, user_id)
 
       module UnshareTable
         extend ::Resque::Metrics
+        include Carto::Common::JobLogger
+
         @queue = :users
 
         def self.perform(table_name, table_owner_name, user_id)
@@ -136,6 +158,8 @@ def self.perform(table_name, table_owner_name, user_id)
 
       module MapLiked
         extend ::Resque::Metrics
+        include Carto::Common::JobLogger
+
         @queue = :users
 
         def self.perform(visualization_id, viewer_user_id, vis_preview_image)
@@ -147,6 +171,8 @@ def self.perform(visualization_id, viewer_user_id, vis_preview_image)
 
       module TableLiked
         extend ::Resque::Metrics
+        include Carto::Common::JobLogger
+
         @queue = :users
 
         def self.perform(visualization_id, viewer_user_id, vis_preview_image)
@@ -158,6 +184,8 @@ def self.perform(visualization_id, viewer_user_id, vis_preview_image)
 
       module DataImportFinished
         extend ::Resque::Metrics
+        include Carto::Common::JobLogger
+
         @queue = :users
 
         def self.perform(user_id, imported_tables, total_tables, first_imported_table, first_table, errors, filenames)
@@ -169,6 +197,8 @@ def self.perform(user_id, imported_tables, total_tables, first_imported_table, f
 
       module GeocoderFinished
         extend ::Resque::Metrics
+        include Carto::Common::JobLogger
+
         @queue = :users
 
         def self.perform(user_id, state, table_name, error_code, processable_rows, number_geocoded_rows)
@@ -181,6 +211,8 @@ def self.perform(user_id, state, table_name, error_code, processable_rows, numbe
       module Sync
         module MaxRetriesReached
           extend ::Resque::Metrics
+          include Carto::Common::JobLogger
+
           @queue = :users
 
           def self.perform(user_id, visualization_id, dataset_name, error_code, error_message)
@@ -192,6 +224,8 @@ def self.perform(user_id, visualization_id, dataset_name, error_code, error_mess
 
       module TrendingMap
         extend ::Resque::Metrics
+        include Carto::Common::JobLogger
+
         @queue = :users
 
         def self.perform(visualization_id, mapviews, vis_preview_image)
@@ -202,6 +236,8 @@ def self.perform(visualization_id, mapviews, vis_preview_image)
 
       module PasswordReset
         extend ::Resque::Metrics
+        include Carto::Common::JobLogger
+
         @queue = :users
 
         def self.perform(user_id)