SRCH-1305 update analytics queries to query by type field (#566)

.rubocop.yml

@@ -2788,7 +2788,9 @@ Style/BlockDelimiters:
 
 Style/BracesAroundHashParameters:
   Description: 'Enforce braces style around hash parameters.'
-  Enabled: true
+  # Disabling per https://github.com/rubocop-hq/rubocop/issues/7641.
+  # This cop will be removed from Rubocop 0.80.
+  Enabled: false
   VersionAdded: 0.14.1
   VersionChanged: 0.28
   EnforcedStyle: no_braces

app/controllers/sites/click_queries_controller.rb

@@ -10,6 +10,7 @@ def show
 
   def top_queries
     query = DateRangeTopNFieldQuery.new(@site.name,
+                                        'click',
                                         @start_date,
                                         @end_date,
                                         'params.url',

app/controllers/sites/query_clicks_controller.rb

@@ -10,6 +10,7 @@ def show
 
   def top_urls
     query = DateRangeTopNFieldQuery.new(@site.name,
+                                        'click',
                                         @start_date,
                                         @end_date,
                                         'params.query.raw',

app/controllers/sites/query_downloads_controller.rb

@@ -55,6 +55,7 @@ def ctr(click_count, query_count)
 
   def date_range_top_n_query
     @date_range_top_n_query ||= DateRangeTopNQuery.new(@site.name,
+                                                       'search',
                                                        @start_date,
                                                        @end_date,
                                                        field: 'params.query.raw',

app/controllers/sites/query_referrers_controller.rb

@@ -10,6 +10,7 @@ def show
 
   def top_urls
     query = DateRangeTopNFieldQuery.new(@site.name,
+                                        'search',
                                         @start_date,
                                         @end_date,
                                         'params.query.raw',

app/controllers/sites/referrer_queries_controller.rb

@@ -10,6 +10,7 @@ def show
 
   def top_queries
     query = DateRangeTopNFieldQuery.new(@site.name,
+                                        'search',
                                         @start_date,
                                         @end_date,
                                         'referrer',

app/models/affiliate.rb

@@ -510,8 +510,9 @@ def header_image_url
 
   def last_month_query_count
     prev_month = Date.current.prev_month
-    count_query = CountQuery.new(name)
-    RtuCount.count(monthly_index_wildcard_spanning_date(prev_month, true), 'search', count_query.body)
+    count_query = CountQuery.new(name, 'search')
+    RtuCount.count(monthly_index_wildcard_spanning_date(prev_month, true),
+                   count_query.body)
   end
 
   def user_emails

app/models/click_counter.rb

@@ -21,6 +21,7 @@ def update_click_counts
 
   def statistically_significant_clicks
     query = DateRangeTopNFieldQuery.new(nil,
+                                        'click',
                                         1.month.ago,
                                         Time.current,
                                         'click_domain',

app/models/logstash_queries/count_query.rb

@@ -3,8 +3,9 @@
 class CountQuery
   include AnalyticsDSL
 
-  def initialize(affiliate_name)
+  def initialize(affiliate_name, type)
     @affiliate_name = affiliate_name
+    @type = type
   end
 
   def body
@@ -14,9 +15,8 @@ def body
   end
 
   def booleans(json)
-    json.filter do
-      json.term { json.set! 'params.affiliate', @affiliate_name }
-    end
+    must_affiliate(json, affiliate_name)
+    must_type(json, type)
     must_not_spider(json)
   end
 end

app/models/logstash_queries/date_range_top_n_exists_query.rb

@@ -1,8 +1,9 @@
 class DateRangeTopNExistsQuery < TopNExistsQuery
 
-  def initialize(affiliate_name, start_date, end_date, agg_options = {})
-    super(affiliate_name, agg_options)
-    @start_date, @end_date = start_date, end_date
+  def initialize(affiliate_name, type, start_date, end_date, agg_options = {})
+    super(affiliate_name, type, agg_options)
+    @start_date = start_date
+    @end_date = end_date
   end
 
   def additional_musts(json)

app/models/logstash_queries/date_range_top_n_field_query.rb

@@ -1,21 +1,20 @@
 class DateRangeTopNFieldQuery < DateRangeTopNQuery
-  attr_reader :filters
+  attr_reader :filter_field, :filter_value
 
-  def initialize(affiliate_name, start_date, end_date, filter_field, filter_value, agg_options = {})
-    super(affiliate_name, start_date, end_date, agg_options)
-    @filters = { 'params.affiliate' => @affiliate_name,
-                 filter_field => filter_value }.compact
+  def initialize(affiliate_name, type, start_date, end_date, filter_field, filter_value, agg_options = {})
+    super(affiliate_name, type, start_date, end_date, agg_options)
+    @filter_field = filter_field
+    @filter_value = filter_value
   end
 
   def booleans(json)
-    filters.each do |field, value|
-      json.filter do
-        json.child! { json.term { json.set! field, value } }
-      end
-    end
+    must_affiliate(json, affiliate_name) if affiliate_name
+    must_type(json, type)
 
     json.filter do
-      json.child! { date_range(json, @start_date, @end_date) }
+      json.child! { json.term { json.set! filter_field, filter_value } }
     end
+
+    must_date_range(json, start_date, end_date)
   end
 end

app/models/logstash_queries/date_range_top_n_missing_query.rb

@@ -1,9 +1,10 @@
 # frozen_string_literal: true
 
 class DateRangeTopNMissingQuery < TopNMissingQuery
-  def initialize(affiliate_name, start_date, end_date, agg_options = {})
-    super(affiliate_name, agg_options)
-    @start_date, @end_date = start_date, end_date
+  def initialize(affiliate_name, type, start_date, end_date, agg_options = {})
+    super(affiliate_name, type, agg_options)
+    @start_date = start_date
+    @end_date = end_date
   end
 
   def additional_musts(json)

app/models/logstash_queries/date_range_top_n_query.rb

@@ -1,13 +1,18 @@
 # frozen_string_literal: true
 
 class DateRangeTopNQuery < TopNQuery
+  attr_reader :start_date, :end_date
 
-  def initialize(affiliate_name, start_date, end_date, agg_options = {})
-    super(affiliate_name, agg_options)
-    @start_date, @end_date = start_date, end_date
+  def initialize(affiliate_name, type, start_date, end_date, agg_options = {})
+    super(affiliate_name, type, agg_options)
+    @start_date = start_date
+    @end_date = end_date
   end
 
   def booleans(json)
-    must_affiliate_date_range(json, @affiliate_name, @start_date, @end_date)
+    must_affiliate(json, affiliate_name)
+    must_type(json, type)
+    must_date_range(json, start_date, end_date)
+    must_not_spider(json)
   end
 end

app/models/logstash_queries/drilldown_query.rb

@@ -24,7 +24,7 @@ def booleans(json)
       json.child! { json.term { json.set! @field, @value } }
     end
     must_affiliate(json, @affiliate_name)
-    must_type(json, @type)
+    must_type(json, type)
     must_not_spider(json)
   end
 end

app/models/logstash_queries/low_ctr_query.rb

@@ -3,6 +3,7 @@
 class LowCtrQuery < DateRangeTopNExistsQuery
   def initialize(affiliate_name, start_date, end_date, agg_options = {})
     super(affiliate_name,
+          %w[search click],
           start_date,
           end_date,
           agg_options.reverse_merge(min_doc_count: 20,

app/models/logstash_queries/module_breakdown_query.rb

@@ -16,6 +16,7 @@ def body
 
   def booleans(json)
     must_affiliate(json, @affiliate_name) if @affiliate_name.present?
+    must_type(json, %w[search click])
     must_not_spider(json)
   end
 end

app/models/logstash_queries/module_sparkline_query.rb

@@ -16,6 +16,7 @@ def body
 
   def booleans(json)
     must_affiliate(json, affiliate_name)
+    must_type(json, %w[search click])
     json.filter do
       json.child! { since(json, 'now-60d/d') }
       json.child! { json.exists { json.field 'modules' } }

app/models/logstash_queries/monthly_histogram_query.rb

@@ -28,8 +28,9 @@ def date_histogram(json)
   end
 
   def booleans(json)
+    must_affiliate(json, affiliate_name)
+    must_type(json, 'search')
     json.filter do
-      json.child! { json.term { json.set! 'params.affiliate', @affiliate_name } }
       json.child! { since(json, @since) }
     end
     must_not_spider(json)

app/models/logstash_queries/query_breakdown_for_site_module_query.rb

@@ -16,10 +16,11 @@ def body
   end
 
   def booleans(json)
+    must_type(json, %w[search click])
     json.filter do
       json.child! { json.term { json.modules @module_tag } }
-      json.child! { json.term { json.set! 'params.affiliate', @site_name } }
     end
+    must_affiliate(json, @site_name)
     must_not_spider(json)
   end
 end

app/models/logstash_queries/rtu_date_range_query.rb

@@ -3,19 +3,21 @@
 class RtuDateRangeQuery
   include AnalyticsDSL
 
-  def initialize(affiliate_name)
+  def initialize(affiliate_name, type)
     @affiliate_name = affiliate_name
+    @type = type
   end
 
   def body
     Jbuilder.encode do |json|
       filter_booleans(json)
-      stats(json, "@timestamp")
+      stats(json, '@timestamp')
     end
   end
 
   def booleans(json)
-    must_affiliate(json, @affiliate_name)
+    must_affiliate(json, affiliate_name)
+    must_type(json, type)
     must_not_spider(json)
   end
 end

app/models/logstash_queries/site_breakdown_for_module_query.rb

@@ -3,6 +3,8 @@
 class SiteBreakdownForModuleQuery
   include AnalyticsDSL
 
+  attr_reader :module_tag
+
   def initialize(module_tag)
     @module_tag = module_tag
   end
@@ -15,9 +17,8 @@ def body
   end
 
   def booleans(json)
-    json.filter do
-      json.term { json.modules @module_tag }
-    end
+    must_module(json, module_tag)
+    must_type(json, %w[search click])
     must_not_spider(json)
   end
 end

app/models/logstash_queries/top_n_modules_query.rb

@@ -2,8 +2,9 @@
 
 class TopNModulesQuery < TopNQuery
   def booleans(json)
+    must_affiliate(json, affiliate_name)
+    must_type(json, type)
     json.filter do
-      json.child! { json.term { json.set! 'params.affiliate', @affiliate_name } }
       modules_must(json)
       additional_musts(json)
     end

app/models/logstash_queries/top_n_query.rb

@@ -3,8 +3,9 @@
 class TopNQuery
   include AnalyticsDSL
 
-  def initialize(affiliate_name, agg_options = {})
+  def initialize(affiliate_name, type, agg_options = {})
     @affiliate_name = affiliate_name
+    @type = type
     @agg_options = agg_options
   end
 
@@ -16,9 +17,8 @@ def body
   end
 
   def booleans(json)
-    json.filter do
-      json.term { json.set! 'params.affiliate', @affiliate_name }
-    end
+    must_affiliate(json, affiliate_name)
+    must_type(json, type)
     must_not_spider(json)
   end
 end

app/models/logstash_queries/top_query_match_query.rb

@@ -39,7 +39,10 @@ def match_query(json)
   end
 
   def booleans(json)
-    must_affiliate_date_range(json, @affiliate_name, @start_date, @end_date)
+    must_affiliate(json, @affiliate_name)
+    must_type(json, %w[search click])
+    must_date_range(json, @start_date, @end_date)
+    must_not_spider(json)
   end
 
   def terms_type_agg(json)

app/models/logstash_queries/watcher_no_results_query.rb

@@ -7,6 +7,7 @@ def initialize(options)
     start_date, end_date = start_end_from_time_window(options[:time_window])
     @query_blocklist = options[:query_blocklist]
     super(options[:affiliate_name],
+          'search',
           start_date,
           end_date,
           options.slice(:min_doc_count, :size).merge(field: 'params.query.raw'))

app/models/low_query_ctr_watcher.rb

@@ -25,7 +25,6 @@ def input(json)
                 query_blocklist: query_blocklist }
     low_query_ctr_query_body = WatcherLowCtrQuery.new(options).body
     input_search_request(json,
-                         types: %w[search click],
                          indices: watcher_indexes_from_window_size(time_window),
                          body: JSON.parse(low_query_ctr_query_body).merge(size: 0))
   end

app/models/no_results_watcher.rb

@@ -19,7 +19,6 @@ def input(json)
                 size: 10 }
     no_results_query_body = WatcherNoResultsQuery.new(options).body
     input_search_request(json,
-                         types: ['search'],
                          indices: watcher_indexes_from_window_size(time_window),
                          body: JSON.parse(no_results_query_body).merge(size: 0))
   end

app/models/rtu_click_raw_human_array.rb

@@ -10,4 +10,8 @@ def aggs_field
   def top_clicks
     most_popular
   end
+
+  def type
+    'click'
+  end
 end

app/models/rtu_clicks_request.rb

@@ -20,7 +20,13 @@ def top_urls
   private
 
   def compute_top_stats
-    query = DateRangeTopNQuery.new(site.name, start_date, end_date, { field: 'params.url', size: MAX_RESULTS })
+    query = DateRangeTopNQuery.new(
+      site.name,
+      'click',
+      start_date,
+      end_date,
+      { field: 'params.url', size: MAX_RESULTS }
+    )
     rtu_top_clicks = RtuTopClicks.new(query.body, filter_bots)
     rtu_top_clicks.top_n
   end

app/models/rtu_count.rb

@@ -1,7 +1,10 @@
-class RtuCount
+# frozen_string_literal: true
 
-  def self.count(index, type, query_body)
-    ES::ELK.client_reader.count(index: index, type: type, body: query_body)["count"] rescue nil
+class RtuCount
+  def self.count(index, query_body)
+    ES::ELK.client_reader.count(index: index, body: query_body)['count']
+  rescue StandardError => error
+    Rails.logger.error("Error extracting RtuCount: #{error}")
+    nil
   end
-
 end