Implementing Quality Intelligence

testsuite/.rubocop.yml

@@ -138,7 +138,7 @@ Style/TopLevelMethodDefinition:
 Style/Copyright:
   Enabled: true
   AutocorrectNotice: |
-    # Copyright (c) 2023 SUSE LLC.
+    # Copyright (c) 2024 SUSE LLC.
     # Licensed under the terms of the MIT license.
   Notice: '.*Copyright.*SUSE LLC.*'
 

testsuite/Gemfile

@@ -21,6 +21,7 @@ gem 'nokogiri', '~> 1.16'
 gem 'parallel', '~> 1.26'
 gem 'parallel_tests', '~> 4.7'
 gem 'pg', '~> 1.5'
+gem 'prometheus-client', '~> 4.2.3'
 gem 'public_suffix', '~> 6.0'
 gem 'rack', '~> 3.1'
 gem 'rack-test', '~> 2.1'
@@ -29,7 +30,7 @@ gem 'redis', '~> 5.2'
 gem 'redis-client', '~> 0.22'
 gem 'require_all', '~> 3.0'
 gem 'rubocop', '~>1.66'
-gem 'selenium-webdriver', '4.24'
+gem 'selenium-webdriver', '~> 4'
 gem 'simplecov', '~> 0.22'
 gem 'syntax', '~> 1.2'
 gem 'websocket', '~> 1.2'

testsuite/features/init_clients/sle_minion.feature

@@ -18,6 +18,7 @@ Feature: Bootstrap a Salt minion via the GUI
     And I select the hostname of "proxy" from "proxies" if present
     And I click on "Bootstrap"
     And I wait until I see "Bootstrap process initiated." text
+    And I report the bootstrap duration for "sle_minion"
 
   Scenario: Check the new bootstrapped minion in System List page
     When I follow the left menu "Salt > Keys"
@@ -27,6 +28,7 @@ Feature: Bootstrap a Salt minion via the GUI
     And I wait until I see the name of "sle_minion", refreshing the page
     And I wait until onboarding is completed for "sle_minion"
     Then the Salt master can reach "sle_minion"
+    And I report the onboarding duration for "sle_minion"
 
 @susemanager
   Scenario: Use correct kernel image on the SLES minion

testsuite/features/reposync/srv_sync_products.feature

@@ -68,6 +68,7 @@ Feature: Synchronize products in the products page of the Setup Wizard
     And I wait until I see "SUSE Linux Enterprise Server 15 SP4 x86_64" product has been added
     Then the SLE15 SP4 product should be added
     When I wait until all synchronized channels for "sles15-sp4" have finished
+    And I report the synchronization duration for "sles15-sp4"
 
 @scc_credentials
 @uyuni
@@ -100,6 +101,7 @@ Feature: Synchronize products in the products page of the Setup Wizard
     Then the SLE15 SP4 product should be added
     When I use spacewalk-common-channel to add channel "sles15-sp4-devel-uyuni-client" with arch "x86_64"
     And I wait until all synchronized channels for "sles15-sp4" have finished
+    And I report the synchronization duration for "sles15-sp4"
     # TODO: Refactor the scenarios in order to not require a full synchronization of SLES 15 SP4 product in Uyuni
     # When I kill running spacewalk-repo-sync for "sles15-sp4"
 

testsuite/features/step_definitions/system_monitoring_steps.rb

@@ -0,0 +1,25 @@
+# Copyright (c) 2024 SUSE LLC.
+# Licensed under the terms of the MIT license.
+
+### This file contains the definitions for the steps extracting and reporting information from the system
+
+When(/^I report the bootstrap duration for "([^"]*)"$/) do |host|
+  next unless $quality_intelligence_mode
+
+  duration = last_bootstrap_duration(host)
+  $quality_intelligence.push_bootstrap_duration(host, duration)
+end
+
+When(/^I report the onboarding duration for "([^"]*)"$/) do |host|
+  next unless $quality_intelligence_mode
+
+  duration = last_onboarding_duration(host)
+  $quality_intelligence.push_onboarding_duration(host, duration)
+end
+
+When(/^I report the synchronization duration for "([^"]*)"$/) do |product|
+  next unless $quality_intelligence_mode
+
+  duration = synchronization_duration(product)
+  $quality_intelligence.push_synchronization_duration(product, duration)
+end

testsuite/features/support/code_coverage.rb

@@ -1,20 +1,16 @@
-# Copyright (c) 2016-2023 SUSE LLC.
+# Copyright (c) 2024 SUSE LLC.
 # Licensed under the terms of the MIT license.
-require 'redis'
+require_relative 'database_handler'
 require 'nokogiri'
 require 'open-uri'
 
 # CodeCoverage handler to produce, parse and report Code Coverage from the Jave Server to our GitHub PRs
 class CodeCoverage
-  include(Nokogiri::XML)
-  # Initialize a connection with a Redis database
-  def initialize(redis_host, redis_port, redis_username, redis_password)
-    @database = Redis.new(host: redis_host, port: redis_port, username: redis_username, password: redis_password)
-  end
+  include Nokogiri::XML
 
-  # Close the connection with the Redis database
-  def close
-    @database.close
+  # Initialize the CodeCoverage handler
+  def initialize
+    @db_handler = DatabaseHandler.new(ENV.fetch('REDIS_HOST', nil), ENV.fetch('REDIS_PORT', nil), ENV.fetch('REDIS_USERNAME', nil), ENV.fetch('REDIS_PASSWORD', nil))
   end
 
   # Parse a JaCoCo XML report, extracting information that will be included in a Set on a Redis database
@@ -36,17 +32,12 @@ def push_feature_coverage(feature_name)
 
           next unless Integer(counter_class.attr('covered').to_s).positive?
 
-          begin
-            @database.sadd("#{package_name}/#{sourcefile_name}", feature_name)
-          rescue StandardError => e
-            warn("#{e.backtrace} > #{package_name}/#{sourcefile_name} : #{feature_name}")
-          end
+          @db_handler.add("#{package_name}/#{sourcefile_name}", feature_name)
         end
       end
     rescue StandardError => e
       warn(e.backtrace)
-    ensure
-      File.delete(filename)
+    ensure File.delete(filename)
     end
   end
 

testsuite/features/support/database_handler.rb

@@ -0,0 +1,37 @@
+# Copyright (c) 2024 SUSE LLC.
+# Licensed under the terms of the MIT license.
+require 'redis'
+
+# Database handler to interact with a Redis database
+class DatabaseHandler
+  # Initialize a connection with a Redis database
+  #
+  # @param redis_host [String] The hostname of the Redis database.
+  # @param redis_port [Integer] The port of the Redis database.
+  # @param redis_username [String] The username to authenticate with the Redis database.
+  # @param redis_password [String] The password to authenticate with the Redis database.
+  # @return [Redis] A connection with the Redis database.
+  def initialize(redis_host, redis_port, redis_username, redis_password)
+    @database = Redis.new(host: redis_host, port: redis_port, username: redis_username, password: redis_password)
+  end
+
+  # Close the connection with the Redis database
+  def close
+    @database.close
+  end
+
+  # Add a key-value pair to a Set, optionally selecting a database
+  #
+  # @param key [String] The key to add the value to.
+  # @param value [String] The value to add to the key.
+  # @param database [Integer] The database to select (default: 0).
+  # @return [String] `OK`
+  def add(key, value, database = 0)
+    begin
+      @database.select(database)
+      @database.sadd(key, value)
+    rescue StandardError => e
+      warn("#{e.backtrace} > #{key} : #{value}")
+    end
+  end
+end

testsuite/features/support/env.rb

@@ -17,6 +17,7 @@
 require 'set'
 require 'timeout'
 require_relative 'code_coverage'
+require_relative 'quality_intelligence'
 require_relative 'remote_nodes_env'
 require_relative 'commonlib'
 
@@ -29,10 +30,14 @@
   $debug_mode = true
   $stdout.puts('DEBUG MODE ENABLED.')
 end
-if ENV['REDIS_HOST']
+if ENV['REDIS_HOST'] && ENV.fetch('CODE_COVERAGE', false)
   $code_coverage_mode = true
   $stdout.puts('CODE COVERAGE MODE ENABLED.')
 end
+if ENV.fetch('QUALITY_INTELLIGENCE', false)
+  $quality_intelligence_mode = true
+  $stdout.puts('QUALITY INTELLIGENCE MODE ENABLED.')
+end
 
 # Context per feature
 $context = {}
@@ -117,7 +122,10 @@ def capybara_register_driver
 $api_test = new_api_client
 
 # Init CodeCoverage Handler
-$code_coverage = CodeCoverage.new(ENV.fetch('REDIS_HOST', nil), ENV.fetch('REDIS_PORT', nil), ENV.fetch('REDIS_USERNAME', nil), ENV.fetch('REDIS_PASSWORD', nil)) if $code_coverage_mode
+$code_coverage = CodeCoverage.new if $code_coverage_mode
+
+# Init Quality Intelligence Handler
+$quality_intelligence = QualityIntelligence.new if $quality_intelligence_mode
 
 # Define the current feature scope
 Before do |scenario|
@@ -139,7 +147,7 @@ def capybara_register_driver
       print_server_logs
     end
   end
-  page.instance_variable_set(:@touched, false)
+  page.instance_variable_set(:@touched, false) if Capybara::Session.instance_created?
 end
 
 # Test is web session is open

testsuite/features/support/namespaces/system.rb

@@ -183,6 +183,25 @@ def get_system_errata(system_id)
   def get_systems_errata(system_ids)
     @test.call('system.getRelevantErrata', sessionKey: @test.token, sids: system_ids)
   end
+
+  # Returns the event history for a system.
+  #
+  # @param system_id [String] The ID of the system.
+  # @param offset [Integer] Number of results to skip
+  # @param limit [Integer] Maximum number of results
+  # @return [Array<Hash>] An array of events
+  def get_event_history(system_id, offset, limit)
+    @test.call('system.getEventHistory', sessionKey: @test.token, sid: system_id, offset: offset, limit: limit)
+  end
+
+  # Returns the event details for a system.
+  #
+  # @param system_id [String] The ID of the system.
+  # @param event_id [String] The ID of the event.
+  # @return [Hash] The event details
+  def get_event_details(system_id, event_id)
+    @test.call('system.getEventDetails', sessionKey: @test.token, sid: system_id, eid: event_id)
+  end
 end
 
 # System Configuration namespace

testsuite/features/support/prometheus_handler.rb

@@ -0,0 +1,33 @@
+# Copyright (c) 2024 SUSE LLC.
+# Licensed under the terms of the MIT license.
+
+require 'net/http'
+require 'prometheus/client'
+require 'prometheus/client/push'
+require 'uri'
+
+# Prometheus handler to push metrics to Prometheus
+class PrometheusHandler
+  def initialize(push_gateway_url)
+    @push_gateway_url = push_gateway_url
+    @prometheus = Prometheus::Client::Registry.new
+  end
+
+  # Push a metric to Prometheus, raising an error if the Prometheus request fails.
+  #
+  # @param job_name [String] the job name to push the metric to
+  # @param metric_name [String] the metric name to push
+  # @param metric_value [Integer] the metric value to push
+  # @param labels [Hash] the labels to add to the metric
+  # @return [void]
+  # @raise [SystemCallError] if the Prometheus request fails
+  def push_metric_to_prometheus(job_name, metric_name, metric_value, labels = {})
+    begin
+      gauge = @prometheus.get(metric_name.to_sym) || @prometheus.gauge(metric_name.to_sym, docstring: job_name, labels: labels.keys)
+      gauge.set(metric_value, labels: labels)
+      Prometheus::Client::Push.new(job: job_name, gateway: @push_gateway_url).add(@prometheus)
+    rescue StandardError => e
+      warn(e.full_message)
+    end
+  end
+end

testsuite/features/support/quality_intelligence.rb

@@ -0,0 +1,47 @@
+# Copyright (c) 2024 SUSE LLC.
+# Licensed under the terms of the MIT license.
+require_relative 'database_handler'
+require_relative 'prometheus_handler'
+
+# Quality Intelligence handler to produce, parse and report Quality Intelligence from the test suite
+class QualityIntelligence
+  QI = 'quality_intelligence'.freeze
+  private_constant :QI
+
+  # Initialize the QualityIntelligence handler
+  def initialize
+    @db_handler = DatabaseHandler.new(ENV.fetch('REDIS_HOST', nil), ENV.fetch('REDIS_PORT', nil), ENV.fetch('REDIS_USERNAME', nil), ENV.fetch('REDIS_PASSWORD', nil))
+    @prometheus_handler = PrometheusHandler.new(ENV.fetch('PROMETHEUS_PUSH_GATEWAY_URL', 'http://nsa.mgr.suse.de:9091'))
+    @environment = ENV.fetch('SERVER', nil)
+  end
+
+  # Report the time to complete a bootstrap of a system passed as parameter,
+  # raising an error if the Prometheus request fails.
+  #
+  # @param system [String] the system to bootstrap
+  # @param time [Integer] the time to complete the bootstrap in seconds
+  # @return [void]
+  def push_bootstrap_duration(system, time)
+    @prometheus_handler.push_metric_to_prometheus(QI, 'system_bootstrap_duration_seconds', time, labels: { 'system' => system, 'environment' => @environment })
+  end
+
+  # Report the time to complete the onboarding of a system passed as parameter,
+  # raising an error if the Prometheus request fails.
+  #
+  # @param system [String] the system to be onboarded
+  # @param time [Integer] the time to complete the onboarding in seconds
+  # @return [void]
+  def push_onboarding_duration(system, time)
+    @prometheus_handler.push_metric_to_prometheus(QI, 'system_onboarding_duration_seconds', time, labels: { 'system' => system, 'environment' => @environment })
+  end
+
+  # Report the time to complete a synchronization of a product passed as parameter,
+  # raising an error if the Prometheus request fails.
+  #
+  # @param product [String] the product to synchronize
+  # @param time [Integer] the time to complete the synchronization in seconds
+  # @return [void]
+  def push_synchronization_duration(product, time)
+    @prometheus_handler.push_metric_to_prometheus(QI, 'product_synch_duration_seconds', time, labels: { 'product' => product, 'environment' => @environment })
+  end
+end

testsuite/features/support/system_monitoring.rb

@@ -0,0 +1,64 @@
+# Copyright (c) 2024 SUSE LLC.
+# Licensed under the terms of the MIT license.
+
+require 'time'
+
+# This method should return the last bootstrap duration for the given host
+# @param host [String] the hostname
+# @return [Float] the duration in seconds or nil if the duration could not be determined
+def last_bootstrap_duration(host)
+  system_name = get_system_name(host)
+  duration = nil
+  lines, _code = get_target('server').run('tail -n100 /var/log/rhn/rhn_web_api.log')
+
+  lines.each_line do |line|
+    if line.include?(system_name) && line.include?('systems.bootstrap')
+      match = line.match(/TIME: (\d+\.\d+) seconds/)
+      duration = match[1].to_f if match
+    end
+  end
+
+  raise ScriptError, "Boostrap duration not found for #{host}" if duration.nil?
+
+  duration
+end
+
+# This method should return the last onboarding duration for the given host
+# @param host [String] the hostname
+# @return [Float] the duration in seconds or nil if the duration could not be determined
+def last_onboarding_duration(host)
+  node = get_target(host)
+  system_id = get_system_id(node)
+  events = $api_test.system.get_event_history(system_id, 0, 10)
+  onboarding_events = events.select { |event| event['summary'].include? 'certs, channels, packages' }
+  last_event_id = onboarding_events.last['id']
+  event_details = $api_test.system.get_event_details(system_id, last_event_id)
+  Time.parse(event_details['completed']) - Time.parse(event_details['picked_up'])
+end
+
+# This method should return the synchronization duration for the given product
+# @param os_product_version [String] the product name
+# @return [Float] the duration in seconds or nil if the duration could not be determined
+def synchronization_duration(os_product_version)
+  channels_to_wait = CHANNEL_TO_SYNC_BY_OS_PRODUCT_VERSION.dig(product, os_product_version)
+  channels_to_wait = filter_channels(channels_to_wait, ['beta']) unless $beta_enabled
+  raise ScriptError, "Synchronization error, channels for #{os_product_version} in #{product} not found" if channels_to_wait.nil?
+
+  duration = 0
+  channel_to_evaluate = false
+  get_target('server').extract('/var/log/rhn/reposync.log', '/tmp/reposync.log')
+  File.foreach('/tmp/reposync.log') do |line|
+    if line.include?('Channel: ')
+      channel_name = line.split('Channel: ')[1].strip
+      channel_to_evaluate = channels_to_wait.include?(channel_name)
+    end
+    if line.include?('Total time: ') && channel_to_evaluate
+      match = line.match(/Total time: (\d+):(\d+):(\d+)/)
+      hours, minutes, seconds = match.captures.map(&:to_i)
+      total_seconds = (hours * 3600) + (minutes * 60) + seconds
+      duration += total_seconds
+      channel_to_evaluate = false
+    end
+  end
+  duration
+end