From 648ae32d75d1adc5f1b844a48ff7350b6e1486ea Mon Sep 17 00:00:00 2001 From: Oscar Barrios Date: Mon, 28 Oct 2024 16:37:20 +0100 Subject: [PATCH] Implementing Quality Intelligence --- testsuite/.rubocop.yml | 2 +- testsuite/Gemfile | 3 +- .../features/init_clients/sle_minion.feature | 2 + .../reposync/srv_sync_products.feature | 2 + .../system_monitoring_steps.rb | 25 ++++++++ testsuite/features/support/code_coverage.rb | 25 +++----- .../features/support/database_handler.rb | 37 +++++++++++ testsuite/features/support/env.rb | 14 +++- .../features/support/namespaces/system.rb | 19 ++++++ .../features/support/prometheus_handler.rb | 33 ++++++++++ .../features/support/quality_intelligence.rb | 47 ++++++++++++++ .../features/support/system_monitoring.rb | 64 +++++++++++++++++++ 12 files changed, 251 insertions(+), 22 deletions(-) create mode 100644 testsuite/features/step_definitions/system_monitoring_steps.rb create mode 100644 testsuite/features/support/database_handler.rb create mode 100644 testsuite/features/support/prometheus_handler.rb create mode 100644 testsuite/features/support/quality_intelligence.rb create mode 100644 testsuite/features/support/system_monitoring.rb diff --git a/testsuite/.rubocop.yml b/testsuite/.rubocop.yml index f91564aea60d..7eb5121c2561 100644 --- a/testsuite/.rubocop.yml +++ b/testsuite/.rubocop.yml @@ -138,7 +138,7 @@ Style/TopLevelMethodDefinition: Style/Copyright: Enabled: true AutocorrectNotice: | - # Copyright (c) 2023 SUSE LLC. + # Copyright (c) 2024 SUSE LLC. # Licensed under the terms of the MIT license. Notice: '.*Copyright.*SUSE LLC.*' diff --git a/testsuite/Gemfile b/testsuite/Gemfile index 3ad302242959..15f08772113a 100644 --- a/testsuite/Gemfile +++ b/testsuite/Gemfile @@ -21,6 +21,7 @@ gem 'nokogiri', '~> 1.16' gem 'parallel', '~> 1.26' gem 'parallel_tests', '~> 4.7' gem 'pg', '~> 1.5' +gem 'prometheus-client', '~> 4.2.3' gem 'public_suffix', '~> 6.0' gem 'rack', '~> 3.1' gem 'rack-test', '~> 2.1' @@ -29,7 +30,7 @@ gem 'redis', '~> 5.2' gem 'redis-client', '~> 0.22' gem 'require_all', '~> 3.0' gem 'rubocop', '~>1.66' -gem 'selenium-webdriver', '4.24' +gem 'selenium-webdriver', '~> 4' gem 'simplecov', '~> 0.22' gem 'syntax', '~> 1.2' gem 'websocket', '~> 1.2' diff --git a/testsuite/features/init_clients/sle_minion.feature b/testsuite/features/init_clients/sle_minion.feature index b690a37972df..c538eb8f58d3 100644 --- a/testsuite/features/init_clients/sle_minion.feature +++ b/testsuite/features/init_clients/sle_minion.feature @@ -18,6 +18,7 @@ Feature: Bootstrap a Salt minion via the GUI And I select the hostname of "proxy" from "proxies" if present And I click on "Bootstrap" And I wait until I see "Bootstrap process initiated." text + And I report the bootstrap duration for "sle_minion" Scenario: Check the new bootstrapped minion in System List page When I follow the left menu "Salt > Keys" @@ -27,6 +28,7 @@ Feature: Bootstrap a Salt minion via the GUI And I wait until I see the name of "sle_minion", refreshing the page And I wait until onboarding is completed for "sle_minion" Then the Salt master can reach "sle_minion" + And I report the onboarding duration for "sle_minion" @susemanager Scenario: Use correct kernel image on the SLES minion diff --git a/testsuite/features/reposync/srv_sync_products.feature b/testsuite/features/reposync/srv_sync_products.feature index b8a9d7617cca..75320b3fbc42 100644 --- a/testsuite/features/reposync/srv_sync_products.feature +++ b/testsuite/features/reposync/srv_sync_products.feature @@ -68,6 +68,7 @@ Feature: Synchronize products in the products page of the Setup Wizard And I wait until I see "SUSE Linux Enterprise Server 15 SP4 x86_64" product has been added Then the SLE15 SP4 product should be added When I wait until all synchronized channels for "sles15-sp4" have finished + And I report the synchronization duration for "sles15-sp4" @scc_credentials @uyuni @@ -100,6 +101,7 @@ Feature: Synchronize products in the products page of the Setup Wizard Then the SLE15 SP4 product should be added When I use spacewalk-common-channel to add channel "sles15-sp4-devel-uyuni-client" with arch "x86_64" And I wait until all synchronized channels for "sles15-sp4" have finished + And I report the synchronization duration for "sles15-sp4" # TODO: Refactor the scenarios in order to not require a full synchronization of SLES 15 SP4 product in Uyuni # When I kill running spacewalk-repo-sync for "sles15-sp4" diff --git a/testsuite/features/step_definitions/system_monitoring_steps.rb b/testsuite/features/step_definitions/system_monitoring_steps.rb new file mode 100644 index 000000000000..a5c73c3e70d2 --- /dev/null +++ b/testsuite/features/step_definitions/system_monitoring_steps.rb @@ -0,0 +1,25 @@ +# Copyright (c) 2024 SUSE LLC. +# Licensed under the terms of the MIT license. + +### This file contains the definitions for the steps extracting and reporting information from the system + +When(/^I report the bootstrap duration for "([^"]*)"$/) do |host| + next unless $quality_intelligence_mode + + duration = last_bootstrap_duration(host) + $quality_intelligence.push_bootstrap_duration(host, duration) +end + +When(/^I report the onboarding duration for "([^"]*)"$/) do |host| + next unless $quality_intelligence_mode + + duration = last_onboarding_duration(host) + $quality_intelligence.push_onboarding_duration(host, duration) +end + +When(/^I report the synchronization duration for "([^"]*)"$/) do |product| + next unless $quality_intelligence_mode + + duration = synchronization_duration(product) + $quality_intelligence.push_synchronization_duration(product, duration) +end diff --git a/testsuite/features/support/code_coverage.rb b/testsuite/features/support/code_coverage.rb index 902a08e6f237..6993a125ae6c 100644 --- a/testsuite/features/support/code_coverage.rb +++ b/testsuite/features/support/code_coverage.rb @@ -1,20 +1,16 @@ -# Copyright (c) 2016-2023 SUSE LLC. +# Copyright (c) 2024 SUSE LLC. # Licensed under the terms of the MIT license. -require 'redis' +require_relative 'database_handler' require 'nokogiri' require 'open-uri' # CodeCoverage handler to produce, parse and report Code Coverage from the Jave Server to our GitHub PRs class CodeCoverage - include(Nokogiri::XML) - # Initialize a connection with a Redis database - def initialize(redis_host, redis_port, redis_username, redis_password) - @database = Redis.new(host: redis_host, port: redis_port, username: redis_username, password: redis_password) - end + include Nokogiri::XML - # Close the connection with the Redis database - def close - @database.close + # Initialize the CodeCoverage handler + def initialize + @db_handler = DatabaseHandler.new(ENV.fetch('REDIS_HOST', nil), ENV.fetch('REDIS_PORT', nil), ENV.fetch('REDIS_USERNAME', nil), ENV.fetch('REDIS_PASSWORD', nil)) end # Parse a JaCoCo XML report, extracting information that will be included in a Set on a Redis database @@ -36,17 +32,12 @@ def push_feature_coverage(feature_name) next unless Integer(counter_class.attr('covered').to_s).positive? - begin - @database.sadd("#{package_name}/#{sourcefile_name}", feature_name) - rescue StandardError => e - warn("#{e.backtrace} > #{package_name}/#{sourcefile_name} : #{feature_name}") - end + @db_handler.add("#{package_name}/#{sourcefile_name}", feature_name) end end rescue StandardError => e warn(e.backtrace) - ensure - File.delete(filename) + ensure File.delete(filename) end end diff --git a/testsuite/features/support/database_handler.rb b/testsuite/features/support/database_handler.rb new file mode 100644 index 000000000000..32c11b066220 --- /dev/null +++ b/testsuite/features/support/database_handler.rb @@ -0,0 +1,37 @@ +# Copyright (c) 2024 SUSE LLC. +# Licensed under the terms of the MIT license. +require 'redis' + +# Database handler to interact with a Redis database +class DatabaseHandler + # Initialize a connection with a Redis database + # + # @param redis_host [String] The hostname of the Redis database. + # @param redis_port [Integer] The port of the Redis database. + # @param redis_username [String] The username to authenticate with the Redis database. + # @param redis_password [String] The password to authenticate with the Redis database. + # @return [Redis] A connection with the Redis database. + def initialize(redis_host, redis_port, redis_username, redis_password) + @database = Redis.new(host: redis_host, port: redis_port, username: redis_username, password: redis_password) + end + + # Close the connection with the Redis database + def close + @database.close + end + + # Add a key-value pair to a Set, optionally selecting a database + # + # @param key [String] The key to add the value to. + # @param value [String] The value to add to the key. + # @param database [Integer] The database to select (default: 0). + # @return [String] `OK` + def add(key, value, database = 0) + begin + @database.select(database) + @database.sadd(key, value) + rescue StandardError => e + warn("#{e.backtrace} > #{key} : #{value}") + end + end +end diff --git a/testsuite/features/support/env.rb b/testsuite/features/support/env.rb index 91fd019f61a3..5b945fa570c6 100644 --- a/testsuite/features/support/env.rb +++ b/testsuite/features/support/env.rb @@ -17,6 +17,7 @@ require 'set' require 'timeout' require_relative 'code_coverage' +require_relative 'quality_intelligence' require_relative 'remote_nodes_env' require_relative 'commonlib' @@ -29,10 +30,14 @@ $debug_mode = true $stdout.puts('DEBUG MODE ENABLED.') end -if ENV['REDIS_HOST'] +if ENV['REDIS_HOST'] && ENV.fetch('CODE_COVERAGE', false) $code_coverage_mode = true $stdout.puts('CODE COVERAGE MODE ENABLED.') end +if ENV.fetch('QUALITY_INTELLIGENCE', false) + $quality_intelligence_mode = true + $stdout.puts('QUALITY INTELLIGENCE MODE ENABLED.') +end # Context per feature $context = {} @@ -117,7 +122,10 @@ def capybara_register_driver $api_test = new_api_client # Init CodeCoverage Handler -$code_coverage = CodeCoverage.new(ENV.fetch('REDIS_HOST', nil), ENV.fetch('REDIS_PORT', nil), ENV.fetch('REDIS_USERNAME', nil), ENV.fetch('REDIS_PASSWORD', nil)) if $code_coverage_mode +$code_coverage = CodeCoverage.new if $code_coverage_mode + +# Init Quality Intelligence Handler +$quality_intelligence = QualityIntelligence.new if $quality_intelligence_mode # Define the current feature scope Before do |scenario| @@ -139,7 +147,7 @@ def capybara_register_driver print_server_logs end end - page.instance_variable_set(:@touched, false) + page.instance_variable_set(:@touched, false) if Capybara::Session.instance_created? end # Test is web session is open diff --git a/testsuite/features/support/namespaces/system.rb b/testsuite/features/support/namespaces/system.rb index b5fc7cdcd68a..da319f81f115 100644 --- a/testsuite/features/support/namespaces/system.rb +++ b/testsuite/features/support/namespaces/system.rb @@ -183,6 +183,25 @@ def get_system_errata(system_id) def get_systems_errata(system_ids) @test.call('system.getRelevantErrata', sessionKey: @test.token, sids: system_ids) end + + # Returns the event history for a system. + # + # @param system_id [String] The ID of the system. + # @param offset [Integer] Number of results to skip + # @param limit [Integer] Maximum number of results + # @return [Array] An array of events + def get_event_history(system_id, offset, limit) + @test.call('system.getEventHistory', sessionKey: @test.token, sid: system_id, offset: offset, limit: limit) + end + + # Returns the event details for a system. + # + # @param system_id [String] The ID of the system. + # @param event_id [String] The ID of the event. + # @return [Hash] The event details + def get_event_details(system_id, event_id) + @test.call('system.getEventDetails', sessionKey: @test.token, sid: system_id, eid: event_id) + end end # System Configuration namespace diff --git a/testsuite/features/support/prometheus_handler.rb b/testsuite/features/support/prometheus_handler.rb new file mode 100644 index 000000000000..99016a617a20 --- /dev/null +++ b/testsuite/features/support/prometheus_handler.rb @@ -0,0 +1,33 @@ +# Copyright (c) 2024 SUSE LLC. +# Licensed under the terms of the MIT license. + +require 'net/http' +require 'prometheus/client' +require 'prometheus/client/push' +require 'uri' + +# Prometheus handler to push metrics to Prometheus +class PrometheusHandler + def initialize(push_gateway_url) + @push_gateway_url = push_gateway_url + @prometheus = Prometheus::Client::Registry.new + end + + # Push a metric to Prometheus, raising an error if the Prometheus request fails. + # + # @param job_name [String] the job name to push the metric to + # @param metric_name [String] the metric name to push + # @param metric_value [Integer] the metric value to push + # @param labels [Hash] the labels to add to the metric + # @return [void] + # @raise [SystemCallError] if the Prometheus request fails + def push_metric_to_prometheus(job_name, metric_name, metric_value, labels = {}) + begin + gauge = @prometheus.get(metric_name.to_sym) || @prometheus.gauge(metric_name.to_sym, docstring: job_name, labels: labels.keys) + gauge.set(metric_value, labels: labels) + Prometheus::Client::Push.new(job: job_name, gateway: @push_gateway_url).add(@prometheus) + rescue StandardError => e + warn(e.full_message) + end + end +end diff --git a/testsuite/features/support/quality_intelligence.rb b/testsuite/features/support/quality_intelligence.rb new file mode 100644 index 000000000000..20a05212bed2 --- /dev/null +++ b/testsuite/features/support/quality_intelligence.rb @@ -0,0 +1,47 @@ +# Copyright (c) 2024 SUSE LLC. +# Licensed under the terms of the MIT license. +require_relative 'database_handler' +require_relative 'prometheus_handler' + +# Quality Intelligence handler to produce, parse and report Quality Intelligence from the test suite +class QualityIntelligence + QI = 'quality_intelligence'.freeze + private_constant :QI + + # Initialize the QualityIntelligence handler + def initialize + @db_handler = DatabaseHandler.new(ENV.fetch('REDIS_HOST', nil), ENV.fetch('REDIS_PORT', nil), ENV.fetch('REDIS_USERNAME', nil), ENV.fetch('REDIS_PASSWORD', nil)) + @prometheus_handler = PrometheusHandler.new(ENV.fetch('PROMETHEUS_PUSH_GATEWAY_URL', 'http://nsa.mgr.suse.de:9091')) + @environment = ENV.fetch('SERVER', nil) + end + + # Report the time to complete a bootstrap of a system passed as parameter, + # raising an error if the Prometheus request fails. + # + # @param system [String] the system to bootstrap + # @param time [Integer] the time to complete the bootstrap in seconds + # @return [void] + def push_bootstrap_duration(system, time) + @prometheus_handler.push_metric_to_prometheus(QI, 'system_bootstrap_duration_seconds', time, labels: { 'system' => system, 'environment' => @environment }) + end + + # Report the time to complete the onboarding of a system passed as parameter, + # raising an error if the Prometheus request fails. + # + # @param system [String] the system to be onboarded + # @param time [Integer] the time to complete the onboarding in seconds + # @return [void] + def push_onboarding_duration(system, time) + @prometheus_handler.push_metric_to_prometheus(QI, 'system_onboarding_duration_seconds', time, labels: { 'system' => system, 'environment' => @environment }) + end + + # Report the time to complete a synchronization of a product passed as parameter, + # raising an error if the Prometheus request fails. + # + # @param product [String] the product to synchronize + # @param time [Integer] the time to complete the synchronization in seconds + # @return [void] + def push_synchronization_duration(product, time) + @prometheus_handler.push_metric_to_prometheus(QI, 'product_synch_duration_seconds', time, labels: { 'product' => product, 'environment' => @environment }) + end +end diff --git a/testsuite/features/support/system_monitoring.rb b/testsuite/features/support/system_monitoring.rb new file mode 100644 index 000000000000..8188b3c9883c --- /dev/null +++ b/testsuite/features/support/system_monitoring.rb @@ -0,0 +1,64 @@ +# Copyright (c) 2024 SUSE LLC. +# Licensed under the terms of the MIT license. + +require 'time' + +# This method should return the last bootstrap duration for the given host +# @param host [String] the hostname +# @return [Float] the duration in seconds or nil if the duration could not be determined +def last_bootstrap_duration(host) + system_name = get_system_name(host) + duration = nil + lines, _code = get_target('server').run('tail -n100 /var/log/rhn/rhn_web_api.log') + + lines.each_line do |line| + if line.include?(system_name) && line.include?('systems.bootstrap') + match = line.match(/TIME: (\d+\.\d+) seconds/) + duration = match[1].to_f if match + end + end + + raise ScriptError, "Boostrap duration not found for #{host}" if duration.nil? + + duration +end + +# This method should return the last onboarding duration for the given host +# @param host [String] the hostname +# @return [Float] the duration in seconds or nil if the duration could not be determined +def last_onboarding_duration(host) + node = get_target(host) + system_id = get_system_id(node) + events = $api_test.system.get_event_history(system_id, 0, 10) + onboarding_events = events.select { |event| event['summary'].include? 'certs, channels, packages' } + last_event_id = onboarding_events.last['id'] + event_details = $api_test.system.get_event_details(system_id, last_event_id) + Time.parse(event_details['completed']) - Time.parse(event_details['picked_up']) +end + +# This method should return the synchronization duration for the given product +# @param os_product_version [String] the product name +# @return [Float] the duration in seconds or nil if the duration could not be determined +def synchronization_duration(os_product_version) + channels_to_wait = CHANNEL_TO_SYNC_BY_OS_PRODUCT_VERSION.dig(product, os_product_version) + channels_to_wait = filter_channels(channels_to_wait, ['beta']) unless $beta_enabled + raise ScriptError, "Synchronization error, channels for #{os_product_version} in #{product} not found" if channels_to_wait.nil? + + duration = 0 + channel_to_evaluate = false + get_target('server').extract('/var/log/rhn/reposync.log', '/tmp/reposync.log') + File.foreach('/tmp/reposync.log') do |line| + if line.include?('Channel: ') + channel_name = line.split('Channel: ')[1].strip + channel_to_evaluate = channels_to_wait.include?(channel_name) + end + if line.include?('Total time: ') && channel_to_evaluate + match = line.match(/Total time: (\d+):(\d+):(\d+)/) + hours, minutes, seconds = match.captures.map(&:to_i) + total_seconds = (hours * 3600) + (minutes * 60) + seconds + duration += total_seconds + channel_to_evaluate = false + end + end + duration +end