From b03451a2af43a2c332881a759d4957a59f293e01 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 8 Nov 2024 19:41:18 -0600 Subject: [PATCH] Add st2tests.config.db_opts_as_env_vars for itests Integration tests need to start subprocesses that use the same database as the test. This matters when running under pantsbuild, because pants runs several instances of pytest in parallel. We configured pants to pass an env var to disambiguate parallel test runs: ST2TESTS_PARALLEL_SLOT. Then, the db name gets suffixed with this slot number at runtime. But, when an integration test runs production code in a subprocess, the production code does not use-- and should not use--any ST2TESTS_* vars, meaning the subprocess ends up using what is configured in the conf file instead of the parallel-safe test db. Thanks to a new-ish oslo_config feature, we can now update config via env variables. So, make use of that in integration tests to override the conf-file provided values with test-provided values. --- .../integration/test_gunicorn_configs.py | 3 ++ .../test_register_content_script.py | 29 ++++++++++++------- .../test_service_setup_log_level_filtering.py | 8 +++-- .../integration/test_garbage_collector.py | 5 ++-- .../integration/test_sensor_container.py | 2 +- st2tests/st2tests/config.py | 15 ++++++++++ 6 files changed, 45 insertions(+), 17 deletions(-) diff --git a/st2api/tests/integration/test_gunicorn_configs.py b/st2api/tests/integration/test_gunicorn_configs.py index c4c05bb155..84fda239f5 100644 --- a/st2api/tests/integration/test_gunicorn_configs.py +++ b/st2api/tests/integration/test_gunicorn_configs.py @@ -22,6 +22,7 @@ import eventlet from eventlet.green import subprocess +import st2tests.config from st2common.models.utils import profiling from st2common.util.shell import kill_process from st2tests.base import IntegrationTestCase @@ -41,6 +42,7 @@ def test_st2api_wsgi_entry_point(self): ) env = os.environ.copy() env["ST2_CONFIG_PATH"] = ST2_CONFIG_PATH + env.update(st2tests.config.db_opts_as_env_vars()) process = subprocess.Popen(cmd, env=env, shell=True, preexec_fn=os.setsid) try: self.add_process(process=process) @@ -60,6 +62,7 @@ def test_st2auth(self): ) env = os.environ.copy() env["ST2_CONFIG_PATH"] = ST2_CONFIG_PATH + env.update(st2tests.config.db_opts_as_env_vars()) process = subprocess.Popen(cmd, env=env, shell=True, preexec_fn=os.setsid) try: self.add_process(process=process) diff --git a/st2common/tests/integration/test_register_content_script.py b/st2common/tests/integration/test_register_content_script.py index b0288a910e..14abf12e45 100644 --- a/st2common/tests/integration/test_register_content_script.py +++ b/st2common/tests/integration/test_register_content_script.py @@ -19,6 +19,7 @@ import sys import glob +import st2tests.config from st2tests.base import IntegrationTestCase from st2common.util.shell import run_command from st2tests import config as test_config @@ -56,7 +57,7 @@ def test_register_from_pack_success(self): "--register-runner-dir=%s" % (runner_dirs), ] cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts - exit_code, _, stderr = run_command(cmd=cmd) + exit_code, _, stderr = self._run_command(cmd=cmd) self.assertIn("Registered 3 actions.", stderr) self.assertEqual(exit_code, 0) @@ -71,7 +72,7 @@ def test_register_from_pack_fail_on_failure_pack_dir_doesnt_exist(self): "--register-no-fail-on-failure", ] cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts - exit_code, _, _ = run_command(cmd=cmd) + exit_code, _, _ = self._run_command(cmd=cmd) self.assertEqual(exit_code, 0) # Fail on failure, should fail @@ -81,7 +82,7 @@ def test_register_from_pack_fail_on_failure_pack_dir_doesnt_exist(self): "--register-fail-on-failure", ] cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts - exit_code, _, stderr = run_command(cmd=cmd) + exit_code, _, stderr = self._run_command(cmd=cmd) self.assertIn('Directory "doesntexistblah" doesn\'t exist', stderr) self.assertEqual(exit_code, 1) @@ -97,7 +98,7 @@ def test_register_from_pack_action_metadata_fails_validation(self): ] cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts - exit_code, _, stderr = run_command(cmd=cmd) + exit_code, _, stderr = self._run_command(cmd=cmd) self.assertIn("Registered 0 actions.", stderr) self.assertEqual(exit_code, 0) @@ -109,7 +110,7 @@ def test_register_from_pack_action_metadata_fails_validation(self): "--register-runner-dir=%s" % (runner_dirs), ] cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts - exit_code, _, stderr = run_command(cmd=cmd) + exit_code, _, stderr = self._run_command(cmd=cmd) self.assertIn("object has no attribute 'get'", stderr) self.assertEqual(exit_code, 1) @@ -127,7 +128,7 @@ def test_register_from_packs_doesnt_throw_on_missing_pack_resource_folder(self): "-v", "--register-sensors", ] - exit_code, _, stderr = run_command(cmd=cmd) + exit_code, _, stderr = self._run_command(cmd=cmd) self.assertIn("Registered 0 sensors.", stderr, "Actual stderr: %s" % (stderr)) self.assertEqual(exit_code, 0) @@ -139,7 +140,7 @@ def test_register_from_packs_doesnt_throw_on_missing_pack_resource_folder(self): "--register-all", "--register-no-fail-on-failure", ] - exit_code, _, stderr = run_command(cmd=cmd) + exit_code, _, stderr = self._run_command(cmd=cmd) self.assertIn("Registered 0 actions.", stderr) self.assertIn("Registered 0 sensors.", stderr) self.assertIn("Registered 0 rules.", stderr) @@ -155,7 +156,7 @@ def test_register_all_and_register_setup_virtualenvs(self): "--register-setup-virtualenvs", "--register-no-fail-on-failure", ] - exit_code, stdout, stderr = run_command(cmd=cmd) + exit_code, stdout, stderr = self._run_command(cmd=cmd) self.assertIn("Registering actions", stderr, "Actual stderr: %s" % (stderr)) self.assertIn("Registering rules", stderr) self.assertIn("Setup virtualenv for %s pack(s)" % ("1"), stderr) @@ -170,7 +171,7 @@ def test_register_setup_virtualenvs(self): "--register-setup-virtualenvs", "--register-no-fail-on-failure", ] - exit_code, stdout, stderr = run_command(cmd=cmd) + exit_code, stdout, stderr = self._run_command(cmd=cmd) self.assertIn('Setting up virtualenv for pack "dummy_pack_1"', stderr) self.assertIn("Setup virtualenv for 1 pack(s)", stderr) @@ -186,7 +187,7 @@ def test_register_recreate_virtualenvs(self): "--register-setup-virtualenvs", "--register-no-fail-on-failure", ] - exit_code, stdout, stderr = run_command(cmd=cmd) + exit_code, stdout, stderr = self._run_command(cmd=cmd) self.assertIn('Setting up virtualenv for pack "dummy_pack_1"', stderr) self.assertIn("Setup virtualenv for 1 pack(s)", stderr) @@ -200,9 +201,15 @@ def test_register_recreate_virtualenvs(self): "--register-recreate-virtualenvs", "--register-no-fail-on-failure", ] - exit_code, stdout, stderr = run_command(cmd=cmd) + exit_code, stdout, stderr = self._run_command(cmd=cmd) self.assertIn('Setting up virtualenv for pack "dummy_pack_1"', stderr) self.assertIn("Virtualenv successfully removed.", stderr) self.assertIn("Setup virtualenv for 1 pack(s)", stderr) self.assertEqual(exit_code, 0) + + @staticmethod + def _run_command(cmd): + env = os.environ.copy() + env.update(st2tests.config.db_opts_as_env_vars()) + return run_command(cmd=cmd, env=env) diff --git a/st2common/tests/integration/test_service_setup_log_level_filtering.py b/st2common/tests/integration/test_service_setup_log_level_filtering.py index 36c891896c..a4f3cc90a5 100644 --- a/st2common/tests/integration/test_service_setup_log_level_filtering.py +++ b/st2common/tests/integration/test_service_setup_log_level_filtering.py @@ -21,6 +21,7 @@ import eventlet from eventlet.green import subprocess +import st2tests.config from st2tests.base import IntegrationTestCase from st2tests.fixtures.conf.fixture import FIXTURE_PATH as CONF_FIXTURES_PATH @@ -223,13 +224,16 @@ def test_kombu_heartbeat_tick_log_messages_are_excluded(self): stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n")) self.assertNotIn("heartbeat_tick", stdout) - def _start_process(self, config_path, env=None): + @staticmethod + def _start_process(config_path, env=None): cmd = CMD + [config_path] cwd = os.path.abspath(os.path.join(BASE_DIR, "../../../")) cwd = os.path.abspath(cwd) + env = env or os.environ.copy() + env.update(st2tests.config.db_opts_as_env_vars()) process = subprocess.Popen( cmd, - env=env or os.environ.copy(), + env=env, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, diff --git a/st2reactor/tests/integration/test_garbage_collector.py b/st2reactor/tests/integration/test_garbage_collector.py index 80e3a39b4d..630dad86de 100644 --- a/st2reactor/tests/integration/test_garbage_collector.py +++ b/st2reactor/tests/integration/test_garbage_collector.py @@ -20,8 +20,7 @@ import signal import datetime -from oslo_config import cfg - +import st2tests.config from st2common.util import concurrency from st2common.constants import action as action_constants from st2common.util import date as date_utils @@ -277,7 +276,7 @@ def _create_inquiry(self, ttl, timestamp): def _start_garbage_collector(self): subprocess = concurrency.get_subprocess_module() env = os.environ.copy() - env["ST2_DATABASE__DB_NAME"] = cfg.CONF.database.db_name + env.update(st2tests.config.db_opts_as_env_vars()) process = subprocess.Popen( CMD_INQUIRY, stdout=subprocess.PIPE, diff --git a/st2reactor/tests/integration/test_sensor_container.py b/st2reactor/tests/integration/test_sensor_container.py index 16fb0e554f..0ebae08604 100644 --- a/st2reactor/tests/integration/test_sensor_container.py +++ b/st2reactor/tests/integration/test_sensor_container.py @@ -245,7 +245,7 @@ def test_single_sensor_mode(self): def _start_sensor_container(self, cmd=DEFAULT_CMD): subprocess = concurrency.get_subprocess_module() env = os.environ.copy() - env["ST2_DATABASE__DB_NAME"] = cfg.CONF.database.db_name + env.update(st2tests.config.db_opts_as_env_vars()) print("Using command: %s" % (" ".join(cmd))) process = subprocess.Popen( cmd, diff --git a/st2tests/st2tests/config.py b/st2tests/st2tests/config.py index 95bf49314f..76fc66316b 100644 --- a/st2tests/st2tests/config.py +++ b/st2tests/st2tests/config.py @@ -16,6 +16,7 @@ from __future__ import absolute_import import os +from typing import Dict from oslo_config import cfg, types @@ -92,6 +93,20 @@ def _override_db_opts(): CONF.set_override(name="host", override="127.0.0.1", group="database") +def db_opts_as_env_vars() -> Dict[str, str]: + env = { + "ST2_DATABASE__HOST": CONF.database.host, + "ST2_DATABASE__PORT": str(CONF.database.port), + "ST2_DATABASE__DB_NAME": CONF.database.db_name, + "ST2_DATABASE__CONNECTION_TIMEOUT": CONF.database.connection_timeout, + } + if hasattr(CONF.database, "username"): + env["ST2_DATABASE__USERNAME"] = CONF.database.username + if hasattr(CONF.database, "password"): + env["ST2_DATABASE__PASSWORD"] = CONF.database.password + return env + + def _override_common_opts(): packs_base_path = get_fixtures_packs_base_path() CONF.set_override(name="base_path", override=packs_base_path, group="system")