Make integration test timeouts configurable

Fixes: #969

Depending on platforms, the default timeouts used in the integration
tests need to be adjusted. In order to prevent code modification, lets
have the three currently available, high-level timeouts be configurable
via tmt.

Signed-off-by: Michael Engel <[email protected]>

tests/README.md

@@ -150,6 +150,19 @@ execution result directory, for example:
 
 `/var/tmp/tmt/run-001/plans/tier0/report/default-0/report`
 
+## Changing timeouts for integration tests
+
+In some cases it might be necessary to adjust the default timeouts that are used in different steps of an integration tests execution cycle. The currently available environment variables as well as their default values are:
+
+```shell
+# in seconds
+TIMEOUT_TEST_SETUP=20
+TIMEOUT_TEST_RUN=45
+TIMEOUT_COLLECT_TEST_RESULTS=20
+```
+
+These can be set either in the `environment` section of the tmt plan or using the `-e` option when running tmt, e.g. `-eTIMEOUT_TEST_SETUP=40`.
+
 ## Developing integration tests
 
 ### Code Style

tests/bluechi_test/fixtures.py

@@ -69,6 +69,33 @@ def machines_ssh_password() -> str:
     return _get_env_value("SSH_PASSWORD", "")
 
 
+def _safely_parse_int(input: str, default: int) -> int:
+    if input.isdigit():
+        return int(input)
+    return default
+
+
+@pytest.fixture(scope="session")
+def timeout_test_setup() -> int:
+    """Returns the timeout for setting up the test setup"""
+
+    return _safely_parse_int(_get_env_value("TIMEOUT_TEST_SETUP", ""), 20)
+
+
+@pytest.fixture(scope="session")
+def timeout_test_run() -> int:
+    """Returns the timeout for executing the actual test"""
+
+    return _safely_parse_int(_get_env_value("TIMEOUT_TEST_RUN", ""), 45)
+
+
+@pytest.fixture(scope="session")
+def timeout_collecting_test_results() -> int:
+    """Returns the timeout for collecting all test results"""
+
+    return _safely_parse_int(_get_env_value("TIMEOUT_COLLECT_TEST_RESULTS", ""), 20)
+
+
 def _read_topology() -> Dict[str, Any]:
     """
     Returns the parsed YAML for the tmt guest topology:
@@ -216,6 +243,9 @@ def bluechi_test(
     additional_ports: dict,
     machines_ssh_user: str,
     machines_ssh_password: str,
+    timeout_test_setup: int,
+    timeout_test_run: int,
+    timeout_collecting_test_results: int,
 ) -> BluechiTest:
 
     if is_multihost_run:
@@ -227,6 +257,9 @@ def bluechi_test(
             tmt_test_data_dir,
             run_with_valgrind,
             run_with_coverage,
+            timeout_test_setup,
+            timeout_test_run,
+            timeout_collecting_test_results,
         )
 
     return BluechiContainerTest(
@@ -238,5 +271,8 @@ def bluechi_test(
         tmt_test_data_dir,
         run_with_valgrind,
         run_with_coverage,
+        timeout_test_setup,
+        timeout_test_run,
+        timeout_collecting_test_results,
         additional_ports,
     )

tests/bluechi_test/test.py

@@ -19,7 +19,7 @@
     BluechiControllerMachine,
     BluechiMachine,
 )
-from bluechi_test.util import TIMEOUT_GATHER, TIMEOUT_SETUP, TIMEOUT_TEST, Timeout
+from bluechi_test.util import Timeout
 from podman import PodmanClient
 
 LOGGER = logging.getLogger(__name__)
@@ -32,12 +32,18 @@ def __init__(
         tmt_test_data_dir: str,
         run_with_valgrind: bool,
         run_with_coverage: bool,
+        timeout_test_setup: int,
+        timeout_test_run: int,
+        timeout_collecting_test_results: int,
     ) -> None:
 
         self.tmt_test_serial_number = tmt_test_serial_number
         self.tmt_test_data_dir = tmt_test_data_dir
         self.run_with_valgrind = run_with_valgrind
         self.run_with_coverage = run_with_coverage
+        self.timeout_test_setup = timeout_test_setup
+        self.timeout_test_run = timeout_test_run
+        self.timeout_collecting_test_results = timeout_collecting_test_results
 
         self.bluechi_controller_config: BluechiControllerConfig = None
         self.bluechi_node_configs: List[BluechiAgentConfig] = []
@@ -149,15 +155,12 @@ def run(
         exec: Callable[
             [BluechiControllerMachine, Dict[str, BluechiAgentMachine]], None
         ],
-        timeout_setup: int = TIMEOUT_SETUP,
-        timeout_test: int = TIMEOUT_TEST,
-        timeout_gather: int = TIMEOUT_GATHER,
     ):
         LOGGER.info("Test execution started")
 
         successful = False
         try:
-            with Timeout(timeout_setup, "Timeout setting up BlueChi system"):
+            with Timeout(self.timeout_test_setup, "Timeout setting up BlueChi system"):
                 successful, container = self.setup()
             ctrl_container, node_container = container
         except TimeoutError as ex:
@@ -171,7 +174,7 @@ def run(
 
         test_result = None
         try:
-            with Timeout(timeout_test, "Timeout running test"):
+            with Timeout(self.timeout_test_run, "Timeout running test"):
                 exec(ctrl_container, node_container)
         except Exception as ex:
             test_result = ex
@@ -186,7 +189,10 @@ def run(
             traceback.print_exc()
 
         try:
-            with Timeout(timeout_gather, "Timeout collecting test artifacts"):
+            with Timeout(
+                self.timeout_collecting_test_results,
+                "Timeout collecting test artifacts",
+            ):
                 self.gather_logs(ctrl_container, node_container)
                 if self.run_with_valgrind:
                     self.check_valgrind_logs()
@@ -215,6 +221,9 @@ def __init__(
         tmt_test_data_dir: str,
         run_with_valgrind: bool,
         run_with_coverage: bool,
+        timeout_test_setup: int,
+        timeout_test_run: int,
+        timeout_collecting_test_results: int,
         additional_ports: Dict,
     ) -> None:
 
@@ -223,6 +232,9 @@ def __init__(
             tmt_test_data_dir,
             run_with_valgrind,
             run_with_coverage,
+            timeout_test_setup,
+            timeout_test_run,
+            timeout_collecting_test_results,
         )
 
         self.podman_client = podman_client
@@ -324,13 +336,19 @@ def __init__(
         tmt_test_data_dir: str,
         run_with_valgrind: bool,
         run_with_coverage: bool,
+        timeout_test_setup: int,
+        timeout_test_run: int,
+        timeout_collecting_test_results: int,
     ) -> None:
 
         super().__init__(
             tmt_test_serial_number,
             tmt_test_data_dir,
             run_with_valgrind,
             run_with_coverage,
+            timeout_test_setup,
+            timeout_test_run,
+            timeout_collecting_test_results,
         )
 
         self.available_hosts = available_hosts

tests/bluechi_test/util.py

@@ -70,14 +70,6 @@ def remove_control_chars(value: str) -> str:
     return _ANSI_SEQUENCE.sub("", value)
 
 
-# timeout for setting up tests in s
-TIMEOUT_SETUP = 20
-# timeout for running tests in s
-TIMEOUT_TEST = 45
-# timeout for collecting test results in s
-TIMEOUT_GATHER = 20
-
-
 class Timeout:
     def __init__(self, seconds=1, error_message="Timeout"):
         self.seconds = seconds