From 63f2e41fa526f037825e7dae36e319c6ff81fbe8 Mon Sep 17 00:00:00 2001 From: Sebastiaan la Fleur Date: Tue, 5 Nov 2024 17:38:57 +0100 Subject: [PATCH 1/2] 87: Add system test to check if job timeout feature correctly cancels timed out jobs. --- .env.template | 3 ++ docker-compose.yml | 3 ++ system_tests/docker-compose.override.yml | 9 +++++- system_tests/src/test_workflows_steps.py | 41 ++++++++++++++++++++++-- 4 files changed, 53 insertions(+), 3 deletions(-) diff --git a/.env.template b/.env.template index 231cd46..305c28e 100644 --- a/.env.template +++ b/.env.template @@ -22,6 +22,9 @@ INFLUXDB_WRITE_USER_PASSWORD=write9012 INFLUXDB_FRONTEND_ADMIN_USER=front1 INFLUXDB_FRONTEND_ADMIN_USER_PASSWORD=front9012 +TIMEOUT_JOB_MANAGER_START_BUFFER_SEC=60 +TIMEOUT_JOB_HANDLER_RERUN_SEC=30 + WORKFLOW_SETTINGS_FILE=./config/workflow_config_nwn_9sept2024.json LOG_LEVEL=INFO diff --git a/docker-compose.yml b/docker-compose.yml index ae1a16b..2e688d9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -118,6 +118,9 @@ services: POSTGRESQL_USERNAME: ${POSTGRES_ORCHESTRATOR_USER_NAME} POSTGRESQL_PASSWORD: ${POSTGRES_ORCHESTRATOR_USER_PASSWORD} + TIMEOUT_JOB_MANAGER_START_BUFFER_SEC: ${TIMEOUT_JOB_MANAGER_START_BUFFER_SEC} + TIMEOUT_JOB_HANDLER_RERUN_SEC: ${TIMEOUT_JOB_HANDLER_RERUN_SEC} + LOG_LEVEL: ${LOG_LEVEL} depends_on: rabbitmq: diff --git a/system_tests/docker-compose.override.yml b/system_tests/docker-compose.override.yml index 7a3d1c7..e2f634f 100644 --- a/system_tests/docker-compose.override.yml +++ b/system_tests/docker-compose.override.yml @@ -15,4 +15,11 @@ services: RABBITMQ_PASSWORD: somepass1 RABBITMQ_VIRTUALHOST: omotes RABBITMQ_HOST: rabbitmq - RABBITMQ_PORT: 5672 \ No newline at end of file + RABBITMQ_PORT: 5672 + + orchestrator: + environment: + TIMEOUT_JOB_MANAGER_START_BUFFER_SEC: 2 + TIMEOUT_JOB_HANDLER_RERUN_SEC: 5 + + LOG_LEVEL: DEBUG diff --git a/system_tests/src/test_workflows_steps.py b/system_tests/src/test_workflows_steps.py index bef9aa8..4b9fd12 100644 --- a/system_tests/src/test_workflows_steps.py +++ b/system_tests/src/test_workflows_steps.py @@ -45,7 +45,9 @@ def __init__(self): self.result = None def wait_until_result(self, timeout: float | None = None): - self.has_result.wait(timeout) + assert self.has_result.wait(timeout), ( + f"The result was not received within the " f"timeout {timeout} seconds." + ) def handle_on_finished(self, _: Job, result: JobResult): self.has_result.set() @@ -93,11 +95,12 @@ def submit_a_job( workflow_type: str, params_dict: ParamsDict, omotes_job_result_handler: OmotesJobHandler, + job_timeout: timedelta = timedelta(hours=1), ) -> Job: return omotes_client.submit_job( esdl=esdl_file, workflow_type=omotes_client.get_workflow_type_manager().get_workflow_by_name(workflow_type), - job_timeout=timedelta(hours=1), + job_timeout=job_timeout, params_dict=params_dict, callback_on_finished=omotes_job_result_handler.handle_on_finished, callback_on_progress_update=omotes_job_result_handler.handle_on_progress_update, @@ -344,3 +347,37 @@ def test__simulator__check_if_progress_updates_are_received(self) -> None: self.assertEqual(submitted_job.id, uuid.UUID(first_update.uuid)) self.assertIsNotNone(last_update) self.assertEqual(submitted_job.id, uuid.UUID(last_update.uuid)) + + def test__simulator__job_timeout_working(self) -> None: + """This test depends on the environment variables: + TIMEOUT_JOB_MANAGER_START_BUFFER_SEC (currently set to 2 seconds) + TIMEOUT_JOB_HANDLER_RERUN_SEC (currently set to 5 seconds) + + They are defined in system_tests/docker-compose.override.yml + """ + # Arrange + result_handler = OmotesJobHandler() + esdl_file = retrieve_esdl_file("./test_esdl/input/simulator_tutorial.esdl") + workflow_type = "simulator" + job_timeout = timedelta(seconds=5) + result_timeout_seconds = 10.0 + params_dict = { + "timestep": datetime.timedelta(hours=1), + "start_time": datetime.datetime(2019, 1, 1, 0, 0, 0), + "end_time": datetime.datetime(2019, 2, 1, 0, 0, 0), + } + + # Act + with omotes_client() as omotes_client_: + submit_a_job( + omotes_client_, + esdl_file, + workflow_type, + params_dict, + result_handler, + job_timeout=job_timeout, + ) + result_handler.wait_until_result(result_timeout_seconds) + + # Assert + self.expect_a_result(result_handler, JobResult.CANCELLED) From 050cd97a0a403480d23912602d2d814a059ce002 Mon Sep 17 00:00:00 2001 From: Sebastiaan la Fleur Date: Tue, 5 Nov 2024 17:42:15 +0100 Subject: [PATCH 2/2] 87: Remove unnecessary log_level in system test docker override. --- system_tests/docker-compose.override.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/system_tests/docker-compose.override.yml b/system_tests/docker-compose.override.yml index e2f634f..9de4319 100644 --- a/system_tests/docker-compose.override.yml +++ b/system_tests/docker-compose.override.yml @@ -21,5 +21,3 @@ services: environment: TIMEOUT_JOB_MANAGER_START_BUFFER_SEC: 2 TIMEOUT_JOB_HANDLER_RERUN_SEC: 5 - - LOG_LEVEL: DEBUG