Fix handling squeue invalid jobid

docs/source/changelog.rst

@@ -1,4 +1,4 @@
-.. Created by changelog.py at 2023-11-29, command
+.. Created by changelog.py at 2023-12-16, command
    '/Users/giffler/.cache/pre-commit/repor6pnmwlm/py_env-python3.10/bin/changelog docs/source/changes compile --categories Added Changed Fixed Security Deprecated --output=docs/source/changelog.rst'
    based on the format of 'https://keepachangelog.com/'
 

tardis/adapters/sites/slurm.py

@@ -45,8 +45,18 @@ async def squeue(
     try:
         slurm_status = await executor.run_command(cmd)
     except CommandExecutionFailure as cf:
+        # In case a job is already completed and
+        # only **one** non-existing job id is provided
+        # squeue is failing with exit code 1
+        # and prints "Invalid job id specified" to stderr
+        if "Invalid job id specified" in cf.stderr and len(resource_attributes) == 1:
+            logger.info(
+                f"{remote_resource_ids} is not valid anymore. Assuming it is completed."
+            )
+            return [{"State": "COMPLETED"}]
         logger.warning(f"Slurm status update has failed due to {cf}.")
         raise
+
     else:
         for row in csv_parser(
             slurm_status.stdout, fieldnames=tuple(attributes.keys()), delimiter="|"

tests/adapters_t/sites_t/test_slurm.py

@@ -304,7 +304,31 @@ def test_resource_state_translation(self):
             self.mock_executor.reset_mock()
 
     @mock_executor_run_command("")
-    def test_resource_status_of_completed_jobs(self):
+    def test_resource_status_of_completed_jobs_w_empty_reply(self):
+        response = run_async(
+            self.slurm_adapter.resource_status,
+            AttributeDict(
+                resource_id="1390065",
+                remote_resource_uuid="1351043",
+            ),
+        )
+
+        self.assertEqual(response.resource_status, ResourceStatus.Deleted)
+
+        self.mock_executor.return_value.run_command.assert_called_with(
+            'squeue -o "%A|%N|%T" -h -t all --job=1351043'
+        )
+
+    @mock_executor_run_command(
+        stdout="",
+        raise_exception=CommandExecutionFailure(
+            message="Run command squeue --job=1351043 via SSHExecutor failed",
+            stdout="",
+            stderr="slurm_load_jobs error: Invalid job id specified",
+            exit_code=1,
+        ),
+    )
+    def test_resource_status_of_completed_jobs_w_raised_exception(self):
         response = run_async(
             self.slurm_adapter.resource_status,
             AttributeDict(