Let job handler work queue call job wrapper fail

This avoids committing the transaction and then have it fail on the next
iteration. Should fix the app startup failing in galaxyproject#17079

lib/galaxy/jobs/handler.py

@@ -2,6 +2,7 @@
 Galaxy job handler, prepares, runs, tracks, and finishes Galaxy jobs
 """
 import datetime
+import functools
 import os
 import time
 from collections import defaultdict
@@ -1268,7 +1269,7 @@ def recover(self, job, job_wrapper):
         except ObjectNotFound:
             msg = "Could not recover job working directory after Galaxy restart"
             log.exception(f"recover(): ({job_wrapper.job_id}) {msg}")
-            job_wrapper.fail(msg)
+            runner.work_queue.put(functools.partial(job_wrapper.fail, msg))
 
     def shutdown(self):
         failures = []

lib/galaxy/jobs/runners/__init__.py

@@ -106,14 +106,14 @@ def __init__(self, app: "GalaxyManagerApplication", nworkers: int, **kwargs):
         self.runner_params = RunnerParams(specs=runner_param_specs, params=kwargs)
         self.runner_state_handlers = build_state_handlers()
         self._should_stop = False
+        self.work_queue = Queue()
 
     def start(self):
         for start_method in self.start_methods:
             getattr(self, start_method, lambda: None)()
 
     def _init_worker_threads(self):
         """Start ``nworkers`` worker threads."""
-        self.work_queue = Queue()
         self.work_threads = []
         log.debug(f"Starting {self.nworkers} {self.runner_name} workers")
         for i in range(self.nworkers):