diff --git a/horde_worker_regen/process_management/process_manager.py b/horde_worker_regen/process_management/process_manager.py
index 0052e0f2..99267581 100644
--- a/horde_worker_regen/process_management/process_manager.py
+++ b/horde_worker_regen/process_management/process_manager.py
@@ -1121,6 +1121,8 @@ def _replace_inference_process(self, process_info: HordeProcessInfo) -> None:
 
         if process_info.last_process_state == HordeProcessState.INFERENCE_STARTING:
             self._inference_semaphore.release()
+        elif process_info.last_process_state == HordeProcessState.DOWNLOADING_AUX_MODEL:
+            self._aux_model_lock.release()
         self._start_inference_process(process_info.process_id)
 
     total_num_completed_jobs: int = 0