diff --git a/horde_worker_regen/process_management/process_manager.py b/horde_worker_regen/process_management/process_manager.py index 840a3561..88f944f7 100644 --- a/horde_worker_regen/process_management/process_manager.py +++ b/horde_worker_regen/process_management/process_manager.py @@ -1439,7 +1439,6 @@ async def api_submit_job(self) -> None: async with self._completed_jobs_lock: self.completed_jobs.remove(completed_job_info) self._consecutive_failed_job_submits = 0 - return if "already submitted" in job_submit_response.message: @@ -1447,7 +1446,13 @@ async def api_submit_job(self) -> None: async with self._completed_jobs_lock: self.completed_jobs.remove(completed_job_info) self._consecutive_failed_job_submits = 0 + return + if "Please check your worker speed" in job_submit_response.message: + logger.error(job_submit_response.message) + async with self._completed_jobs_lock: + self.completed_jobs.remove(completed_job_info) + self._consecutive_failed_job_submits = 0 return error_string = "Failed to submit job (API Error)" @@ -1620,9 +1625,12 @@ async def api_job_pop(self) -> None: # Assuming a megapixelstep takes 0.75 seconds, if 2/3 of the time has passed since the limit was triggered, # we can assume that the pending megapixelsteps will be below the limit soon. Otherwise we continue to wait - if not (time.time() - self._triggered_max_pending_megapixelsteps_time) > ( - (self._max_pending_megapixelsteps * 0.75) * (2 / 3) - ): + seconds_to_wait = (self._max_pending_megapixelsteps * 0.75) * (2 / 3) + + if self.get_pending_megapixelsteps() > 200: + seconds_to_wait = self._max_pending_megapixelsteps * 0.75 + + if not (time.time() - self._triggered_max_pending_megapixelsteps_time) > seconds_to_wait: return self._triggered_max_pending_megapixelsteps = False