Merge pull request #27 from Haidra-Org/regen-day2

fix: treat certain API errors better; give more time for high queued MPS

horde_worker_regen/process_management/process_manager.py

@@ -1439,15 +1439,20 @@ async def api_submit_job(self) -> None:
                     async with self._completed_jobs_lock:
                         self.completed_jobs.remove(completed_job_info)
                         self._consecutive_failed_job_submits = 0
-
                     return
 
                 if "already submitted" in job_submit_response.message:
                     logger.debug(f"Job {job_info.id_} has already been submitted, removing from completed jobs")
                     async with self._completed_jobs_lock:
                         self.completed_jobs.remove(completed_job_info)
                         self._consecutive_failed_job_submits = 0
+                    return
 
+                if "Please check your worker speed" in job_submit_response.message:
+                    logger.error(job_submit_response.message)
+                    async with self._completed_jobs_lock:
+                        self.completed_jobs.remove(completed_job_info)
+                        self._consecutive_failed_job_submits = 0
                     return
 
                 error_string = "Failed to submit job (API Error)"
@@ -1620,9 +1625,12 @@ async def api_job_pop(self) -> None:
             # Assuming a megapixelstep takes 0.75 seconds, if 2/3 of the time has passed since the limit was triggered,
             # we can assume that the pending megapixelsteps will be below the limit soon. Otherwise we continue to wait
 
-            if not (time.time() - self._triggered_max_pending_megapixelsteps_time) > (
-                (self._max_pending_megapixelsteps * 0.75) * (2 / 3)
-            ):
+            seconds_to_wait = (self._max_pending_megapixelsteps * 0.75) * (2 / 3)
+
+            if self.get_pending_megapixelsteps() > 200:
+                seconds_to_wait = self._max_pending_megapixelsteps * 0.75
+
+            if not (time.time() - self._triggered_max_pending_megapixelsteps_time) > seconds_to_wait:
                 return
 
             self._triggered_max_pending_megapixelsteps = False