diff --git a/bridgeData_template.yaml b/bridgeData_template.yaml index 48fcbb92..48c7fd5f 100644 --- a/bridgeData_template.yaml +++ b/bridgeData_template.yaml @@ -261,7 +261,7 @@ models_to_skip: # Suppress speed warnings if jobs are taking too long. # Note: If you are getting these messages, you are serving jobs much slower than ideal. # Lower your max_power for more kudos/hr. -suppress_speed_warnings: false # Currently unused in reGen +suppress_speed_warnings: false # Exit if an unhandled fault occurs. Useful for setting up the worker as a system service. exit_on_unhandled_faults: false diff --git a/horde_worker_regen/__init__.py b/horde_worker_regen/__init__.py index 3fea66df..0a901426 100644 --- a/horde_worker_regen/__init__.py +++ b/horde_worker_regen/__init__.py @@ -8,7 +8,7 @@ ASSETS_FOLDER_PATH = Path(__file__).parent / "assets" -__version__ = "9.0.6" +__version__ = "9.0.7" import pkg_resources # noqa: E402 diff --git a/horde_worker_regen/_version_meta.json b/horde_worker_regen/_version_meta.json index c140cae0..c508e732 100644 --- a/horde_worker_regen/_version_meta.json +++ b/horde_worker_regen/_version_meta.json @@ -1,5 +1,5 @@ { - "recommended_version": "9.0.6", + "recommended_version": "9.0.7", "required_min_version": "9.0.2", "required_min_version_update_date": "2024-09-26", "required_min_version_info": { diff --git a/horde_worker_regen/bridge_data/data_model.py b/horde_worker_regen/bridge_data/data_model.py index 7ee989ec..fc26e0ee 100644 --- a/horde_worker_regen/bridge_data/data_model.py +++ b/horde_worker_regen/bridge_data/data_model.py @@ -50,6 +50,8 @@ class reGenBridgeData(CombinedHordeBridgeData): download_timeout: int = Field(default=TOTAL_LORA_DOWNLOAD_TIMEOUT + 1) preload_timeout: int = Field(default=60, ge=15) + minutes_allowed_without_jobs: int = Field(default=30, ge=0, lt=60 * 60) + horde_model_stickiness: float = Field(default=0.0, le=1.0, ge=0.0, alias="model_stickiness") """ A percent chance (expressed as a decimal between 0 and 1) that the currently loaded models will diff --git a/horde_worker_regen/process_management/process_manager.py b/horde_worker_regen/process_management/process_manager.py index 6f0257c9..63022950 100644 --- a/horde_worker_regen/process_management/process_manager.py +++ b/horde_worker_regen/process_management/process_manager.py @@ -3408,15 +3408,14 @@ async def api_job_pop(self) -> None: info_string += f"(Skipped reasons: {job_pop_response.skipped.model_dump(exclude_defaults=True)})" if job_pop_response.id_ is None: + self._last_pop_no_jobs_available = True logger.info(info_string) - cur_time = time.time() - if self._last_pop_no_jobs_available_time == 0.0: - self._last_pop_no_jobs_available_time = cur_time + if len(self.job_deque) == 0: + if self._last_pop_no_jobs_available_time == 0.0: + self._last_pop_no_jobs_available_time = cur_time - self._time_spent_no_jobs_available += cur_time - self._last_pop_no_jobs_available_time - self._last_pop_no_jobs_available_time = cur_time - - self._last_pop_no_jobs_available = True + self._time_spent_no_jobs_available += cur_time - self._last_pop_no_jobs_available_time + self._last_pop_no_jobs_available_time = cur_time return self.job_faults[job_pop_response.id_] = [] @@ -3889,7 +3888,8 @@ def _print_deadlock_info() -> None: def print_status_method(self) -> None: """Print the status of the worker if it's time to do so.""" - if time.time() - self._last_status_message_time > self._status_message_frequency: + cur_time = time.time() + if cur_time - self._last_status_message_time > self._status_message_frequency: process_info_strings = self._process_map.get_process_info_strings() logger.info("Process info:") for process_info_string in process_info_strings: @@ -4023,7 +4023,7 @@ def print_status_method(self) -> None: ) if self._too_many_consecutive_failed_jobs: - time_since_failure = time.time() - self._too_many_consecutive_failed_jobs_time + time_since_failure = cur_time - self._too_many_consecutive_failed_jobs_time logger.error( "Too many consecutive failed jobs. This may be due to a misconfiguration or other issue. " "Please check your logs and configuration.", @@ -4033,17 +4033,27 @@ def print_status_method(self) -> None: f"{self._too_many_consecutive_failed_jobs_wait_time} seconds must pass before resuming.", ) - if self._time_spent_no_jobs_available > 60 * 5: - logger.warning( - "Your worker spent more than 5 minutes without jobs. This may be due to low demand. " - "However, offering more models or increasing your max_power may help increase the number of jobs " - "you receive.", - ) + minutes_allowed_without_jobs = self.bridge_data.minutes_allowed_without_jobs + seconds_allowed_without_jobs = minutes_allowed_without_jobs * 60 + cur_session_minutes = (cur_time - self.session_start_time) / 60 + if self._time_spent_no_jobs_available > seconds_allowed_without_jobs: + if not self.bridge_data.suppress_speed_warnings: + logger.warning( + f"Your worker spent more than {minutes_allowed_without_jobs} minutes combined throughout this " + f"session ({cur_session_minutes:.2f} minutes) " + "without jobs. This may be due to low demand. However, offering more models or increasing " + "your max_power may help increase the number of jobs you receive and reduce downtime.", + ) + else: + logger.debug( + "Suppressed warning about time spent without jobs " + f"for {minutes_allowed_without_jobs} minutes", + ) if self._shutting_down: logger.warning("Shutting down after current jobs are finished...") - self._last_status_message_time = time.time() + self._last_status_message_time = cur_time _bridge_data_loop_interval = 1.0 _last_bridge_data_reload_time = 0.0 diff --git a/pyproject.toml b/pyproject.toml index 0015f162..13e53cb4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "horde_worker_regen" -version = "9.0.6" +version = "9.0.7" description = "Allows you to connect to the AI Horde and generate images for users." authors = [ {name = "tazlin", email = "tazlin.on.github@gmail.com"},