Merge pull request #173 from Haidra-Org/remix-sdk-changes

feat: cascade remix support; refactor: better source image handling

.gitignore

@@ -174,3 +174,6 @@ cython_debug/
 bin/*
 conda/*
 models/*
+clip_blip/*
+hf_transformers/*
+horde_model_reference/*

.pre-commit-config.yaml

@@ -40,7 +40,7 @@ repos:
         - horde_safety
         - torch
         - ruamel.yaml
-        - hordelib==2.7.6
-        - horde_sdk==0.8.3
+        - hordelib==2.8.1
+        - horde_sdk==0.9.2
         - horde_model_reference==0.6.3
         - semver

environment.yaml

@@ -5,7 +5,7 @@ channels:
   - defaults
 # These should only contain the minimal essentials to get the binaries going, everything else is managed in requirements.txt to keep it universal.
 dependencies:
-  - cudatoolkit==11.8.0
+  - nvidia::cuda-toolkit
   - git
   - pip
-  - python==3.10
+  - python==3.10.12

horde-bridge.cmd

@@ -4,7 +4,7 @@ cd /d %~dp0
 : This first call to runtime activates the environment for the rest of the script
 call runtime python -s -m pip -V
 
-call python -s -m pip install horde_sdk~=0.8.3 horde_model_reference~=0.6.3 hordelib~=2.7.6 -U
+call python -s -m pip install horde_sdk~=0.9.2 horde_model_reference~=0.6.3 hordelib~=2.8.1 -U
 if %ERRORLEVEL% NEQ 0 (
     echo "Please run update-runtime.cmd."
     GOTO END

horde_worker_regen/__init__.py

@@ -8,4 +8,4 @@
 
 ASSETS_FOLDER_PATH = Path(__file__).parent / "assets"
 
-__version__ = "4.3.9"
+__version__ = "5.0.1"

horde_worker_regen/_version_meta.json

@@ -1,5 +1,5 @@
 {
-    "recommended_version": "4.3.9",
+    "recommended_version": "5.0.1",
     "required_min_version": "4.2.7",
     "required_min_version_update_date": "2024-03-09",
     "required_min_version_info": {

horde_worker_regen/consts.py

@@ -15,3 +15,5 @@
 MAX_LORAS = 5
 
 TOTAL_LORA_DOWNLOAD_TIMEOUT = BASE_LORA_DOWNLOAD_TIMEOUT + (EXTRA_LORA_DOWNLOAD_TIMEOUT * MAX_LORAS)
+
+MAX_SOURCE_IMAGE_RETRIES = 5

horde_worker_regen/process_management/horde_process.py

@@ -140,14 +140,17 @@ def send_process_state_change_message(
 
     _heartbeat_limit_interval_seconds: float = 1.0
     _last_heartbeat_time: float = 0.0
+    _last_heartbeat_type: HordeHeartbeatType = HordeHeartbeatType.OTHER
 
     def send_heartbeat_message(self, heartbeat_type: HordeHeartbeatType) -> None:
         """Send a heartbeat message to the main process, indicating that the process is still alive.
 
         Note that this will only send a heartbeat message if the last heartbeat was sent more than
-        `_heartbeat_limit_interval_seconds` ago.
+        `_heartbeat_limit_interval_seconds` ago or if the heartbeat type has changed.
         """
-        if (time.time() - self._last_heartbeat_time) < self._heartbeat_limit_interval_seconds:  # FIXME?
+        if (heartbeat_type != self._last_heartbeat_type) and (
+            time.time() - self._last_heartbeat_time
+        ) < self._heartbeat_limit_interval_seconds:
             return
 
         message = HordeProcessHeartbeatMessage(
@@ -158,6 +161,7 @@ def send_heartbeat_message(self, heartbeat_type: HordeHeartbeatType) -> None:
         )
         self.process_message_queue.put(message)
 
+        self._last_heartbeat_type = heartbeat_type
         self._last_heartbeat_time = time.time()
 
     @abstractmethod

horde_worker_regen/process_management/inference_process.py

@@ -431,7 +431,7 @@ def progress_callback(
             except Exception as e:
                 logger.error(f"Failed to release inference semaphore: {type(e).__name__} {e}")
 
-        if progress_report.comfyui_progress is not None and progress_report.comfyui_progress.current_step >= 0:
+        if progress_report.comfyui_progress is not None and progress_report.comfyui_progress.current_step > 0:
             self.send_heartbeat_message(heartbeat_type=HordeHeartbeatType.INFERENCE_STEP)
         else:
             self.send_heartbeat_message(heartbeat_type=HordeHeartbeatType.PIPELINE_STATE_CHANGE)
@@ -451,9 +451,11 @@ def start_inference(self, job_info: ImageGenerateJobPopResponse) -> list[Resulti
         self._is_busy = True
         try:
             logger.info(f"Starting inference for job(s) {job_info.ids}")
+            esi_count = len(job_info.extra_source_images) if job_info.extra_source_images is not None else 0
             logger.debug(
-                f"has source_image: {job_info.source_image is not None} "
-                f"has source_mask: {job_info.source_mask is not None}",
+                f"has source_image: {job_info.source_image is not None}, "
+                f"has source_mask: {job_info.source_mask is not None}, "
+                f"extra_source_images: {esi_count}",
             )
             logger.debug(f"{job_info.payload.model_dump(exclude={'prompt'})}")