Merge pull request #632 from golemfactory/scx1332/timeouts

Added more info about time steps. Trying to prevent timeouts in future.

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+# [0.15.1] - 2023-09-11
+
+- Add info & warning that help with detecting potential timeouts in tasks.
+
 # [0.15.0] - 2023-06-28
 
 - Fix schema IDs

goth/runner/step.py

@@ -10,9 +10,20 @@
 if TYPE_CHECKING:
     from goth.runner.probe import Probe
 
-
 logger = logging.getLogger(__name__)
 
+TIMEOUT_LEFT_WARNING_THRESHOLD = 5.0
+
+
+def _check_timeout_and_warn(step_name: str, step_time: float, timeout: float):
+    if timeout - step_time < TIMEOUT_LEFT_WARNING_THRESHOLD:
+        logger.warning(
+            "Step '%s' was very close to being timed out: %.1f s."
+            " - consider increasing time limit for this step.",
+            step_name,
+            timeout - step_time,
+        )
+
 
 def step(default_timeout: float = 10.0):
     """Wrap a step function to implement timeout and log progress."""
@@ -42,12 +53,17 @@ async def wrapper(self: "Probe", *args, timeout: Optional[float] = None):
             except Exception as exc:
                 step_time = time.time() - start_time
                 logger.error(
-                    "Step '%s' raised %s in %.1f",
+                    "Step '%s' raised %s in %.1f/%.1f s",
                     step_name,
                     exc.__class__.__name__,
                     step_time,
+                    timeout,
                 )
+                _check_timeout_and_warn(step_name, step_time, timeout)
                 raise
+            step_time = time.time() - start_time
+            logger.info("Step '%s' finished: %.1f/%.1f s", step_name, step_time, timeout)
+            _check_timeout_and_warn(step_name, step_time, timeout)
             return result
 
         return wrapper

pyproject.toml

@@ -5,7 +5,7 @@ exclude = '/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|venv|\.svn|_build
 
 [tool.poetry]
 name = "goth"
-version = "0.15.0"
+version = "0.15.1"
 description = "Golem Test Harness - integration testing framework"
 authors = ["Golem Factory <[email protected]>"]
 license = "GPL-3.0"