set customisable thresholds for zenduty alerts (#75)

* set customisable thresholds for zenduty alerts

* mention defaults

* bump version

* cleanup readme

* fix

README.md

@@ -4,7 +4,7 @@ Observe Pyth on-chain price feeds and run sanity checks on the data.
 
 ## Usage
 
-Container images are available at https://gallery.ecr.aws/pyth-network/observer.
+Container images are available at https://github.com/pyth-network/pyth-observer/pkgs/container/pyth-observer
 
 To run Observer locally, make sure you have a recent version of [Poetry](https://python-poetry.org) installed and run:
 
@@ -38,6 +38,14 @@ Event types are configured via environment variables:
  - `ZENDUTY_INTEGRATION_KEY` - Integration key for Zenduty service API integration
  - `OPEN_ALERTS_FILE` - Path to local file used for persisting open alerts
 
+### Zenduty Alert Thresholds
+- Zenduty alert will fire if a check fails 5 or more times within 5 minutes.
+- The alert will be resolved if the check failed < 4 times within 5 minutes.
+- Checks run approximately once per minute.
+- These thresholds can be overridden per check type in config.yaml
+ - `zenduty_alert_threshold`: number of failures in 5 minutes >= to this value trigger an alert (default: 5)
+ - `zenduty_resolution_threshold`: number of failures in 5 minutes <= this value resolve the alert (default: 3)
+
 ## Finding the Telegram Group Chat ID
 
 To integrate Telegram events with the Observer, you need the Telegram group chat ID. Here's how you can find it:

pyproject.toml

@@ -4,7 +4,7 @@ ignore_missing_imports = true
 
 [tool.poetry]
 name = "pyth-observer"
-version = "0.2.10"
+version = "0.2.11"
 description = "Alerts and stuff"
 authors = []
 readme = "README.md"

pyth_observer/dispatch.py

@@ -84,6 +84,7 @@ async def run(self, states: List[State]):
  alert = self.open_alerts.get(alert_identifier)
  if alert is None:
  self.open_alerts[alert_identifier] = {
+ "type": check.__class__.__name__,
  "window_start": current_time.isoformat(),
  "failures": 1,
  "last_window_failures": None,
@@ -175,21 +176,31 @@ async def process_zenduty_events(self, current_time):
 
  for identifier, info in self.open_alerts.items():
  self.check_zd_alert_status(identifier, current_time)
- # Resolve the alert if raised and failed < 5 times in the last 5m window
+ check_config = self.config["checks"]["global"][info["type"]]
+ alert_threshold = check_config.get("zenduty_alert_threshold", 5)
+ resolution_threshold = check_config.get("zenduty_resolution_threshold", 3)
+ # Resolve the alert if raised and failed < $threshold times in the last 5m window
+ resolved = False
  if (
- info["sent"]
- and info["last_window_failures"] is not None
- and info["last_window_failures"] < 5
+ info["last_window_failures"] is not None
+ and info["last_window_failures"] <= resolution_threshold
  ):
  logger.debug(f"Resolving Zenduty alert {identifier}")
- response = await send_zenduty_alert(
- identifier, identifier, resolved=True
- )
- if response and 200 <= response.status < 300:
+ resolved = True
+ if info["sent"]:
+ response = await send_zenduty_alert(
+ identifier, identifier, resolved=True
+ )
+ if response and 200 <= response.status < 300:
+ to_remove.append(identifier)
+ else:
  to_remove.append(identifier)
- # Raise alert if failed > 5 times within the last 5m window
- # re-alert every 5 minutes
- elif info["failures"] >= 5 and (
+ # Raise alert if failed > $threshold times within the last 5m window
+ # or if already alerted and not yet resolved.
+ # Re-alert every 5 minutes but not more often.
+ elif (
+ info["failures"] >= alert_threshold or (info["sent"] and not resolved)
+ ) and (
  not info.get("last_alert")
  or current_time - datetime.fromisoformat(info["last_alert"])
  > timedelta(minutes=5)

sample.config.yaml

@@ -11,13 +11,16 @@ events:
  # - DatadogEvent
  - LogEvent
  # - TelegramEvent
+ - ZendutyEvent
 checks:
  global:
  # Price feed checks
  PriceFeedOfflineCheck:
  enable: true
  max_slot_distance: 25
  abandoned_slot_distance: 100000
+ zenduty_alert_threshold: 3
+ zenduty_resolution_threshold: 0
  PriceFeedCoinGeckoCheck:
  enable: true
  max_deviation: 5
@@ -44,11 +47,15 @@ checks:
  enable: true
  max_slot_distance: 25
  max_aggregate_distance: 6
+ zenduty_alert_threshold: 5
+ zenduty_resolution_threshold: 2
  PublisherStalledCheck:
  enable: false
  stall_time_limit: 30
  abandoned_time_limit: 600
  max_slot_distance: 25
+ zenduty_alert_threshold: 1
+ zenduty_resolution_threshold: 0
  # Per-symbol config
  Crypto.MNGO/USD:
  PriceFeedOfflineCheck: