[RW-7320][risk=no] Alert on egress event processing failures (#33)

all-of-us · Oct 12, 2021 · b1e5080 · b1e5080
1 parent 8ddb576
commit b1e5080
Showing 1 changed file with 34 additions and 0 deletions.
diff --git a/...oring/modules/alert_policies/assets/alert_policies/task_failure_process_egress_event.json b/...oring/modules/alert_policies/assets/alert_policies/task_failure_process_egress_event.json
@@ -0,0 +1,34 @@
+{
+  "combiner": "OR",
+  "conditions": [
+    {
+      "conditionThreshold": {
+        "aggregations": [
+          {
+            "alignmentPeriod": "60s",
+            "crossSeriesReducer": "REDUCE_SUM",
+            "perSeriesAligner": "ALIGN_RATE"
+          }
+        ],
+        "filter": "metric.type=\"cloudtasks.googleapis.com/queue/task_attempt_count\" resource.type=\"cloud_tasks_queue\" metric.label.\"response_code\"!=\"ok\" resource.label.\"queue_id\"=\"egressEventQueue\"",
+        "comparison": "COMPARISON_GT",
+
+        "duration": "60s",
+        "thresholdValue": 0,
+        "trigger": {
+          "count": 1
+        }
+      },
+      "displayName": "egressEventQueue task failure"
+    }
+  ],
+  "displayName": "Cloud Task failure: egressEventQueue",
+  "documentation": {
+    "content": "Playbook: https://broad.io/aou-playbook",
+    "mimeType": "text/markdown"
+  },
+  "enabled": true,
+  "notificationChannels": [
+    "projects/${project_id}/notificationChannels/${notification_channel_id}"
+  ]
+}