Add a view for failed webhooks (#212)

* 📦 Add failed webhooks view

docs/_data/navigation.yml

@@ -55,6 +55,8 @@
       url: "/housekeeping-repo-location"
     - title: "Forks"
       url: "/housekeeping-forks"
+    - title: "Failed Webhooks"
+      url: "/housekeeping-failed-webhooks"
 - title: "Recommendations"
   url: "/recommendations-tokenless-auth"
   subnavigation:

docs/demo-data/failed-webhooks-detailed.tsv

@@ -0,0 +1,5 @@
+hook_id	type	host	message	count
+1	business	10.0.0.1	request timed out	3
+2	integration	10.0.0.2	Couldn't connect to server	2
+3	repository	10.0.0.3	Couldn't connect to server	1
+4	organization	10.0.0.4	Couldn't resolve host name	1

docs/demo-data/failed-webhooks.tsv

@@ -0,0 +1,7 @@
+date	failed webhooks
+2020-10-01	0
+2020-10-02	1
+2020-10-03	5
+2020-10-04	7
+2020-10-05	5
+2020-10-06	4

docs/housekeeping-failed-webhooks.html

@@ -0,0 +1,62 @@
+---
+layout: default
+title: Failed Webhooks
+permalink: /housekeeping-failed-webhooks
+---
+
+<div class="chart-placeholder">
+	<h3>Failed Webhooks</h3>
+	<canvas
+		data-url="{{ site.dataURL }}/failed-webhooks.tsv"
+		data-type="history"
+		data-config='{
+			"views":
+			[
+				{
+					"label": "2 m",
+					"tooltip": "Show the last 2 months",
+					"aggregate": false,
+					"slice": [0, 61],
+					"default": true
+				},
+				{
+					"label": "2 y",
+					"tooltip": "Show the last 2 years",
+					"aggregate":
+					{
+						"period": "week",
+						"method": "sum"
+					},
+					"slice": [0, 106]
+				},
+				{
+					"label": "all",
+					"tooltip": "Show all data",
+					"aggregate":
+					{
+						"period": "week",
+						"method": "sum"
+					}
+				}
+			]
+    }'></canvas>
+	<div class="info-box"></div>
+</div>
+
+<div class="chart-placeholder">
+  <table data-url="{{ site.dataURL }}/failed-webhooks-detailed.tsv" data-type="table"></table>
+
+	<div class="info-box">
+		<p>
+			<ul>
+        <li><code>business</code> a <a href="https://docs.github.com/enterprise-server/admin/user-management/managing-global-webhooks">global webhook</a></li>
+        <li><code>integration</code> a <a href="https://docs.github.com/enterprise-server/developers/apps/getting-started-with-apps">GitHub App</a> webhook URL</li>
+        <li><code>repository</code> an <a href="https://docs.github.com/enterprise-server/rest/reference/repos#webhooks">repository hook</a></li>
+        <li><code>organization</code> an <a href="https://docs.github.com/enterprise-server/rest/reference/orgs#webhooks">organization hook</a></li>
+      </ul>
+		</p>
+	</div>
+</div>
+
+
+

updater/reports/ReportFailedWebhooks.py

@@ -0,0 +1,19 @@
+from .ReportDaily import *
+
+# Report how many webhooks failed
+class ReportFailedWebhooks(ReportDaily):
+	def name(self):
+		return "failed-webhooks"
+
+	def updateDailyData(self):
+		self.detailedHeader, newData = self.parseData(
+			self.executeScript(self.scriptPath("failed-webhooks.sh")))
+		self.header = ["date", "failed webhooks"]
+		self.data.append(
+			[
+				str(self.yesterday()),
+				sum(map(lambda x: int(x[4] if len(x) > 3 else 0), newData)),
+			])
+		self.detailedData = newData[:25]
+		self.truncateData(self.timeRangeTotal())
+		self.sortDataByDate()

updater/scripts/failed-webhooks.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+#
+# List failed webhooks
+#
+
+echo -e "hook_id\ttype\thost\tmessage\tcount"
+
+zcat -f /var/log/hookshot/exceptions.log.1* |
+  jq --slurp '.[] | del(.backtrace) | {hook_id,service_host,message,class,parent}' |
+  jq --slurp -c 'sort_by(.hook_id) | .[] | {id: .hook_id,url: .service_host, data: (.parent|tostring), msg: .message}' |
+  # remove this part as it's not real JSON and breaks the remaining chain
+  sed -e 's/{\\"url[^}]*}, //' |
+  # this can probably be done more elegantly
+  jq -r -c --slurp '.[] | "\(.id)\t\((.data|tostring|fromjson)[0] | sub("-[0-9]+"; ""))\t\(.url)\t\(.msg)"' |
+  sort |
+  uniq -ic |
+  sort -rn |
+  perl -pne 's/([0-9]+)\s(.*)/\2\t\1/'

updater/update-stats.py

@@ -12,6 +12,7 @@
 from reports.ReportAPIRequestsByUser import *
 from reports.ReportContributorsByOrg import *
 from reports.ReportContributorsByRepo import *
+from reports.ReportFailedWebhooks import *
 from reports.ReportForksToOrgs import *
 from reports.ReportGitDownload import *
 from reports.ReportGitProtocol import *
@@ -80,6 +81,7 @@ def main():
 	ReportAPIRequestsByUser(configuration, dataDirectory, metaStats).update()
 	ReportContributorsByOrg(configuration, dataDirectory, metaStats).update()
 	ReportContributorsByRepo(configuration, dataDirectory, metaStats).update()
+	ReportFailedWebhooks(configuration, dataDirectory, metaStats).update()
 	ReportForksToOrgs(configuration, dataDirectory, metaStats).update()
 	ReportGitDownload(configuration, dataDirectory, metaStats).update()
 	ReportGitProtocol(configuration, dataDirectory, metaStats).update()