From d5bbf77506a0a3e3f19a2ac9201b82a0efefd561 Mon Sep 17 00:00:00 2001
From: Tim Nguyen Van <37105868+DaTiMy@users.noreply.github.com>
Date: Sun, 1 Sep 2024 18:20:11 +0200
Subject: [PATCH] ilert Integration (#18447)

---
 src/health/notifications/alarm-notify.sh.in   | 62 ++++++++++++++++++-
 .../notifications/health_alarm_notify.conf    | 21 +++++++
 src/health/notifications/ilert/metadata.yaml  | 55 ++++++++++++++++
 3 files changed, 135 insertions(+), 3 deletions(-)
 create mode 100644 src/health/notifications/ilert/metadata.yaml

diff --git a/src/health/notifications/alarm-notify.sh.in b/src/health/notifications/alarm-notify.sh.in
index c7c44cb116a287..d7baa7345022ea 100755
--- a/src/health/notifications/alarm-notify.sh.in
+++ b/src/health/notifications/alarm-notify.sh.in
@@ -769,6 +769,9 @@ fi
 # check custom
 [ -z "${DEFAULT_RECIPIENT_CUSTOM}" ] && SEND_CUSTOM="NO"
 
+# check ilert
+[ -z "${ILERT_ALERT_SOURCE_URL}" ] && SEND_ILERT="NO"
+
 # -----------------------------------------------------------------------------
 # check the availability of targets
 
@@ -798,7 +801,8 @@ check_supported_targets() {
     [ "${SEND_DYNATRACE}" = "YES" ] ||
     [ "${SEND_OPSGENIE}" = "YES" ] ||
     [ "${SEND_GOTIFY}" = "YES" ] ||
-    [ "${SEND_NTFY}" = "YES" ]; then
+    [ "${SEND_NTFY}" = "YES" ] ||
+    [ "${SEND_ILERT}" = "YES" ]; then
     # if we need curl, check for the curl command
     if [ -z "${curl}" ]; then
       curl="$(command -v curl 2>/dev/null)"
@@ -828,6 +832,7 @@ check_supported_targets() {
       SEND_OPSGENIE="NO"
       SEND_GOTIFY="NO"
       SEND_NTFY="NO"
+      SEND_ILERT="NO"
     fi
   fi
 
@@ -983,7 +988,8 @@ for method in "${SEND_EMAIL}" \
   "${SEND_DYNATRACE}" \
   "${SEND_OPSGENIE}" \
   "${SEND_GOTIFY}" \
-  "${SEND_NTFY}" ; do
+  "${SEND_NTFY}" \
+  "${SEND_ILERT}" ; do
 
   if [ "${method}" == "YES" ]; then
     proceed=1
@@ -2431,6 +2437,50 @@ send_ntfy() {
   return 1
 }
 
+# -----------------------------------------------------------------------------
+# ilert sender
+
+send_ilert() {
+  local payload httpcode
+  [ "${SEND_ILERT}" != "YES" ] && return 1
+
+  if [ -z "${ILERT_ALERT_SOURCE_URL}" ] ; then
+    info "Can't send ilert notification, because ILERT_ALERT_SOURCE_URL is not defined"
+    return 1
+  fi
+
+  payload=$(cat <<EOF
+  {
+    "alert" : "${name}",
+    "alert_url" : "${goto_url}",
+    "alarm_id" : ${alarm_id},
+    "chart" : "${chart}",
+    "date" : "${when}",
+    "duration" : "${duration_txt}",
+    "host" : "${host}",
+    "info" : "${info}",
+    "message" : "${status_message}",
+    "severity": ${status},
+    "total_critical" : "${total_critical}",
+    "total_warnings" : "${total_warnings}",
+    "value" : "${value_string}",
+    "image_url": "${image}",
+    "src" : "${src}"
+  }
+EOF
+)
+
+  httpcode=$(docurl -X POST -H "Content-Type: application/json" -d "${payload}" "${ILERT_ALERT_SOURCE_URL}")
+  if [ "${httpcode}" = "200" ] || [ "${httpcode}" = "202" ]; then
+    info "sent ilert event for ${notification_description}"
+  else
+    error "failed to send ilert event for ${notification_description}, with HTTP response status code ${httpcode}."
+    return 1
+  fi
+
+  return 0
+}
+
 # -----------------------------------------------------------------------------
 # prepare the content of the notification
 
@@ -3603,6 +3653,11 @@ SENT_GOTIFY=$?
 send_ntfy "${DEFAULT_RECIPIENT_NTFY}"
 SENT_NTFY=$?
 
+# -----------------------------------------------------------------------------
+# send messages to ilert
+send_ilert
+SENT_ILERT=$?
+
 # -----------------------------------------------------------------------------
 # let netdata know
 for state in "${SENT_EMAIL}" \
@@ -3632,7 +3687,8 @@ for state in "${SENT_EMAIL}" \
   "${SENT_DYNATRACE}" \
   "${SENT_OPSGENIE}" \
   "${SENT_GOTIFY}" \
-  "${SENT_NTFY}"; do
+  "${SENT_NTFY}" \
+  "${SENT_ILERT}"; do
   if [ "${state}" -eq 0 ]; then
     # we sent something
     exit 0
diff --git a/src/health/notifications/health_alarm_notify.conf b/src/health/notifications/health_alarm_notify.conf
index 9dcec27aeb281b..84d6d6225e2735 100755
--- a/src/health/notifications/health_alarm_notify.conf
+++ b/src/health/notifications/health_alarm_notify.conf
@@ -844,6 +844,15 @@ NTFY_ACCESS_TOKEN=""
 # Multiple recipients can be given like this: "https://SERVER1/TOPIC1 https://SERVER2/TOPIC2 ..."
 DEFAULT_RECIPIENT_NTFY=""
 
+#------------------------------------------------------------------------------
+# ilert global notification options
+SEND_ILERT="YES"
+
+# Api key
+ILERT_ALERT_SOURCE_URL=""
+
+DEFAULT_RECIPIENT_ILERT=""
+
 #------------------------------------------------------------------------------
 # custom notifications
 #
@@ -984,6 +993,8 @@ custom_sender() {
 
 # role_recipients_ntfy[sysadmin]="${DEFAULT_RECIPIENT_NTFY}"
 
+# role_recipients_ilert[sysadmin]="${DEFAULT_RECIPIENT_ILERT}"
+
 # -----------------------------------------------------------------------------
 # DNS related alarms
 
@@ -1041,6 +1052,8 @@ custom_sender() {
 
 # role_recipients_ntfy[domainadmin]="${DEFAULT_RECIPIENT_NTFY}"
 
+# role_recipients_ilert[domainadmin]="${DEFAULT_RECIPIENT_ILERT}"
+
 # -----------------------------------------------------------------------------
 # database servers alarms
 # mysql, redis, memcached, postgres, etc
@@ -1099,6 +1112,8 @@ custom_sender() {
 
 # role_recipients_ntfy[dba]="${DEFAULT_RECIPIENT_NTFY}"
 
+# role_recipients_ilert[dba]="databases ${DEFAULT_RECIPIENT_ILERT}"
+
 # -----------------------------------------------------------------------------
 # web servers alarms
 # apache, nginx, lighttpd, etc
@@ -1157,6 +1172,8 @@ custom_sender() {
 
 # role_recipients_ntfy[webmaster]="${DEFAULT_RECIPIENT_NTFY}"
 
+# role_recipients_ilert[webmaster]="${DEFAULT_RECIPIENT_ILERT}"
+
 # -----------------------------------------------------------------------------
 # proxy servers alarms
 # squid, etc
@@ -1215,6 +1232,8 @@ custom_sender() {
 
 # role_recipients_ntfy[proxyadmin]="${DEFAULT_RECIPIENT_NTFY}"
 
+# role_recipients_ilert[proxyadmin]="${DEFAULT_RECIPIENT_ILERT}"
+
 # -----------------------------------------------------------------------------
 # peripheral devices
 # UPS, photovoltaics, etc
@@ -1270,3 +1289,5 @@ custom_sender() {
 # role_recipients_gotify[sitemgr]="${DEFAULT_RECIPIENT_GOTIFY}"
 
 # role_recipients_ntfy[sitemgr]="${DEFAULT_RECIPIENT_NTFY}"
+
+# role_recipients_ilert[sitemgr]="${DEFAULT_RECIPIENT_ILERT}"
diff --git a/src/health/notifications/ilert/metadata.yaml b/src/health/notifications/ilert/metadata.yaml
new file mode 100644
index 00000000000000..7e2454834a7495
--- /dev/null
+++ b/src/health/notifications/ilert/metadata.yaml
@@ -0,0 +1,55 @@
+# yamllint disable rule:line-length
+---
+- id: "notify-ilert"
+  meta:
+    name: "ilert"
+    link: "https://www.ilert.com/"
+    categories:
+      - notify.agent
+    icon_filename: "ilert.svg"
+  keywords:
+    - ilert
+  overview:
+    notification_description: |
+      ilert is an alerting and incident management tool. It helps teams reduce response times by enhancing monitoring and ticketing tools with reliable alerts, automatic escalations, on-call schedules, and features for incident response, communication, and status updates.
+      Sending notification to ilert via Netdata's Agent alert notification feature includes links, images and resolving of corresponding alerts.
+    notification_limitations: ""
+  setup:
+    prerequisites:
+      list:
+        - title: ""
+          description: |
+            - A Netdata alert source in ilert. You can create a [Netdata alert source](https://docs.ilert.com/inbound-integrations/netdata) in [ilert](https://www.ilert.com/).
+            - Access to the terminal where Netdata Agent is running
+    configuration:
+      file:
+        name: "health_alarm_notify.conf"
+      options:
+        description: "The following options can be defined for this notification"
+        folding:
+          title: "Config Options"
+          enabled: true
+        list:
+          - name: "SEND_ILERT"
+            default_value: "YES"
+            description: "Set `SEND_ILERT` to YES"
+            required: true
+          - name: "ILERT_ALERT_SOURCE_URL"
+            default_value: ""
+            description: "Set `ILERT_ALERT_SOURCE_URL` to your Netdata alert source url in ilert."
+            required: true
+      examples:
+        folding:
+          enabled: true
+          title: ""
+        list:
+          - name: "Basic Configuration"
+            folding:
+              enabled: false
+            description: ""
+            config: |
+              SEND_ILERT="YES"
+              ILERT_ALERT_SOURCE_URL="https://api.ilert.com/api/v1/events/netdata/{API-KEY}"
+  troubleshooting:
+    problems:
+      list: []