Skip to content

Commit

Permalink
ilert Integration (netdata#18447)
Browse files Browse the repository at this point in the history
  • Loading branch information
DaTiMy authored Sep 1, 2024
1 parent 8fbf2b5 commit d5bbf77
Show file tree
Hide file tree
Showing 3 changed files with 135 additions and 3 deletions.
62 changes: 59 additions & 3 deletions src/health/notifications/alarm-notify.sh.in
Original file line number Diff line number Diff line change
Expand Up @@ -769,6 +769,9 @@ fi
# check custom
[ -z "${DEFAULT_RECIPIENT_CUSTOM}" ] && SEND_CUSTOM="NO"

# check ilert
[ -z "${ILERT_ALERT_SOURCE_URL}" ] && SEND_ILERT="NO"

# -----------------------------------------------------------------------------
# check the availability of targets

Expand Down Expand Up @@ -798,7 +801,8 @@ check_supported_targets() {
[ "${SEND_DYNATRACE}" = "YES" ] ||
[ "${SEND_OPSGENIE}" = "YES" ] ||
[ "${SEND_GOTIFY}" = "YES" ] ||
[ "${SEND_NTFY}" = "YES" ]; then
[ "${SEND_NTFY}" = "YES" ] ||
[ "${SEND_ILERT}" = "YES" ]; then
# if we need curl, check for the curl command
if [ -z "${curl}" ]; then
curl="$(command -v curl 2>/dev/null)"
Expand Down Expand Up @@ -828,6 +832,7 @@ check_supported_targets() {
SEND_OPSGENIE="NO"
SEND_GOTIFY="NO"
SEND_NTFY="NO"
SEND_ILERT="NO"
fi
fi

Expand Down Expand Up @@ -983,7 +988,8 @@ for method in "${SEND_EMAIL}" \
"${SEND_DYNATRACE}" \
"${SEND_OPSGENIE}" \
"${SEND_GOTIFY}" \
"${SEND_NTFY}" ; do
"${SEND_NTFY}" \
"${SEND_ILERT}" ; do

if [ "${method}" == "YES" ]; then
proceed=1
Expand Down Expand Up @@ -2431,6 +2437,50 @@ send_ntfy() {
return 1
}

# -----------------------------------------------------------------------------
# ilert sender

send_ilert() {
local payload httpcode
[ "${SEND_ILERT}" != "YES" ] && return 1

if [ -z "${ILERT_ALERT_SOURCE_URL}" ] ; then
info "Can't send ilert notification, because ILERT_ALERT_SOURCE_URL is not defined"
return 1
fi

payload=$(cat <<EOF
{
"alert" : "${name}",
"alert_url" : "${goto_url}",
"alarm_id" : ${alarm_id},
"chart" : "${chart}",
"date" : "${when}",
"duration" : "${duration_txt}",
"host" : "${host}",
"info" : "${info}",
"message" : "${status_message}",
"severity": ${status},
"total_critical" : "${total_critical}",
"total_warnings" : "${total_warnings}",
"value" : "${value_string}",
"image_url": "${image}",
"src" : "${src}"
}
EOF
)

httpcode=$(docurl -X POST -H "Content-Type: application/json" -d "${payload}" "${ILERT_ALERT_SOURCE_URL}")
if [ "${httpcode}" = "200" ] || [ "${httpcode}" = "202" ]; then
info "sent ilert event for ${notification_description}"
else
error "failed to send ilert event for ${notification_description}, with HTTP response status code ${httpcode}."
return 1
fi

return 0
}

# -----------------------------------------------------------------------------
# prepare the content of the notification

Expand Down Expand Up @@ -3603,6 +3653,11 @@ SENT_GOTIFY=$?
send_ntfy "${DEFAULT_RECIPIENT_NTFY}"
SENT_NTFY=$?

# -----------------------------------------------------------------------------
# send messages to ilert
send_ilert
SENT_ILERT=$?

# -----------------------------------------------------------------------------
# let netdata know
for state in "${SENT_EMAIL}" \
Expand Down Expand Up @@ -3632,7 +3687,8 @@ for state in "${SENT_EMAIL}" \
"${SENT_DYNATRACE}" \
"${SENT_OPSGENIE}" \
"${SENT_GOTIFY}" \
"${SENT_NTFY}"; do
"${SENT_NTFY}" \
"${SENT_ILERT}"; do
if [ "${state}" -eq 0 ]; then
# we sent something
exit 0
Expand Down
21 changes: 21 additions & 0 deletions src/health/notifications/health_alarm_notify.conf
Original file line number Diff line number Diff line change
Expand Up @@ -844,6 +844,15 @@ NTFY_ACCESS_TOKEN=""
# Multiple recipients can be given like this: "https://SERVER1/TOPIC1 https://SERVER2/TOPIC2 ..."
DEFAULT_RECIPIENT_NTFY=""

#------------------------------------------------------------------------------
# ilert global notification options
SEND_ILERT="YES"

# Api key
ILERT_ALERT_SOURCE_URL=""

DEFAULT_RECIPIENT_ILERT=""

#------------------------------------------------------------------------------
# custom notifications
#
Expand Down Expand Up @@ -984,6 +993,8 @@ custom_sender() {

# role_recipients_ntfy[sysadmin]="${DEFAULT_RECIPIENT_NTFY}"

# role_recipients_ilert[sysadmin]="${DEFAULT_RECIPIENT_ILERT}"

# -----------------------------------------------------------------------------
# DNS related alarms

Expand Down Expand Up @@ -1041,6 +1052,8 @@ custom_sender() {

# role_recipients_ntfy[domainadmin]="${DEFAULT_RECIPIENT_NTFY}"

# role_recipients_ilert[domainadmin]="${DEFAULT_RECIPIENT_ILERT}"

# -----------------------------------------------------------------------------
# database servers alarms
# mysql, redis, memcached, postgres, etc
Expand Down Expand Up @@ -1099,6 +1112,8 @@ custom_sender() {

# role_recipients_ntfy[dba]="${DEFAULT_RECIPIENT_NTFY}"

# role_recipients_ilert[dba]="databases ${DEFAULT_RECIPIENT_ILERT}"

# -----------------------------------------------------------------------------
# web servers alarms
# apache, nginx, lighttpd, etc
Expand Down Expand Up @@ -1157,6 +1172,8 @@ custom_sender() {

# role_recipients_ntfy[webmaster]="${DEFAULT_RECIPIENT_NTFY}"

# role_recipients_ilert[webmaster]="${DEFAULT_RECIPIENT_ILERT}"

# -----------------------------------------------------------------------------
# proxy servers alarms
# squid, etc
Expand Down Expand Up @@ -1215,6 +1232,8 @@ custom_sender() {

# role_recipients_ntfy[proxyadmin]="${DEFAULT_RECIPIENT_NTFY}"

# role_recipients_ilert[proxyadmin]="${DEFAULT_RECIPIENT_ILERT}"

# -----------------------------------------------------------------------------
# peripheral devices
# UPS, photovoltaics, etc
Expand Down Expand Up @@ -1270,3 +1289,5 @@ custom_sender() {
# role_recipients_gotify[sitemgr]="${DEFAULT_RECIPIENT_GOTIFY}"

# role_recipients_ntfy[sitemgr]="${DEFAULT_RECIPIENT_NTFY}"

# role_recipients_ilert[sitemgr]="${DEFAULT_RECIPIENT_ILERT}"
55 changes: 55 additions & 0 deletions src/health/notifications/ilert/metadata.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
# yamllint disable rule:line-length
---
- id: "notify-ilert"
meta:
name: "ilert"
link: "https://www.ilert.com/"
categories:
- notify.agent
icon_filename: "ilert.svg"
keywords:
- ilert
overview:
notification_description: |
ilert is an alerting and incident management tool. It helps teams reduce response times by enhancing monitoring and ticketing tools with reliable alerts, automatic escalations, on-call schedules, and features for incident response, communication, and status updates.
Sending notification to ilert via Netdata's Agent alert notification feature includes links, images and resolving of corresponding alerts.
notification_limitations: ""
setup:
prerequisites:
list:
- title: ""
description: |
- A Netdata alert source in ilert. You can create a [Netdata alert source](https://docs.ilert.com/inbound-integrations/netdata) in [ilert](https://www.ilert.com/).
- Access to the terminal where Netdata Agent is running
configuration:
file:
name: "health_alarm_notify.conf"
options:
description: "The following options can be defined for this notification"
folding:
title: "Config Options"
enabled: true
list:
- name: "SEND_ILERT"
default_value: "YES"
description: "Set `SEND_ILERT` to YES"
required: true
- name: "ILERT_ALERT_SOURCE_URL"
default_value: ""
description: "Set `ILERT_ALERT_SOURCE_URL` to your Netdata alert source url in ilert."
required: true
examples:
folding:
enabled: true
title: ""
list:
- name: "Basic Configuration"
folding:
enabled: false
description: ""
config: |
SEND_ILERT="YES"
ILERT_ALERT_SOURCE_URL="https://api.ilert.com/api/v1/events/netdata/{API-KEY}"
troubleshooting:
problems:
list: []

0 comments on commit d5bbf77

Please sign in to comment.