From c6a72b74ed8f27b4093a4c7b41379f4056073f8a Mon Sep 17 00:00:00 2001 From: 24kushang Date: Sun, 11 Aug 2024 18:28:07 +0530 Subject: [PATCH 1/3] Slack alerts implemented for a Docker CPU usage --- alertmanager/alertmanager.yml | 17 +++++++++++++++++ prometheus.yml | 10 ++++++++++ prometheus/alert_rules.yml | 11 +++++++++++ prometheus/prometheus.yml | 9 +++++++++ 4 files changed, 47 insertions(+) create mode 100644 alertmanager/alertmanager.yml create mode 100644 prometheus/alert_rules.yml diff --git a/alertmanager/alertmanager.yml b/alertmanager/alertmanager.yml new file mode 100644 index 0000000000..7f9833e82c --- /dev/null +++ b/alertmanager/alertmanager.yml @@ -0,0 +1,17 @@ +global: + resolve_timeout: 5m + +route: + receiver: 'slack-notifications' + group_wait: 10s + group_interval: 10s + repeat_interval: 1h + +receivers: +- name: 'slack-notifications' + slack_configs: + - api_url: 'https://hooks.slack.com/services/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' + channel: '#alerts' + send_resolved: true + title: '{{ .CommonAnnotations.summary }}' + text: '{{ .CommonAnnotations.description }}' diff --git a/prometheus.yml b/prometheus.yml index ffe0bda1c9..c699d732cf 100644 --- a/prometheus.yml +++ b/prometheus.yml @@ -9,5 +9,15 @@ services: restart: ${RESTART_POLICY:-unless-stopped} volumes: - ./prometheus:/etc/prometheus + networks: + meet.jitsi: + alertmanager: + image: prom/alertmanager + container_name: alertmanager + ports: + - 9093:9093 + restart: unless-stopped + volumes: + - ./alertmanager:/etc/alertmanager networks: meet.jitsi: \ No newline at end of file diff --git a/prometheus/alert_rules.yml b/prometheus/alert_rules.yml new file mode 100644 index 0000000000..e2ae10fab1 --- /dev/null +++ b/prometheus/alert_rules.yml @@ -0,0 +1,11 @@ +groups: +- name: example + rules: + - alert: HighCPUUsage + expr: container_cpu_utilization_ratio > 0.05 + for: 5s + labels: + severity: critical + annotations: + summary: "High CPU usage detected" + description: "CPU usage is above 5% for more than 10 seconds." diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index 368ff2a06c..d258c9cc5a 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -1,5 +1,14 @@ +alerting: + alertmanagers: + - static_configs: + - targets: + - 'alertmanager:9093' + scrape_configs: - job_name: "prometheus" scrape_interval: 5s static_configs: - targets: ["prosody:5280","jvb:8080","jicofo:8888","otel:9464"] + +rule_files: + - "alert_rules.yml" \ No newline at end of file From 3125b0d96c354f7afe56bc01dc227c5e77f6d88f Mon Sep 17 00:00:00 2001 From: 24kushang <102860007+24kushang@users.noreply.github.com> Date: Sun, 11 Aug 2024 18:35:08 +0530 Subject: [PATCH 2/3] Removed slack webhook URL (Security action) --- alertmanager/alertmanager.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alertmanager/alertmanager.yml b/alertmanager/alertmanager.yml index 7f9833e82c..0b345948e0 100644 --- a/alertmanager/alertmanager.yml +++ b/alertmanager/alertmanager.yml @@ -10,7 +10,7 @@ route: receivers: - name: 'slack-notifications' slack_configs: - - api_url: 'https://hooks.slack.com/services/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' + - api_url: URL_Slack_app channel: '#alerts' send_resolved: true title: '{{ .CommonAnnotations.summary }}' From cb7594a138697098943ad00789ba81c28c9dd4ad Mon Sep 17 00:00:00 2001 From: 24kushang Date: Tue, 20 Aug 2024 18:52:51 +0530 Subject: [PATCH 3/3] Made relevant changes in alertmanager configuration --- alert-manager.yml | 11 +++++++++++ alertmanager/README.md | 0 alertmanager/alertmanager.yml | 14 +++++++------- prometheus.yml | 10 ---------- 4 files changed, 18 insertions(+), 17 deletions(-) create mode 100644 alert-manager.yml create mode 100644 alertmanager/README.md diff --git a/alert-manager.yml b/alert-manager.yml new file mode 100644 index 0000000000..e0204b0579 --- /dev/null +++ b/alert-manager.yml @@ -0,0 +1,11 @@ +services: + alertmanager: + image: prom/alertmanager + container_name: alertmanager + ports: + - 9093:9093 + restart: unless-stopped + volumes: + - ./alertmanager:/etc/alertmanager + networks: + meet.jitsi: \ No newline at end of file diff --git a/alertmanager/README.md b/alertmanager/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/alertmanager/alertmanager.yml b/alertmanager/alertmanager.yml index 0b345948e0..ff88e3a552 100644 --- a/alertmanager/alertmanager.yml +++ b/alertmanager/alertmanager.yml @@ -1,11 +1,11 @@ -global: - resolve_timeout: 5m +# global: +# resolve_timeout: 5m -route: - receiver: 'slack-notifications' - group_wait: 10s - group_interval: 10s - repeat_interval: 1h +# route: +# receiver: 'slack-notifications' +# group_wait: 10s +# group_interval: 10s +# repeat_interval: 1h receivers: - name: 'slack-notifications' diff --git a/prometheus.yml b/prometheus.yml index c699d732cf..ffe0bda1c9 100644 --- a/prometheus.yml +++ b/prometheus.yml @@ -9,15 +9,5 @@ services: restart: ${RESTART_POLICY:-unless-stopped} volumes: - ./prometheus:/etc/prometheus - networks: - meet.jitsi: - alertmanager: - image: prom/alertmanager - container_name: alertmanager - ports: - - 9093:9093 - restart: unless-stopped - volumes: - - ./alertmanager:/etc/alertmanager networks: meet.jitsi: \ No newline at end of file