forked from HHS/TANF-app
-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #3252 from raft-tech/3242-local-alert-manager-new
Local Alertmanager MVP
- Loading branch information
Showing
16 changed files
with
456 additions
and
40 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,71 @@ | ||
| global: | ||
| # The smarthost and SMTP sender used for mail notifications. | ||
| smtp_smarthost: 'smtp.sendgrid.net:587' | ||
| smtp_from: '[email protected]' | ||
| smtp_auth_username: 'apikey' | ||
| smtp_auth_password: '{{ sendgrid_api_key }}' | ||
|
|
||
| # The directory from which notification templates are read. | ||
| templates: | ||
| - '/etc/alertmanager/template/*.tmpl' | ||
|
|
||
| # The root route on which each incoming alert enters. | ||
| route: | ||
| # The labels by which incoming alerts are grouped together. For example, | ||
| # multiple alerts coming in for cluster=A and alertname=LatencyHigh would | ||
| # be batched into a single group. | ||
| group_by: ['alertname', 'env', 'service'] | ||
|
|
||
| # When a new group of alerts is created by an incoming alert, wait at | ||
| # least 'group_wait' to send the initial notification. | ||
| # This way ensures that you get multiple alerts for the same group that start | ||
| # firing shortly after another are batched together on the first | ||
| # notification. | ||
| group_wait: 30s | ||
|
|
||
| # When the first notification was sent, wait 'group_interval' to send a batch | ||
| # of new alerts that started firing for that group. | ||
| group_interval: 5m | ||
|
|
||
| # If an alert has successfully been sent, wait 'repeat_interval' to | ||
| # resend them. | ||
| repeat_interval: 5m | ||
|
|
||
| # A default receiver | ||
| receiver: admin-team-emails | ||
|
|
||
| # All the above attributes are inherited by all child routes and can | ||
| # overwritten on each. | ||
|
|
||
| # The child route trees. | ||
| routes: | ||
| # This routes performs a regular expression match on alert labels to | ||
| # catch alerts that are related to a list of services. | ||
| - matchers: | ||
| - alertname=~"UpTime" | ||
| receiver: dev-team-emails | ||
| group_wait: 30m | ||
|
|
||
| # Inhibition rules allow to mute a set of alerts given that another alert is | ||
| # firing. | ||
| # We use this to mute any warning-level notifications if the same alert is | ||
| # already critical. | ||
| inhibit_rules: | ||
| - source_matchers: [severity="CRITICAL"] | ||
| target_matchers: [severity="WARNING"] | ||
| # Apply inhibition if the alertname is the same. | ||
| # CAUTION: | ||
| # If all label names listed in `equal` are missing | ||
| # from both the source and target alerts, | ||
| # the inhibition rule will apply! | ||
| equal: [alertname, env, service] | ||
|
|
||
|
|
||
| receivers: | ||
| - name: 'admin-team-emails' | ||
| email_configs: | ||
| - to: '{{ admin_team_emails }}' | ||
|
|
||
| - name: 'dev-team-emails' | ||
| email_configs: | ||
| - to: '{{ dev_team_emails }}' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| version: 1 | ||
| applications: | ||
| - name: alertmanager | ||
| memory: 512M | ||
| disk_quota: 1G | ||
| instances: 1 | ||
| command: | | ||
| mkdir /tmp | ||
| buildpacks: | ||
| - https://github.com/cloudfoundry/binary-buildpack |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| groups: | ||
| - name: database.alerts | ||
| rules: | ||
| - alert: LocalDatabaseDown | ||
| expr: last_over_time(pg_up{job="postgres"}[1m]) == 0 | ||
| for: 1m | ||
| labels: | ||
| severity: CRITICAL | ||
| annotations: | ||
| summary: "The {{ $labels.service }} service is down." | ||
| description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 1 minute." | ||
| - name: backend.alerts | ||
| rules: | ||
| - alert: LocalBackendDown | ||
| expr: last_over_time(up{job=~"tdp-backend.*"}[1m]) == 0 | ||
| for: 1m | ||
| labels: | ||
| severity: ERROR | ||
| annotations: | ||
| summary: "The {{ $labels.service }} service is down." | ||
| description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 1 minute." | ||
| - name: plg.alerts | ||
| rules: | ||
| - alert: LocalLokiDown | ||
| expr: last_over_time(up{job="loki"}[1m]) == 0 | ||
| labels: | ||
| severity: ERROR | ||
| annotations: | ||
| summary: "The {{ $labels.service }} service is down." | ||
| description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 1 minute." | ||
| - name: app.alerts | ||
| rules: | ||
| - alert: UpTime | ||
| expr: avg_over_time(up[1m]) < 0.95 | ||
| labels: | ||
| severity: WARNING | ||
| annotations: | ||
| summary: "The {{ $labels.service }} service has a uptime warning." | ||
| description: "The {{ $labels.service }} service in the {{ $labels.env }} environment is not maintaining 95% uptime." |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| groups: | ||
| - name: database.alerts | ||
| rules: | ||
| - alert: DevDatabaseDown | ||
| expr: last_over_time(pg_up{job="postgres-dev"}[1m]) == 0 | ||
| labels: | ||
| severity: ERROR | ||
| annotations: | ||
| summary: "The {{ $labels.service }} service is down." | ||
| description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 1 minute." | ||
| - alert: StagingDatabaseDown | ||
| expr: last_over_time(pg_up{job="postgres-staging"}[1m]) == 0 | ||
| labels: | ||
| severity: ERROR | ||
| annotations: | ||
| summary: "The {{ $labels.service }} service is down." | ||
| description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 1 minute." | ||
| - alert: ProductionDatabaseDown | ||
| expr: last_over_time(pg_up{job="postgres-production"}[1m]) == 0 | ||
| labels: | ||
| severity: CRITICAL | ||
| annotations: | ||
| summary: "The {{ $labels.service }} service is down." | ||
| description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 1 minute." | ||
| - name: backend.alerts | ||
| rules: | ||
| - alert: DevEnvironmentBackendDown | ||
| expr: last_over_time(up{job=~"tdp-backend.*", job!~".*prod", job!~".*staging"}[5m]) == 0 | ||
| labels: | ||
| severity: ERROR | ||
| annotations: | ||
| summary: "The {{ $labels.service }} service is down." | ||
| description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 5 minutes." | ||
| - alert: StagingBackendDown | ||
| expr: last_over_time(up{job=~"tdp-backend-staging""}[1m]) == 0 | ||
| labels: | ||
| severity: ERROR | ||
| annotations: | ||
| summary: "The {{ $labels.service }} service is down." | ||
| description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 1 minute." | ||
| - alert: ProductionBackendDown | ||
| expr: last_over_time(up{job=~"tdp-backend-prod"}[1m]) == 0 | ||
| labels: | ||
| severity: CRITICAL | ||
| annotations: | ||
| summary: "The {{ $labels.service }} service is down." | ||
| description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 1 minute." | ||
| - name: plg.alerts | ||
| rules: | ||
| - alert: LokiDown | ||
| expr: last_over_time(up{job="loki"}[1m]) == 0 | ||
| labels: | ||
| severity: ERROR | ||
| annotations: | ||
| summary: "The {{ $labels.service }} service is down." | ||
| description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 1 minute." | ||
| - alert: GrafanaDown | ||
| expr: last_over_time(up{job="grafana"}[1m]) == 0 | ||
| labels: | ||
| severity: ERROR | ||
| annotations: | ||
| summary: "The {{ $labels.service }} service is down." | ||
| description: "The {{ $labels.service }} service in the {{ $labels.env }} environment has been down for more than 1 minute." | ||
| - name: app.alerts | ||
| rules: | ||
| - alert: UpTime | ||
| expr: avg_over_time(up[1d]) < 0.95 | ||
| for: 30m | ||
| labels: | ||
| severity: WARNING | ||
| annotations: | ||
| summary: "The {{ $labels.service }} service has a uptime warning." | ||
| description: "The {{ $labels.service }} service in the {{ $labels.env }} environment is not maintaining 95% uptime." |
File renamed without changes.
Oops, something went wrong.