Merge pull request #2114 from vladzcloudius/alert-manager-restrict-to…

…o-many-files-scope prometheus.rules.yml: restrict tooManyFiles rule
scylladb · Nov 7, 2023 · a2276c6 · a2276c6
2 parents e040a90 + 4bd2ec8
commit a2276c6
Showing 1 changed file with 9 additions and 9 deletions.
diff --git a/prometheus/prom_rules/prometheus.rules.yml b/prometheus/prom_rules/prometheus.rules.yml
@@ -262,26 +262,26 @@ groups:
       description: 'OOM Kill on {{ $labels.instance }}'
       summary: A process was terminated on Instance {{ $labels.instance }}
   - alert: tooManyFiles
-    expr: (node_filesystem_files - node_filesystem_files_free) / on(instance) group_left count(scylla_reactor_cpu_busy_ms) by (instance)>20000
+    expr: (node_filesystem_files{mountpoint="/var/lib/scylla"} - node_filesystem_files_free{mountpoint="/var/lib/scylla"}) / on(instance) group_left count(scylla_reactor_cpu_busy_ms) by (instance)>20000
     for: 10s
     labels:
       severity: "info"
-      description: 'Over 20k open files per shard {{ $labels.instance }}'
-      summary: There are over 20K open files per shard on Insace {{ $labels.instance }}
+      description: 'Over 20k open files in /var/lib/scylla per shard {{ $labels.instance }}'
+      summary: There are over 20K open files per shard on Instance {{ $labels.instance }}
   - alert: tooManyFiles
-    expr: (node_filesystem_files - node_filesystem_files_free) / on(instance) group_left count(scylla_reactor_cpu_busy_ms) by (instance)>30000
+    expr: (node_filesystem_files{mountpoint="/var/lib/scylla"} - node_filesystem_files_free{mountpoint="/var/lib/scylla"}) / on(instance) group_left count(scylla_reactor_cpu_busy_ms) by (instance)>30000
     for: 10s
     labels:
       severity: "warn"
-      description: 'Over 30k open files per shard {{ $labels.instance }}'
-      summary: There are over 30K open files per shard on Insace {{ $labels.instance }}
+      description: 'Over 30k open files in /var/lib/scylla per shard {{ $labels.instance }}'
+      summary: There are over 30K open files per shard on Instance {{ $labels.instance }}
   - alert: tooManyFiles
-    expr: (node_filesystem_files - node_filesystem_files_free) / on(instance) group_left count(scylla_reactor_cpu_busy_ms) by (instance)>40000
+    expr: (node_filesystem_files{mountpoint="/var/lib/scylla"} - node_filesystem_files_free{mountpoint="/var/lib/scylla"}) / on(instance) group_left count(scylla_reactor_cpu_busy_ms) by (instance)>40000
     for: 10s
     labels:
       severity: "error"
-      description: 'Over 40k open files per shard {{ $labels.instance }}'
-      summary: There are over 40K open files per shard on Insace {{ $labels.instance }}
+      description: 'Over 40k open files in /var/lib/scylla per shard {{ $labels.instance }}'
+      summary: There are over 40K open files per shard on Instance {{ $labels.instance }}
   - alert: nodeInJoinMode
     expr: scylla_node_operation_mode == 2
     for: 5h