OM153 - Node exporter dashboard (#93)

* OM153 - node exporter dashboard
* OM153 - added node exporter alerts to docker-compose files

added alert for secret-agent too

config/prometheus/aerospike_rules.yml

@@ -19,7 +19,6 @@ groups:
  summary: "Node {{ $labels.instance }} down"
  description: "{{ $labels.instance }} node is down."
 
-
 - name: aerospike_aerospike.rules > NAMESPACE
  rules:
  - alert: NamespaceStopWrites

config/prometheus/aerospike_secret_agent_rules.yml

@@ -0,0 +1,12 @@
+groups:
+- name: secret_agent.rules
+ rules:
+ - alert: AerospikeSecretAgentDown
+ expr: absent(aerospike_sa_connections_active{job="aerospike_secret_agent"}) == 1
+ for: 1m
+ labels:
+ severity: critical
+ annotations:
+ summary: "Aerospike Secret Agent instance {{ $labels.instance }} in cluster {{$labels.cluster_name}} has been down for more than 1m."
+ description: "Aerospike Secret Agent instance {{ $labels.instance }} in cluster {{$labels.cluster_name}} has been down for more than 1m."
+

config/prometheus/prometheus.yml

@@ -21,9 +21,11 @@ rule_files:
  # new severities are info, warn, error, critical in the increasing order of severity
  # these priority are updates in aerospike_rules.yml
  #
- - "/etc/prometheus/deprecated_aerospike_rules.yml"
+ # - "/etc/prometheus/deprecated_aerospike_rules.yml"
  - "/etc/prometheus/aerospike_rules.yml"
  - "/etc/prometheus/aerospike_connector_rules.yml"
+ - "/etc/prometheus/node_exporter_alerts.yml"
+ - "/etc/prometheus/aerospike_secret_agent_rules.yml"
 
 # A scrape configuration containing exactly one endpoint to scrape:
 # Here it's Prometheus itself.
@@ -36,3 +38,11 @@ scrape_configs:
 
  static_configs:
  - targets: ['172.17.0.2:9145', '172.17.0.3:9145', '172.17.0.4:9145']
+
+ - job_name: 'node-exporter'
+
+ # metrics_path defaults to '/metrics'
+ # scheme defaults to 'http'.
+
+ static_configs:
+ - targets: ['172.17.0.2:9100', '172.17.0.3:9100', '172.17.0.4:9100']

config/prometheus/templates/node_exporter_config_data.json

@@ -0,0 +1,44 @@
+{
+ "node_down_critical_duration": "1m",
+ "node_warn_memory_pct": "70",
+ "node_warn_outofmemory_duration": "1m",
+ "node_warn_outOfDisk_duration": "1m",
+ "node_warn_OutOfdisk_pct": "70",
+ "node_warn_OutOfInodes_pct": "70",
+ "node_warn_outOfInodes_duration": "1m",
+ "node_warn_unusual_disklatency_time": "0.1",
+ "node_warn_unusual_diskWrite_latency_duration": "1m",
+ "node_warn_unusual_diskRead_latency_duration": "1m",
+ "node_high_cpuload_threshold_duration": "30s",
+ "node_warn_highCPU_pct": "70",
+ "node_cpu_steal_threshold_duration": "30s",
+ "node_warn_cpu_steal_pct": "3",
+ "node_network_receiveError_duration": "1m",
+ "node_warn_network_err": "3",
+ "node_network_transmitError_duration": "1m",
+ "node_warn_network_interface_saturation": "0.8",
+ "node_network_interface_saturated_duration": "1m",
+ "node_clock_notsync_duration": "2m",
+ "node_warn_clock_duration": "16",
+ "node_warn_swapPages_count": "5",
+ "node_warn_swap_in_duration": "1m",
+ "node_warn_swap_out_duration": "1m",
+
+ "node_critical_memory_pct": "90",
+ "node_critical_outofmemory_duration": "1m",
+ "node_critical_outOfDisk_duration": "1m",
+ "node_critical_OutOfdisk_pct": "90",
+ "node_critical_OutOfInodes_pct": "90",
+ "node_critical_outOfInodes_duration": "1m",
+ "node_critical_unusual_disklatency_time": "0.5",
+ "node_critical_unusual_diskWrite_latency_duration": "1m",
+ "node_critical_unusual_diskRead_latency_duration": "1m",
+ "node_critical_highCPU_pct": "90",
+ "node_critical_cpu_steal_pct": "5",
+ "node_critical_network_err": "5",
+ "node_critical_network_interface_saturation": "0.9",
+ "node_critical_swapPages_count": "10",
+ "node_critical_swap_in_duration": "1m",
+ "node_critical_swap_out_duration": "1m"
+
+}

examples/docker/docker-compose.yml

@@ -53,6 +53,12 @@ services:
  - source: ./config/prometheus/aerospike_connector_rules.yml
  target: /etc/prometheus/aerospike_connector_rules.yml
  type: bind
+ - source: ./config/prometheus/node_exporter_alerts.yml
+ target: /etc/prometheus/node_exporter_alerts.yml
+ type: bind
+ - source: ./config/prometheus/aerospike_secret_agent_rules.yml
+ target: /etc/prometheus/aerospike_secret_agent_rules.yml
+ type: bind
  depends_on:
  - exporter1
  command:

examples/docker/easy-prometheus.yml

@@ -18,6 +18,8 @@ rule_files:
  - "/etc/prometheus/aerospike_rules.yml"
  - "/etc/prometheus/deprecated_aerospike_rules.yml"
  - "/etc/prometheus/aerospike_connector_rules.yml"
+ - "/etc/prometheus/node_exporter_alerts.yml"
+ - "/etc/prometheus/aerospike_secret_agent_rules.yml"
 
 # A scrape configuration containing exactly one endpoint to scrape:
 # Here it's Prometheus itself.