test_alerts.yaml

rule_files:
  - prometheus_alert_rules.yaml
evaluation_interval: 5s
tests:
  - interval: 1m
    input_series:
      - series: 'ceph_osd_metadata{job="rook-ceph-mgr", ceph_version="1"}'
        values: '1+0x16'
      - series: 'ceph_osd_metadata{job="rook-ceph-mgr", ceph_version="2"}'
        values: '1+0x16'
    alert_rule_test:
      - alertname: CephOSDVersionMismatch
        eval_time: 11m
        exp_alerts:
         - exp_labels:
             severity: warning
           exp_annotations:
             message: 'There are multiple versions of storage services running.'
             description: 'There are 2 different versions of Ceph OSD components running.'
             storage_type: 'ceph'
             severity_level: 'warning'

  - interval: 1m
    input_series:
      - series: 'ceph_mon_metadata{job="rook-ceph-mgr", ceph_version="1"}'
        values: '1+0x16'
      - series: 'ceph_mon_metadata{job="rook-ceph-mgr", ceph_version="2"}'
        values: '1+0x16'
    alert_rule_test:
      - alertname: CephMonVersionMismatch
        eval_time: 11m
        exp_alerts:
         - exp_labels:
             severity: warning
           exp_annotations:
             message: 'There are multiple versions of storage services running.'
             description: 'There are 2 different versions of Ceph Mon components running.'
             storage_type: 'ceph'
             severity_level: 'warning'

  - interval: 1m
    input_series:
      - series: 'ceph_mon_metadata{ceph_daemon="mon.0", hostname="host1.lab.comp.com", job="rook-ceph-mgr"}'
        values: '1+0x12'
      - series: 'ceph_mon_num_elections{ceph_daemon="mon.0", job="rook-ceph-mgr"}'
        values: '10+10x12'
    alert_rule_test:
      - alertname: CephMonHighNumberOfLeaderChanges
        eval_time: 11m
        exp_alerts:
         - exp_labels:
             severity: warning
             ceph_daemon: mon.0
             hostname: host1.lab.comp.com
             job: rook-ceph-mgr
           exp_annotations:
             message: 'Storage Cluster has seen many leader changes recently.'
             description: 'Ceph Monitor mon.0 on host host1.lab.comp.com has seen 10.00 leader changes per minute recently.'
             storage_type: 'ceph'
             severity_level: 'warning'

  - interval: 1m
    input_series:
      - series: 'ceph_health_status{job="rook-ceph-mgr"}'
        values: '0+0x5 1+0x15'
    alert_rule_test:
      - alertname: CephClusterWarningState
        eval_time: 4m
      - alertname: CephClusterWarningState
        eval_time: 16m
        exp_alerts:
         - exp_labels:
             severity: warning
             job: rook-ceph-mgr
           exp_annotations:
             message: 'Storage cluster is in degraded state'
             description: 'Storage cluster is in warning state for more than 10m.'
             storage_type: 'ceph'
             severity_level: 'warning'
  - interval: 1m
    input_series:
      - series: 'ceph_health_status{job="rook-ceph-mgr"}'
        values: '0+0x5 1+0x15 2+0x15'
    alert_rule_test:
      - alertname: CephClusterWarningState
        eval_time: 4m
      - alertname: CephClusterWarningState
        eval_time: 16m
        exp_alerts:
         - exp_labels:
             severity: warning
             job: rook-ceph-mgr
           exp_annotations:
             message: 'Storage cluster is in degraded state'
             description: 'Storage cluster is in warning state for more than 10m.'
             storage_type: 'ceph'
             severity_level: 'warning'
      - alertname: CephClusterErrorState
        eval_time: 32m
        exp_alerts:
         - exp_labels:
             severity: critical
             job: rook-ceph-mgr
           exp_annotations:
             message: 'Storage cluster is in error state'
             description: 'Storage cluster is in error state for more than 10m.'
             storage_type: 'ceph'
             severity_level: 'error'
  - interval: 1m
    input_series:
      - series: 'ceph_health_status{job="rook-ceph-mgr"}'
        values: '2+0x15 1+0x15 2+0x15'
    alert_rule_test:
      - alertname: CephClusterErrorState
        eval_time: 11m
        exp_alerts:
         - exp_labels:
             severity: critical
             job: rook-ceph-mgr
           exp_annotations:
             message: 'Storage cluster is in error state'
             description: 'Storage cluster is in error state for more than 10m.'
             storage_type: 'ceph'
             severity_level: 'error'
      - alertname: CephClusterWarningState
        eval_time: 26m
        exp_alerts:
         - exp_labels:
             severity: warning
             job: rook-ceph-mgr
           exp_annotations:
             message: 'Storage cluster is in degraded state'
             description: 'Storage cluster is in warning state for more than 10m.'
             storage_type: 'ceph'
             severity_level: 'warning'
      - alertname: CephClusterErrorState
        eval_time: 31m
  - interval: 5s
    input_series:
      - series: 'ceph_osd_stat_bytes_used'
        values: '21474836480+0x5 26622320128+0x10'
      - series: 'ceph_osd_stat_bytes'
        values: '26264513505+0x5 27526451350+0x10'
    alert_rule_test:
      - alertname: CephClusterNearFull
        eval_time: 20s
      - alertname: CephClusterNearFull
        eval_time: 40s
        exp_alerts:
         - exp_labels:
             severity: warning
           exp_annotations:
             message: 'Storage cluster is nearing full. Data deletion or cluster expansion is required.'
             description: 'Storage cluster utilization has crossed 75% and will become read only at 85%. Free up some space or expand the storage cluster.'
             storage_type: 'ceph'
             severity_level: 'warning'
  - interval: 5s
    input_series:
      - series: 'ceph_osd_stat_bytes_used'
        values: '21474836480+0x5 26622320218+0x10 27526451350+0x10'
      - series: 'ceph_osd_stat_bytes'
        values: '26264513505+0x5 27526451350+0x10 27526451350+0x10'
    alert_rule_test:
      - alertname: CephClusterNearFull
        eval_time: 20s
      - alertname: CephClusterNearFull
        eval_time: 40s
        exp_alerts:
         - exp_labels:
             severity: warning
           exp_annotations:
             message: 'Storage cluster is nearing full. Data deletion or cluster expansion is required.'
             description: 'Storage cluster utilization has crossed 75% and will become read only at 85%. Free up some space or expand the storage cluster.'
             storage_type: 'ceph'
             severity_level: 'warning'
      - alertname: CephClusterCriticallyFull
        eval_time: 1m
        exp_alerts:
         - exp_labels:
             severity: critical
           exp_annotations:
             message: 'Storage cluster is critically full and needs immediate data deletion or cluster expansion.'
             description: 'Storage cluster utilization has crossed 80% and will become readonly at 85%. Free up some space or expand the storage cluster immediately.'
             storage_type: 'ceph'
             severity_level: 'error'
  - interval: 5s
    input_series:
      - series: 'ceph_osd_stat_bytes_used'
        values: '27526451000+0x10 26622320218+0x10 21474836480+0x5'
      - series: 'ceph_osd_stat_bytes'
        values: '27526451350+0x10 27526451350+0x10 26264513505+0x5'
    alert_rule_test:
      - alertname: CephClusterCriticallyFull
        eval_time: 1m
        exp_alerts:
         - exp_labels:
             severity: critical
           exp_annotations:
             message: 'Storage cluster is critically full and needs immediate data deletion or cluster expansion.'
             description: 'Storage cluster utilization has crossed 80% and will become readonly at 85%. Free up some space or expand the storage cluster immediately.'
             storage_type: 'ceph'
             severity_level: 'error'
      - alertname: CephClusterNearFull
        eval_time: 1m
        exp_alerts:
         - exp_labels:
             severity: warning
           exp_annotations:
             message: 'Storage cluster is nearing full. Data deletion or cluster expansion is required.'
             description: 'Storage cluster utilization has crossed 75% and will become read only at 85%. Free up some space or expand the storage cluster.'
             storage_type: 'ceph'
             severity_level: 'warning'
      - alertname: CephClusterNearFull
        eval_time: 22m
  - interval: 1m
    input_series:
      - series: 'ceph_mon_quorum_status{job="rook-ceph-mgr",ceph_daemon="mon.0"}'
        values: '1+0x05 0+0x20 0+0x20'
      - series: 'ceph_mon_metadata{job="rook-ceph-mgr",ceph_daemon="mon.0"}'
        values: '1+0x05 1+0x20 1+0x20'
      - series: 'ceph_mon_quorum_status{job="rook-ceph-mgr",ceph_daemon="mon.1"}'
        values: '1+0x05 1+0x20 0+0x20'
      - series: 'ceph_mon_metadata{job="rook-ceph-mgr",ceph_daemon="mon.1"}'
        values: '1+0x05 1+0x20 1+0x20'
      - series: 'ceph_mon_quorum_status{job="rook-ceph-mgr",ceph_daemon="mon.2"}'
        values: '1+0x05 1+0x20 1+0x20'
      - series: 'ceph_mon_metadata{job="rook-ceph-mgr",ceph_daemon="mon.2"}'
        values: '1+0x05 1+0x20 1+0x20'
    alert_rule_test:
      - alertname: CephMonQuorumAtRisk
        eval_time: 4m
      - alertname: CephMonQuorumAtRisk
        eval_time: 21m
        exp_alerts:
         - exp_labels:
             severity: critical
           exp_annotations:
             message: 'Storage quorum at risk'
             description: 'Storage cluster quorum is low. Contact Support.'
             storage_type: 'ceph'
             severity_level: 'error'
      - alertname: CephMonQuorumAtRisk
        eval_time: 36m
        exp_alerts:
         - exp_labels:
             severity: critical
           exp_annotations:
             message: 'Storage quorum at risk'
             description: 'Storage cluster quorum is low. Contact Support.'
             storage_type: 'ceph'
             severity_level: 'error'
  - interval: 1m
    input_series:
      - series: 'ceph_mon_quorum_status{job="rook-ceph-mgr",ceph_daemon="mon.0"}'
        values: '0+0x20 1+0x10'
      - series: 'ceph_mon_metadata{job="rook-ceph-mgr",ceph_daemon="mon.0"}'
        values: '1+0x20 1+0x10'
      - series: 'ceph_mon_quorum_status{job="rook-ceph-mgr",ceph_daemon="mon.1"}'
        values: '0+0x20 1+0x10'
      - series: 'ceph_mon_metadata{job="rook-ceph-mgr",ceph_daemon="mon.1"}'
        values: '1+0x20 1+0x10'
      - series: 'ceph_mon_quorum_status{job="rook-ceph-mgr",ceph_daemon="mon.2"}'
        values: '1+0x20 1+0x10'
      - series: 'ceph_mon_metadata{job="rook-ceph-mgr",ceph_daemon="mon.2"}'
        values: '1+0x20 1+0x10'
    alert_rule_test:
      - alertname: CephMonQuorumAtRisk
        eval_time: 16m
        exp_alerts:
         - exp_labels:
             severity: critical
           exp_annotations:
             message: 'Storage quorum at risk'
             description: 'Storage cluster quorum is low. Contact Support.'
             storage_type: 'ceph'
             severity_level: 'error'
      - alertname: CephMonQuorumAtRisk
        eval_time: 21m
  - interval: 1m
    input_series:
      - series: 'ceph_pg_undersized'
        values: '0+0x05 1+0x130'
    alert_rule_test:
      - alertname: CephDataRecoveryTakingTooLong
        eval_time: 4m
      - alertname: CephDataRecoveryTakingTooLong
        eval_time: 126m
        exp_alerts:
         - exp_labels:
             severity: warning
           exp_annotations:
             message: 'Data recovery is slow'
             description: 'Data recovery has been active for too long. Contact Support.'
             storage_type: 'ceph'
             severity_level: 'warning'
  - interval: 1m
    input_series:
      - series: 'ceph_pg_undersized'
        values: '1+0x125 0+0x05'
    alert_rule_test:
      - alertname: CephDataRecoveryTakingTooLong
        eval_time: 121m
        exp_alerts:
         - exp_labels:
             severity: warning
           exp_annotations:
             message: 'Data recovery is slow'
             description: 'Data recovery has been active for too long. Contact Support.'
             storage_type: 'ceph'
             severity_level: 'warning'
      - alertname: CephDataRecoveryTakingTooLong
        eval_time: 126m
  - interval: 1m
    input_series:
      - series: 'ceph_pg_inconsistent'
        values: '0+0x05 1+0x65'
    alert_rule_test:
      - alertname: CephPGRepairTakingTooLong
        eval_time: 4m
      - alertname: CephPGRepairTakingTooLong
        eval_time: 66m
        exp_alerts:
         - exp_labels:
             severity: warning
           exp_annotations:
             message: 'Self heal problems detected'
             description: 'Self heal operations taking too long. Contact Support.'
             storage_type: 'ceph'
             severity_level: 'warning'
  - interval: 1m
    input_series:
      - series: 'ceph_pg_inconsistent'
        values: '1+0x65 0+0x05'
    alert_rule_test:
      - alertname: CephPGRepairTakingTooLong
        eval_time: 61m
        exp_alerts:
         - exp_labels:
             severity: warning
           exp_annotations:
             message: 'Self heal problems detected'
             description: 'Self heal operations taking too long. Contact Support.'
             storage_type: 'ceph'
             severity_level: 'warning'
      - alertname: CephPGRepairTakingTooLong
        eval_time: 66m
  - interval: 1m
    input_series:
      - series: 'ceph_osd_in{ceph_daemon="osd.0"}'
        values: '1+0x01 1+0x05'
      - series: 'ceph_osd_up{ceph_daemon="osd.0"}'
        values: '1+0x01 0+0x05'
      - series: 'ceph_disk_occupation{ceph_daemon="osd.0", device="dev1", exported_instance="host1.lab.comp.com"}'
        values: '1+0x01 1+0x05'
    alert_rule_test:
      - alertname: CephOSDDiskNotResponding
        eval_time: 1m
      - alertname: CephOSDDiskNotResponding
        eval_time: 3m
        exp_alerts:
         - exp_labels:
             severity: critical
             ceph_daemon: osd.0
             device: dev1
             disk: 0
             host: host1.lab.comp.com
           exp_annotations:
             message: 'Disk not responding'
             description: 'Disk device dev1 not responding, on host host1.lab.comp.com.'
             storage_type: 'ceph'
             severity_level: 'error'
  - interval: 1m
    input_series:
      - series: 'ceph_osd_in{ceph_daemon="osd.0"}'
        values: '1+0x05 1+0x01'
      - series: 'ceph_osd_up{ceph_daemon="osd.0"}'
        values: '0+0x05 1+0x01'
      - series: 'ceph_disk_occupation{ceph_daemon="osd.0", device="dev1", exported_instance="host1.lab.comp.com"}'
        values: '1+0x05 1+0x01'
    alert_rule_test:
      - alertname: CephOSDDiskNotResponding
        eval_time: 2m
        exp_alerts:
         - exp_labels:
             severity: critical
             ceph_daemon: osd.0
             device: dev1
             disk: 0
             host: host1.lab.comp.com
           exp_annotations:
             message: 'Disk not responding'
             description: 'Disk device dev1 not responding, on host host1.lab.comp.com.'
             storage_type: 'ceph'
             severity_level: 'error'
      - alertname: CephOSDDiskNotResponding
        eval_time: 5m
        exp_alerts:
         - exp_labels:
             severity: critical
             ceph_daemon: osd.0
             device: dev1
             disk: 0
             host: host1.lab.comp.com
           exp_annotations:
             message: 'Disk not responding'
             description: 'Disk device dev1 not responding, on host host1.lab.comp.com.'
             storage_type: 'ceph'
             severity_level: 'error'
      - alertname: CephOSDDiskNotResponding
        eval_time: 6m
  - interval: 1m
    input_series:
      - series: 'ceph_osd_in{ceph_daemon="osd.0"}'
        values: '1+0x01 0+0x05'
      - series: 'ceph_osd_up{ceph_daemon="osd.0"}'
        values: '1+0x01 0+0x05'
      - series: 'ceph_disk_occupation{ceph_daemon="osd.0", device="dev1", exported_instance="host1.lab.comp.com"}'
        values: '1+0x01 1+0x05'
    alert_rule_test:
      - alertname: CephOSDDiskUnavailable
        eval_time: 1m
      - alertname: CephOSDDiskUnavailable
        eval_time: 3m
        exp_alerts:
         - exp_labels:
             severity: critical
             ceph_daemon: osd.0
             device: dev1
             disk: 0
             host: host1.lab.comp.com
           exp_annotations:
             message: 'Disk not accessible'
             description: 'Disk device dev1 not accessible on host host1.lab.comp.com.'
             storage_type: 'ceph'
             severity_level: 'error'
  - interval: 1m
    input_series:
      - series: 'ceph_osd_in{ceph_daemon="osd.0"}'
        values: '0+0x05 1+0x01'
      - series: 'ceph_osd_up{ceph_daemon="osd.0"}'
        values: '0+0x05 1+0x01'
      - series: 'ceph_disk_occupation{ceph_daemon="osd.0", device="dev1", exported_instance="host1.lab.comp.com"}'
        values: '1+0x05 1+0x01'
    alert_rule_test:
      - alertname: CephOSDDiskUnavailable
        eval_time: 2m
        exp_alerts:
         - exp_labels:
             severity: critical
             ceph_daemon: osd.0
             device: dev1
             disk: 0
             host: host1.lab.comp.com
           exp_annotations:
             message: 'Disk not accessible'
             description: 'Disk device dev1 not accessible on host host1.lab.comp.com.'
             storage_type: 'ceph'
             severity_level: 'error'
      - alertname: CephOSDDiskUnavailable
        eval_time: 5m
        exp_alerts:
         - exp_labels:
             severity: critical
             ceph_daemon: osd.0
             device: dev1
             disk: 0
             host: host1.lab.comp.com
           exp_annotations:
             message: 'Disk not accessible'
             description: 'Disk device dev1 not accessible on host host1.lab.comp.com.'
             storage_type: 'ceph'
             severity_level: 'error'
      - alertname: CephOSDDiskUnavailable
        eval_time: 6m
  - interval: 1m
    input_series:
      - series: 'ceph_mds_metadata{ceph_daemon="mds.myfs-a", job="rook-ceph-mgr"}'
        values: '1+0x15'
      - series: 'ceph_mds_metadata{ceph_daemon="mds.myfs-b", job="rook-ceph-mgr"}'
        values: '1+0x05 0+0x10'
    alert_rule_test:
      - alertname: CephMdsMissingReplicas
        eval_time: 4m
      - alertname: CephMdsMissingReplicas
        eval_time: 11m
        exp_alerts:
         - exp_labels:
             severity: warning
           exp_annotations:
             message: 'Insufficient replicas for storage metadata service.'
             description: 'Minimum required replicas for storage metadata service not available. Might affect the working of storage cluster.'
             storage_type: 'ceph'
             severity_level: 'warning'
  - interval: 1m
    input_series:
      - series: 'ceph_mds_metadata{ceph_daemon="mds.myfs-a", job="rook-ceph-mgr"}'
        values: '0+0x10 1+0x05'
      - series: 'ceph_mds_metadata{ceph_daemon="mds.myfs-b", job="rook-ceph-mgr"}'
        values: '1+0x15'
    alert_rule_test:
      - alertname: CephMdsMissingReplicas
        eval_time: 6m
        exp_alerts:
         - exp_labels:
             severity: warning
           exp_annotations:
             message: 'Insufficient replicas for storage metadata service.'
             description: 'Minimum required replicas for storage metadata service not available. Might affect the working of storage cluster.'
             storage_type: 'ceph'
             severity_level: 'warning'
      - alertname: CephMdsMissingReplicas
        eval_time: 11m