From 39298cfea0c81c1c6813d33ef2d638d0bca05c41 Mon Sep 17 00:00:00 2001
From: mmd-afegbua <momodu@cloudkite.io>
Date: Thu, 8 Feb 2024 13:56:55 -0600
Subject: [PATCH 1/7] added prometheus rules

---
 .../workflows/prometheusrules-publish.yaml    | 41 +++++++++++
 prometheus-rules/Chart.yaml                   |  6 ++
 prometheus-rules/README.md                    | 73 +++++++++++++++++++
 .../templates/prometheusRule.yaml             | 11 +++
 prometheus-rules/values.yaml                  | 57 +++++++++++++++
 5 files changed, 188 insertions(+)
 create mode 100644 .github/workflows/prometheusrules-publish.yaml
 create mode 100644 prometheus-rules/Chart.yaml
 create mode 100644 prometheus-rules/README.md
 create mode 100644 prometheus-rules/templates/prometheusRule.yaml
 create mode 100644 prometheus-rules/values.yaml

diff --git a/.github/workflows/prometheusrules-publish.yaml b/.github/workflows/prometheusrules-publish.yaml
new file mode 100644
index 0000000..bcd59a7
--- /dev/null
+++ b/.github/workflows/prometheusrules-publish.yaml
@@ -0,0 +1,41 @@
+name: Publish prometheus-rules
+
+env:
+  CHART_NAME: prometheus-rules
+  GAR_LOCATION: us-central1
+  PROJECT_NAME: cloudkite-public
+  PROJECT_ID: 297731695546
+  REPOSITORY: public-helm-charts
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+    paths: [prometheus-rules/**]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@v0
+        with:
+          workload_identity_provider: projects/${{ env.PROJECT_ID }}/locations/global/workloadIdentityPools/github-actions/providers/github-oidc
+          service_account: github-actions@${{ env.PROJECT_NAME }}.iam.gserviceaccount.com
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v0
+      - name: Docker configuration
+        run: gcloud auth configure-docker $GAR_LOCATION-docker.pkg.dev --quiet
+      - name: Update dependencies
+        working-directory: ${{ env.CHART_NAME }}
+        run: helm dependency update
+      - name: Package chart
+        run: helm package $CHART_NAME
+      - name: Push chart
+        run: helm push $CHART_NAME-*.tgz oci://$GAR_LOCATION-docker.pkg.dev/$PROJECT_NAME/$REPOSITORY
\ No newline at end of file
diff --git a/prometheus-rules/Chart.yaml b/prometheus-rules/Chart.yaml
new file mode 100644
index 0000000..2e8cadf
--- /dev/null
+++ b/prometheus-rules/Chart.yaml
@@ -0,0 +1,6 @@
+apiVersion: v2
+name: prometheus-rules
+description: Helm chart for prometheus rules
+type: application
+version: 0.1.0
+appVersion: v0.1.0
diff --git a/prometheus-rules/README.md b/prometheus-rules/README.md
new file mode 100644
index 0000000..cf8656d
--- /dev/null
+++ b/prometheus-rules/README.md
@@ -0,0 +1,73 @@
+# Prometheus Rule Helm Chart
+
+This Helm chart installs Promrtheus Rules, which integrates with either `kube-prometheus-stack` or stand alone prometheus installation.
+
+## Table of Contents
+
+- [Installation](#installation)
+- [Configuration](#configuration)
+- [Usage](#usage)
+
+## Prerequisites
+
+Before you begin, ensure you have the following prerequisites in place:
+
+- **Helm**: Make sure you have Helm installed on your local machine and it's configured to work with your Kubernetes cluster. If not, you can follow the [official Helm installation guide](https://helm.sh/docs/intro/install/).
+
+## Installation
+
+### Local
+To install this Helm chart, follow these steps:
+
+1. Clone the repository to your local machine:
+
+    ```sh
+    git clone https://github.com/cloudkite-io/helm-charts
+    ```
+
+2. Change into the chart directory:
+
+    ```sh
+    cd helm-charts/prometheus-rules
+    ```
+   Modify the `values.yaml` file with your custom values.
+
+3. Install the Helm chart with a release name (e.g., `cloudkitepromrules`):
+
+    ```sh
+    helm install cloudkitepromrules .
+    ```
+
+### Helm Templating
+1. Create a values.yaml file to customise your values.
+
+    ```values.yaml
+    prometheusRule:
+    - name: custom-groups
+    groups:
+        - name: nginx_ingress
+        rules:
+            - alert: NGINXTooMany500s
+            expr: '100 * ( sum( nginx_ingress_controller_requests{status=~"5.+"} ) / sum(nginx_ingress_controller_requests) ) > 5'
+            for: 5m
+            labels:
+                severity: warning
+            annotations:
+                description: Too many 5XXs
+                summary: More than 5% of all requests returned 5XX, this requires your attention
+    ```
+
+Charts.yaml file:
+
+    ```Charts.yaml
+    apiVersion: v2
+    description: Cloudkite prometheus-rules chart
+    name: prometheus-rules
+    version: 0.1.0
+    dependencies:
+    - name: prometheus-rules
+        version: 0.1.0
+        repository: oci://us-central1-docker.pkg.dev/cloudkite-public/public-helm-charts
+    ```
+
+2. Install or upgrade the Helm chart.
\ No newline at end of file
diff --git a/prometheus-rules/templates/prometheusRule.yaml b/prometheus-rules/templates/prometheusRule.yaml
new file mode 100644
index 0000000..f25826a
--- /dev/null
+++ b/prometheus-rules/templates/prometheusRule.yaml
@@ -0,0 +1,11 @@
+{{- range .Values.prometheusRule}}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: {{ .name }}
+spec:
+  groups:
+{{ toYaml .groups | indent 4 }}
+---
+{{- end }}
diff --git a/prometheus-rules/values.yaml b/prometheus-rules/values.yaml
new file mode 100644
index 0000000..3272eb5
--- /dev/null
+++ b/prometheus-rules/values.yaml
@@ -0,0 +1,57 @@
+prometheusRule:
+- name: default-groups
+  groups:
+    - name: nginx_ingress
+      rules:
+        - alert: NGINXTooMany500s
+          expr: '100 * ( sum( nginx_ingress_controller_requests{status=~"5.+"} ) / sum(nginx_ingress_controller_requests) ) > 5'
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            description: Too many 5XXs
+            summary: More than 5% of all requests returned 5XX, this requires your attention
+
+    - name: kubernetes_apps
+      rules:
+        - alert: KubePodCrashLooping
+          annotations:
+            description: Pod stuck in crashbackloop
+          expr: 'rate(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[15m]) * 60 * 5 > 0'
+          for: 1h
+          labels:
+            severity: warning
+
+    - name: kubernetes_jobs
+      rules:
+        - alert: KubernetesJobFailed
+          expr: 'kube_job_status_failed > 5'
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: Kubernetes Job failed
+            description: "Job"
+
+    - name: solr_cloud
+      rules:
+        - alert: SolrCloudLowLiveNodeCount
+          annotations:
+            summary: Solr low live node count
+            description: "Solr has less than two live nodes for replica"
+          expr: 'solr_collections_live_nodes > 2'
+          for: 5m
+          labels:
+            severity: warning
+            instance: solr
+
+    - name: velero
+      rules:
+        - alert: VeleroBackupFailures
+          expr: 'velero_backup_failure_total{schedule!=""} / velero_backup_attempt_total{schedule!=""} > 0.25'
+          for: 30m
+          labels:
+            severity: warning
+          annotations:
+            description: Velero backup failure
+            summary: Velero backup has failed backups.

From 65cea53ec43817a3da44a061a3180f3afc842053 Mon Sep 17 00:00:00 2001
From: mmd-afegbua <momodu@cloudkite.io>
Date: Thu, 8 Feb 2024 15:05:41 -0600
Subject: [PATCH 2/7] added readme.md

---
 prometheus-rules/values.yaml | 32 +++++++++++++++++++++++++-------
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/prometheus-rules/values.yaml b/prometheus-rules/values.yaml
index 3272eb5..4018ace 100644
--- a/prometheus-rules/values.yaml
+++ b/prometheus-rules/values.yaml
@@ -16,7 +16,8 @@ prometheusRule:
       rules:
         - alert: KubePodCrashLooping
           annotations:
-            description: Pod stuck in crashbackloop
+            description: 'Pod {{ "{{" }} $labels.namespace }}/{{ "{{" }} $labels.pod }} ({{ "{{" }} $labels.container
+          }}) is restarting {{ "{{" }} printf "%.2f" $value }} times / 5 minutes.'
           expr: 'rate(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[15m]) * 60 * 5 > 0'
           for: 1h
           labels:
@@ -25,22 +26,38 @@ prometheusRule:
     - name: kubernetes_jobs
       rules:
         - alert: KubernetesJobFailed
-          expr: 'kube_job_status_failed > 5'
+          expr: 'kube_job_status_failed > 2'
           for: 10m
           labels:
             severity: warning
           annotations:
-            summary: Kubernetes Job failed
+            summary: 'Job {{ "{{" }} $labels.namespace }}/{{ "{{" }} $labels.job_name }} failed to complete.'
             description: "Job"
 
+    - name: prometheus
+      rules:
+        - alert: PrometheusTooManyRestarts
+          expr: 'changes(process_start_time_seconds{job=~"prometheus|pushgateway|alertmanager"}[15m]) > 2'
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: 'Prometheus too many restarts (instance {{ "{{" }} $labels.instance }})'
+            description: 'Prometheus has restarted more than twice in the last 15 minutes. It might be crashlooping.
+            VALUE = {{ "{{" }} $value }}
+            LABELS = {{ "{{" }} $labels }}'
+
+
     - name: solr_cloud
       rules:
         - alert: SolrCloudLowLiveNodeCount
           annotations:
             summary: Solr low live node count
-            description: "Solr has less than two live nodes for replica"
-          expr: 'solr_collections_live_nodes > 2'
-          for: 5m
+            description: 'Solr collection {{ "{{" }} $labels.collection }} has less than two live nodes for replica {{ "{{" }} $labels.replica }} on {{ "{{" }} $labels.base_url }}.
+             VALUE = {{ "{{" }} $value }}  
+             LABELS = {{ "{{" }} $labels }}'
+          expr: 'solr_collections_live_nodes < 2'
+          for: 20m
           labels:
             severity: warning
             instance: solr
@@ -53,5 +70,6 @@ prometheusRule:
           labels:
             severity: warning
           annotations:
-            description: Velero backup failure
+            description: 'Velero backup {{ "{{" }} $labels.schedule }} has {{ "{{" }} $value | humanizePercentage }} failed backups.'
             summary: Velero backup has failed backups.
+

From 77e523b9d3e9e0f05485c08e3970f8107d54e7d8 Mon Sep 17 00:00:00 2001
From: mmd-afegbua <momodu@cloudkite.io>
Date: Thu, 8 Feb 2024 15:09:57 -0600
Subject: [PATCH 3/7] adjust thresholds

---
 prometheus-rules/values.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/prometheus-rules/values.yaml b/prometheus-rules/values.yaml
index 4018ace..d3ff907 100644
--- a/prometheus-rules/values.yaml
+++ b/prometheus-rules/values.yaml
@@ -18,8 +18,8 @@ prometheusRule:
           annotations:
             description: 'Pod {{ "{{" }} $labels.namespace }}/{{ "{{" }} $labels.pod }} ({{ "{{" }} $labels.container
           }}) is restarting {{ "{{" }} printf "%.2f" $value }} times / 5 minutes.'
-          expr: 'rate(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[15m]) * 60 * 5 > 0'
-          for: 1h
+          expr: 'rate(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[15m]) * 60 * 5 > 5'
+          for: 30m
           labels:
             severity: warning
 

From e21588879a480739077ab7e8c5f8d2be9a216d52 Mon Sep 17 00:00:00 2001
From: mmd-afegbua <momodu@cloudkite.io>
Date: Thu, 8 Feb 2024 15:24:16 -0600
Subject: [PATCH 4/7] added new lines function

---
 prometheus-rules/values.yaml | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/prometheus-rules/values.yaml b/prometheus-rules/values.yaml
index d3ff907..d554938 100644
--- a/prometheus-rules/values.yaml
+++ b/prometheus-rules/values.yaml
@@ -43,9 +43,7 @@ prometheusRule:
             severity: warning
           annotations:
             summary: 'Prometheus too many restarts (instance {{ "{{" }} $labels.instance }})'
-            description: 'Prometheus has restarted more than twice in the last 15 minutes. It might be crashlooping.
-            VALUE = {{ "{{" }} $value }}
-            LABELS = {{ "{{" }} $labels }}'
+            description: 'Prometheus has restarted more than twice in the last 15 minutes. It might be crashlooping.{{ "\n" }} VALUE = {{ "{{" }} $value }}{{ "\n" }} LABELS = {{ "{{" }} $labels }}'
 
 
     - name: solr_cloud
@@ -53,9 +51,7 @@ prometheusRule:
         - alert: SolrCloudLowLiveNodeCount
           annotations:
             summary: Solr low live node count
-            description: 'Solr collection {{ "{{" }} $labels.collection }} has less than two live nodes for replica {{ "{{" }} $labels.replica }} on {{ "{{" }} $labels.base_url }}.
-             VALUE = {{ "{{" }} $value }}  
-             LABELS = {{ "{{" }} $labels }}'
+            description: 'Solr collection {{ "{{" }} $labels.collection }} has less than two live nodes for replica {{ "{{" }} $labels.replica }} on {{ "{{" }} $labels.base_url }}.{{ "\n" }} VALUE = {{ "{{" }} $value }}{{ "\n" }} LABELS = {{ "{{" }} $labels }}'
           expr: 'solr_collections_live_nodes < 2'
           for: 20m
           labels:

From 0e0391a5944453904eff8e9e7646d464b28030ee Mon Sep 17 00:00:00 2001
From: mmd-afegbua <momodu@cloudkite.io>
Date: Thu, 8 Feb 2024 18:00:00 -0600
Subject: [PATCH 5/7] one rule per group

---
 .../workflows/prometheusrules-publish.yaml    |  2 +-
 .../templates/prometheusRule.yaml             | 11 ---
 .../templates/prometheusrules.k8s.yaml        | 27 +++++++
 .../prometheusrules.nginxingress.yaml         | 18 +++++
 .../templates/prometheusrules.solr.yaml       | 19 +++++
 .../templates/prometheusrules.velero.yaml     | 18 +++++
 prometheus-rules/values.yaml                  | 79 ++-----------------
 7 files changed, 91 insertions(+), 83 deletions(-)
 delete mode 100644 prometheus-rules/templates/prometheusRule.yaml
 create mode 100644 prometheus-rules/templates/prometheusrules.k8s.yaml
 create mode 100644 prometheus-rules/templates/prometheusrules.nginxingress.yaml
 create mode 100644 prometheus-rules/templates/prometheusrules.solr.yaml
 create mode 100644 prometheus-rules/templates/prometheusrules.velero.yaml

diff --git a/.github/workflows/prometheusrules-publish.yaml b/.github/workflows/prometheusrules-publish.yaml
index bcd59a7..d9ccb12 100644
--- a/.github/workflows/prometheusrules-publish.yaml
+++ b/.github/workflows/prometheusrules-publish.yaml
@@ -38,4 +38,4 @@ jobs:
       - name: Package chart
         run: helm package $CHART_NAME
       - name: Push chart
-        run: helm push $CHART_NAME-*.tgz oci://$GAR_LOCATION-docker.pkg.dev/$PROJECT_NAME/$REPOSITORY
\ No newline at end of file
+        run: helm push $CHART_NAME-*.tgz oci://$GAR_LOCATION-docker.pkg.dev/$PROJECT_NAME/$REPOSITORY
diff --git a/prometheus-rules/templates/prometheusRule.yaml b/prometheus-rules/templates/prometheusRule.yaml
deleted file mode 100644
index f25826a..0000000
--- a/prometheus-rules/templates/prometheusRule.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-{{- range .Values.prometheusRule}}
----
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
-  name: {{ .name }}
-spec:
-  groups:
-{{ toYaml .groups | indent 4 }}
----
-{{- end }}
diff --git a/prometheus-rules/templates/prometheusrules.k8s.yaml b/prometheus-rules/templates/prometheusrules.k8s.yaml
new file mode 100644
index 0000000..070c25c
--- /dev/null
+++ b/prometheus-rules/templates/prometheusrules.k8s.yaml
@@ -0,0 +1,27 @@
+{{- if .Values.kubernetes.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: kubernetes-rules
+spec:
+  groups:
+    - name: kubernetes_apps
+      rules:
+        - alert: KubePodCrashLooping
+          annotations:
+            description: 'Pod {{ "{{" }} $labels.namespace }}/{{ "{{" }} $labels.pod }} ({{ "{{" }} $labels.container }}) is restarting {{ "{{" }} printf "%.2f" $value }} times / 5 minutes.'
+          expr: 'rate(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[15m]) * 60 * 5 > 5'
+          for: 30m
+          labels:
+            severity: warning
+    - name: kubernetes_jobs
+      rules:
+        - alert: KubernetesJobFailed
+          expr: 'kube_job_status_failed > 2'
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: 'Job {{ "{{" }} $labels.namespace }}/{{ "{{" }} $labels.job_name }} failed to complete.'
+            description: "Job"
+{{- end }}
diff --git a/prometheus-rules/templates/prometheusrules.nginxingress.yaml b/prometheus-rules/templates/prometheusrules.nginxingress.yaml
new file mode 100644
index 0000000..fb1c53b
--- /dev/null
+++ b/prometheus-rules/templates/prometheusrules.nginxingress.yaml
@@ -0,0 +1,18 @@
+{{- if .Values.nginx_ingress.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: nginx-ingress-rules
+spec:
+  groups:
+    - name: nginx_ingress
+      rules:
+        - alert: NGINXTooMany500s
+          expr: '100 * ( sum( nginx_ingress_controller_requests{status=~"5.+"} ) / sum(nginx_ingress_controller_requests) ) > 5'
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            description: Too many 5XXs
+            summary: More than 5% of all requests returned 5XX, this requires your attention
+{{- end }}
diff --git a/prometheus-rules/templates/prometheusrules.solr.yaml b/prometheus-rules/templates/prometheusrules.solr.yaml
new file mode 100644
index 0000000..b2ad241
--- /dev/null
+++ b/prometheus-rules/templates/prometheusrules.solr.yaml
@@ -0,0 +1,19 @@
+{{- if .Values.solr.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: solr-cloud-rules
+spec:
+  groups:
+    - name: solr_cloud
+      rules:
+        - alert: SolrCloudLowLiveNodeCount
+          annotations:
+            summary: Solr low live node count
+            description: 'Solr collection {{ "{{" }} $labels.collection }} has less than two live nodes for replica {{ "{{" }} $labels.replica }} on {{ "{{" }} $labels.base_url }}.'
+          expr: 'solr_collections_live_nodes < 2'
+          for: 20m
+          labels:
+            severity: warning
+            instance: solr
+{{- end }}
diff --git a/prometheus-rules/templates/prometheusrules.velero.yaml b/prometheus-rules/templates/prometheusrules.velero.yaml
new file mode 100644
index 0000000..a749aef
--- /dev/null
+++ b/prometheus-rules/templates/prometheusrules.velero.yaml
@@ -0,0 +1,18 @@
+{{- if .Values.velero.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: velero-rules
+spec:
+  groups:
+    - name: velero
+      rules:
+        - alert: VeleroBackupFailures
+          expr: 'velero_backup_failure_total{schedule!=""} / velero_backup_attempt_total{schedule!=""} > 0.25'
+          for: 30m
+          labels:
+            severity: warning
+          annotations:
+            description: 'Velero backup {{ "{{" }} $labels.schedule }} has {{ "{{" }} $value | humanizePercentage }} failed backups.'
+            summary: Velero backup has failed backups.
+{{- end }}
diff --git a/prometheus-rules/values.yaml b/prometheus-rules/values.yaml
index d554938..3de421a 100644
--- a/prometheus-rules/values.yaml
+++ b/prometheus-rules/values.yaml
@@ -1,71 +1,8 @@
-prometheusRule:
-- name: default-groups
-  groups:
-    - name: nginx_ingress
-      rules:
-        - alert: NGINXTooMany500s
-          expr: '100 * ( sum( nginx_ingress_controller_requests{status=~"5.+"} ) / sum(nginx_ingress_controller_requests) ) > 5'
-          for: 5m
-          labels:
-            severity: warning
-          annotations:
-            description: Too many 5XXs
-            summary: More than 5% of all requests returned 5XX, this requires your attention
-
-    - name: kubernetes_apps
-      rules:
-        - alert: KubePodCrashLooping
-          annotations:
-            description: 'Pod {{ "{{" }} $labels.namespace }}/{{ "{{" }} $labels.pod }} ({{ "{{" }} $labels.container
-          }}) is restarting {{ "{{" }} printf "%.2f" $value }} times / 5 minutes.'
-          expr: 'rate(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[15m]) * 60 * 5 > 5'
-          for: 30m
-          labels:
-            severity: warning
-
-    - name: kubernetes_jobs
-      rules:
-        - alert: KubernetesJobFailed
-          expr: 'kube_job_status_failed > 2'
-          for: 10m
-          labels:
-            severity: warning
-          annotations:
-            summary: 'Job {{ "{{" }} $labels.namespace }}/{{ "{{" }} $labels.job_name }} failed to complete.'
-            description: "Job"
-
-    - name: prometheus
-      rules:
-        - alert: PrometheusTooManyRestarts
-          expr: 'changes(process_start_time_seconds{job=~"prometheus|pushgateway|alertmanager"}[15m]) > 2'
-          for: 0m
-          labels:
-            severity: warning
-          annotations:
-            summary: 'Prometheus too many restarts (instance {{ "{{" }} $labels.instance }})'
-            description: 'Prometheus has restarted more than twice in the last 15 minutes. It might be crashlooping.{{ "\n" }} VALUE = {{ "{{" }} $value }}{{ "\n" }} LABELS = {{ "{{" }} $labels }}'
-
-
-    - name: solr_cloud
-      rules:
-        - alert: SolrCloudLowLiveNodeCount
-          annotations:
-            summary: Solr low live node count
-            description: 'Solr collection {{ "{{" }} $labels.collection }} has less than two live nodes for replica {{ "{{" }} $labels.replica }} on {{ "{{" }} $labels.base_url }}.{{ "\n" }} VALUE = {{ "{{" }} $value }}{{ "\n" }} LABELS = {{ "{{" }} $labels }}'
-          expr: 'solr_collections_live_nodes < 2'
-          for: 20m
-          labels:
-            severity: warning
-            instance: solr
-
-    - name: velero
-      rules:
-        - alert: VeleroBackupFailures
-          expr: 'velero_backup_failure_total{schedule!=""} / velero_backup_attempt_total{schedule!=""} > 0.25'
-          for: 30m
-          labels:
-            severity: warning
-          annotations:
-            description: 'Velero backup {{ "{{" }} $labels.schedule }} has {{ "{{" }} $value | humanizePercentage }} failed backups.'
-            summary: Velero backup has failed backups.
-
+nginx_ingress:
+  enabled: false
+velero:
+  enabled: true
+solr:
+  enabled: false
+kubernetes:
+  enabled: true

From e8be79899cfc951b2cc46ba0f73c175dbb92a494 Mon Sep 17 00:00:00 2001
From: mmd-afegbua <momodu@cloudkite.io>
Date: Thu, 8 Feb 2024 18:02:13 -0600
Subject: [PATCH 6/7] updated readme.md

---
 prometheus-rules/README.md | 26 +++++++++++---------------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/prometheus-rules/README.md b/prometheus-rules/README.md
index cf8656d..a916e62 100644
--- a/prometheus-rules/README.md
+++ b/prometheus-rules/README.md
@@ -30,7 +30,7 @@ To install this Helm chart, follow these steps:
     ```sh
     cd helm-charts/prometheus-rules
     ```
-   Modify the `values.yaml` file with your custom values.
+   Modify the `values.yaml` to enable the list of rules you want.
 
 3. Install the Helm chart with a release name (e.g., `cloudkitepromrules`):
 
@@ -42,19 +42,15 @@ To install this Helm chart, follow these steps:
 1. Create a values.yaml file to customise your values.
 
     ```values.yaml
-    prometheusRule:
-    - name: custom-groups
-    groups:
-        - name: nginx_ingress
-        rules:
-            - alert: NGINXTooMany500s
-            expr: '100 * ( sum( nginx_ingress_controller_requests{status=~"5.+"} ) / sum(nginx_ingress_controller_requests) ) > 5'
-            for: 5m
-            labels:
-                severity: warning
-            annotations:
-                description: Too many 5XXs
-                summary: More than 5% of all requests returned 5XX, this requires your attention
+    nginx_ingress:
+    enabled: false
+    velero:
+    enabled: true
+    solr:
+    enabled: false
+    kubernetes:
+    enabled: true
+
     ```
 
 Charts.yaml file:
@@ -70,4 +66,4 @@ Charts.yaml file:
         repository: oci://us-central1-docker.pkg.dev/cloudkite-public/public-helm-charts
     ```
 
-2. Install or upgrade the Helm chart.
\ No newline at end of file
+2. Install or upgrade the Helm chart.

From c1cbeb949dabf51f53fd9a8de30d6b4afc382086 Mon Sep 17 00:00:00 2001
From: mmd-afegbua <momodu@cloudkite.io>
Date: Fri, 9 Feb 2024 09:35:53 -0600
Subject: [PATCH 7/7] updated readme.md vales sample

---
 prometheus-rules/README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/prometheus-rules/README.md b/prometheus-rules/README.md
index a916e62..4e01b86 100644
--- a/prometheus-rules/README.md
+++ b/prometheus-rules/README.md
@@ -43,13 +43,13 @@ To install this Helm chart, follow these steps:
 
     ```values.yaml
     nginx_ingress:
-    enabled: false
+      enabled: false
     velero:
-    enabled: true
+      enabled: true
     solr:
-    enabled: false
+      enabled: false
     kubernetes:
-    enabled: true
+      enabled: true
 
     ```