Merge pull request #52 from lsst-sqre/tickets/DM-48286

[DM-48286] Automate Chronograf and kapacitor backups

Dockerfile

@@ -5,7 +5,7 @@ FROM influxdb:1.11.8-meta
 # Install pipx and use it to install gsutil which is required for the backup script
 # to upload the backup files to Google Cloud Storage
 RUN apt-get update && \
-    apt-get install -y python3 python3-pip pipx && \
+    apt-get install -y python3 python3-pip pipx jq && \
     pipx install gsutil && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
@@ -16,6 +16,11 @@ ENV PATH="/root/.local/bin:$PATH"
 # Verify gsutil installation
 RUN gsutil --version
 
+# Install kubectl
+RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && \
+    chmod +x kubectl && \
+    mv kubectl /usr/local/bin/
+
 # Add the backup script
 COPY backup/backup.sh /usr/local/bin/backup.sh
 RUN chmod +x /usr/local/bin/backup.sh

backup/backup.sh

@@ -2,22 +2,82 @@
 
 set -e
 
-# Configuration
-BACKUP_DIR="/backup/sasquatch-influxdb-enterprise-backup"
-GCS_BUCKET="gs://your-gcs-bucket"
-
-# Ensure the backup directory exists
-mkdir -p "$BACKUP_DIR"
-
-echo "Starting InfluxDB Enterprise backup..."
+backup_chronograf() {
+  echo "Backing up Chronograf..."
+  pod=$(kubectl get pods -n sasquatch -l app=sasquatch-chronograf -o jsonpath='{.items[0].metadata.name}')
+  backup_dir="/backup/chronograf-$(date +%Y-%m-%d)"
+  mkdir -p "$backup_dir"
+  kubectl cp -n sasquatch "$pod:/var/lib/chronograf/chronograf-v1.db" "$backup_dir/chronograf-v1.db" > /dev/null 2>&1
+  if [ $? -eq 0 ] && [ -f "$backup_dir/chronograf-v1.db" ]; then
+    echo "Backup completed successfully at $backup_dir."
+    echo "Cleaning up backups older than $retention_days day(s)..."
+    find /backup -type d -name "chronograf-*" -mtime +$retention_days -exec rm -rf {} \;
+  else
+    echo "Backup failed!" >&2
+    exit 1
+  fi
+}
 
-influxd-ctl -bind sasquatch-influxdb-enterprise-meta.sasquatch:8091 backup -strategy incremental "$BACKUP_DIR" 
+backup_kapacitor() {
+  echo "Backing up Kapacitor..."
+  pod=$(kubectl get pods -n sasquatch -l app=sasquatch-kapacitor -o jsonpath='{.items[0].metadata.name}')
+  backup_dir="/backup/kapacitor-$(date +%Y-%m-%d)"
+  mkdir -p "$backup_dir"
+  kubectl cp -n sasquatch "$pod:/var/lib/kapacitor/kapacitor.db" "$backup_dir/kapacitor.db" > /dev/null 2>&1
+  if [ $? -eq 0 ] && [ -f "$backup_dir/kapacitor.db" ]; then
+    echo "Backup completed successfully at $backup_dir."
+    echo "Cleaning up backups older than $retention_days day(s)..."
+    find /backup -type d -name "kapacitor-*" -mtime +$retention_days -exec rm -rf {} \;
+  else
+    echo "Backup failed!" >&2
+    exit 1
+  fi
+}
 
-if [ $? -eq 0 ]; then
-    echo "Backup completed successfully at $BACKUP_DIR."
-else
+backup_influxdb_enterprise_incremental() {
+  echo "Backing up InfluxDB Enterprise (incremental backup)..."
+  backup_dir="/backup/sasquatch-influxdb-enterprise-backup"
+  backup_logs="/backup/sasquatch-influxdb-enterprise-backup/backup-$(date +%Y-%m-%d).logs"
+  mkdir -p "$backup_dir"
+  influxd-ctl -bind sasquatch-influxdb-enterprise-meta.sasquatch:8091 backup -strategy incremental "$backup_dir" > $backup_logs 2>&1
+  if [ $? -eq 0 ]; then
+    echo "Backup completed successfully at $backup_dir."
+  else
     echo "Backup failed!" >&2
     exit 1
+  fi
+}
+
+if [ -z "$BACKUP_ITEMS" ]; then
+  echo "No backup items specified. Exiting."
+  exit 0
 fi
 
+BACKUP_ITEMS=$(echo "$BACKUP_ITEMS" | jq -c '.[]')
+
+for item in $BACKUP_ITEMS; do
+  name=$(echo "$item" | jq -r '.name')
+  enabled=$(echo "$item" | jq -r '.enabled')
+  retention_days=$(echo "$item" | jq -r '.retention_days')
 
+  if [ "$enabled" == "true" ]; then
+    case "$name" in
+      "chronograf")
+        backup_chronograf
+        ;;
+      "kapacitor")
+        backup_kapacitor
+        ;;
+      "influxdb-enterprise-incremental")
+        backup_influxdb_enterprise_incremental
+        ;;
+      *)
+        echo "Unknown backup item: $name. Skipping..."
+        ;;
+    esac
+  else
+    echo "Skipping $name..."
+  fi
+done
+echo "Backup contents:"
+ls -lhtR /backup