[WIP] Prune stale environments.

services/environment-pruner/.lagoon.yml

@@ -0,0 +1,141 @@
+apiVersion: v1
+kind: Template
+metadata:
+  creationTimestamp: null
+  name: lagoon-openshift-template-cli
+parameters:
+  - name: SERVICE_NAME
+    description: Name of this service
+    required: true
+  - name: SAFE_BRANCH
+    description: Which branch this belongs to, special chars replaced with dashes
+    required: true
+  - name: SAFE_PROJECT
+    description: Which project this belongs to, special chars replaced with dashes
+    required: true
+  - name: BRANCH
+    description: Which branch this belongs to, original value
+    required: true
+  - name: PROJECT
+    description: Which project this belongs to, original value
+    required: true
+  - name: LAGOON_GIT_SHA
+    description: git hash sha of the current deployment
+    required: true
+  - name: SERVICE_ROUTER_URL
+    description: URL of the Router for this service
+    value: ""
+  - name: OPENSHIFT_PROJECT
+    description: Name of the Project that this service is in
+    required: true
+  - name: REGISTRY
+    description: Registry where Images are pushed to
+    required: true
+  - name: DEPLOYMENT_STRATEGY
+    description: Strategy of Deploymentconfig
+    value: "Rolling"
+  - name: SERVICE_IMAGE
+    description: Pullable image of service
+    required: true
+  - name: CRONJOBS
+    description: Oneliner of Cronjobs
+    value: ""
+objects:
+- apiVersion: v1
+  kind: DeploymentConfig
+  metadata:
+    creationTimestamp: null
+    labels:
+      service: ${SERVICE_NAME}
+      branch: ${SAFE_BRANCH}
+      project: ${SAFE_PROJECT}
+    name: ${SERVICE_NAME}
+  spec:
+    replicas: 1
+    selector:
+      service: ${SERVICE_NAME}
+    strategy:
+      type: ${DEPLOYMENT_STRATEGY}
+    template:
+      metadata:
+        creationTimestamp: null
+        labels:
+          service: ${SERVICE_NAME}
+          branch: ${SAFE_BRANCH}
+          project: ${SAFE_PROJECT}
+      spec:
+        volumes:
+          - name: lagoon-sshkey
+            secret:
+              defaultMode: 420
+              secretName: lagoon-sshkey
+        containers:
+        - image: ${SERVICE_IMAGE}
+          name: ${SERVICE_NAME}
+          envFrom:
+          - configMapRef:
+              name: lagoon-env
+          env:
+            ## LAGOON_GIT_SHA is injected directly and not loaded via `lagoon-env` config
+            ## This will cause the cli to redeploy on every deployment, even the files have not changed
+            - name: LAGOON_GIT_SHA
+              value: ${LAGOON_GIT_SHA}
+            - name: SERVICE_NAME
+              value: ${SERVICE_NAME}
+            - name: CRONJOBS
+              value: ${CRONJOBS}
+            - name: JWTSECRET
+              valueFrom:
+                secretKeyRef:
+                  name: jwtsecret
+                  key: JWTSECRET
+            - name: LOGSDB_ADMIN_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  key: LOGSDB_ADMIN_PASSWORD
+                  name: logs-db-admin-password
+          volumeMounts:
+            - mountPath: /var/run/secrets/lagoon/sshkey/
+              name: lagoon-sshkey
+              readOnly: true
+          resources:
+            requests:
+              cpu: 10m
+              memory: 10Mi
+    test: false
+    triggers:
+    - type: ConfigChange
+- apiVersion: batch/v1beta1
+  kind: CronJob
+  metadata:
+    name: cronjob-remote
+  spec:
+    schedule: "*/15 * * * *"
+    suspend: true # we're running cronjobs now via the lagoon cronjob system
+    concurrencyPolicy: Forbid
+    jobTemplate:
+      spec:
+        template:
+          metadata:
+            annotations:
+              alpha.image.policy.openshift.io/resolve-names: "*"
+            labels:
+              cronjob: prune-environments
+              parent: cronjob-remote
+          spec:
+            containers:
+            - name: prune-environments
+              image: ${REGISTRY}/${OPENSHIFT_PROJECT}/${SERVICE_NAME}:latest
+              command:
+                - /lagoon/cronjob.sh
+                - "/prune.sh"
+              envFrom:
+              - configMapRef:
+                  name: lagoon-env
+              env:
+                - name: JWTSECRET
+                  valueFrom:
+                    secretKeyRef:
+                      name: jwtsecret
+                      key: JWTSECRET
+            restartPolicy: OnFailure

services/environment-pruner/Dockerfile

@@ -0,0 +1,20 @@
+ARG IMAGE_REPO
+FROM ${IMAGE_REPO:-lagoon}/oc
+
+ENV LAGOON=environment-pruner
+
+RUN apk add --no-cache tini jq openssl bash curl nodejs nodejs-npm \
+    && npm config set unsafe-perm true \
+    && npm -g install jwtgen
+
+COPY create_jwt.sh prune.sh /
+
+ENV JWTSECRET=super-secret-string \
+    JWTAUDIENCE=api.dev \
+    PROJECT_REGEX=".+"  \
+    ROUTER_LOG_INTERVAL=4h \
+    POD_RUN_INTERVAL=14400 \
+    ELASTICSEARCH_URL="http://logs-db-service:9200"
+
+ENTRYPOINT ["/sbin/tini", "--", "/lagoon/entrypoints.sh"]
+CMD ["/bin/docker-sleep"]

services/environment-pruner/create_jwt.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+PAYLOAD='{
+  "role": "admin",
+  "iss": "auto-idler",
+  "aud": "'$JWTAUDIENCE'",
+  "sub": "auto-idler"
+}'
+
+jwtgen -a HS256 -s "${JWTSECRET}" --claims "${PAYLOAD}" $@

services/environment-pruner/prune.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# set -e -o pipefail
+
+# Create a JWT admin token to talk to the API.
+API_ADMIN_JWT_TOKEN=$(./create_jwt.sh)
+BEARER="Authorization: bearer $API_ADMIN_JWT_TOKEN"
+
+CUTOFF_DATE=$(date +'%Y-%m-%d %H:%M:%S' --date '-90 days')
+
+# Load all the projects.
+GRAPHQL='query environments {
+  environments:allProjects {
+    name
+    productionEnvironment
+    openshift {
+      consoleUrl
+      token
+      name
+    }
+    environments {
+      openshiftProjectName
+      name
+      updated
+    }
+  }
+}'
+
+# Convert GraphQL file into single line (but with still \n existing), turn \n into \\n, esapee the Quotes
+query=$(echo $GRAPHQL | sed 's/"/\\"/g' | sed 's/\\n/\\\\n/g' | awk -F'\n' '{if(NR == 1) {printf $0} else {printf "\\n"$0}}')
+ALL_ENVIRONMENTS=$(curl -s -XPOST -H 'Content-Type: application/json' -H "$BEARER" api:3000/graphql -d "{\"query\": \"$query\"}")
+
+echo "$ALL_ENVIRONMENTS" | jq -c '.data.environments[] | select((.environments|length)>=1)' | while read project
+  do
+    PROJECT_NAME=$(echo "$project" | jq -r '.name')
+    OPENSHIFT_URL=$(echo "$project" | jq -r '.openshift.consoleUrl')
+    PRODUCTION_BRANCH=$(echo "$project" | jq -r '.productionEnvironment')
+
+    echo "$project" | jq -c '.environments[]' | while read environment
+    do
+      ENVIRONMENT_OPENSHIFT_PROJECTNAME=$(echo "$environment" | jq -r '.openshiftProjectName')
+      ENVIRONMENT_NAME=$(echo "$environment" | jq -r '.name')
+      UPDATED_DATE=$(echo "$environment" | jq -r '.updated')
+
+      echo "$OPENSHIFT_URL - $PROJECT_NAME: handling environment $ENVIRONMENT_NAME"
+
+      if [ "$ENVIRONMENT_NAME" = "$PRODUCTION_BRANCH" ]; then
+        # This should only ever remove non-prods.
+        echo "$PROJECT_NAME: $ENVIRONMENT_NAME is the production environment"
+        continue;
+      fi
+
+      Skip if no openshift project exists.
+      if ! oc --insecure-skip-tls-verify --token="$OPENSHIFT_TOKEN" --server="$OPENSHIFT_URL" get project "$ENVIRONMENT_OPENSHIFT_PROJECTNAME" > /dev/null; then
+        echo "$OPENSHIFT_URL - $PROJECT_NAME: $ENVIRONMENT_NAME: OpenShift Project not found"
+        continue;
+      fi
+
+      # The updated date of the environment is newer than the expected cutoff.
+      if [[ $UPDATED_DATE > $CUTOFF_DATE ]]; then
+        continue;
+      fi
+
+      # Use Lagoon to delete the environment.
+      GQL_MUTATION='mutation DelEnv($name !String $env !String) {
+        deleteEnvironment(input: {
+          name: $name
+          project: $env
+          execute: true
+        })
+      }'
+      mutation=$(echo $GQL_MUTATION | sed 's/"/\\"/g' | sed 's/\\n/\\\\n/g' | awk -F'\n' '{if(NR == 1) {printf $0} else {printf "\\n"$0}}')
+      DELETE=$(curl -s -XPOST -H 'Content-Type: application/json' -H "$BEARER" api:3000/graphql -d "{\"query\": \"$mutation\", \"variables\": {\"name\": \"$PROJECT_NAME\", \"env\": \"$ENVIRONMENT_NAME\"} }")
+  done
+done