WIP: routing for shimmy

nomad/alertmanager.hcl

@@ -91,13 +91,12 @@ route:
   group_wait: 10s
   group_interval: 10s
   repeat_interval: 1h
-  receiver: slack_alerts
+  receiver: email_alerts
 
   routes:
     - matchers:
-      - service = "skip"
       - severity =~ "severe|warn|smoke"
-      receiver: 'notification_hook'
+      receiver: 'email_alerts'
       continue: true
     - matchers:
       - severity =~ "severe|warn"
@@ -114,13 +113,13 @@ route:
       continue: true%{ endif }
 
 # suppress warn/smoke alerts if a severe alert is already firing with the same alertname
-inhibit_rules:
-  - source_matchers: [severity="severe"]
-    target_matchers: [severity=~"warn|smoke"]
-    equal: [alertname, service]
+#inhibit_rules:
+#  - source_matchers: [severity="severe"]
+#    target_matchers: [severity=~"warn|smoke"]
+#    equal: [alertname, service]
 
 receivers:
-- name: notification_hook
+- name: email_alerts
   webhook_configs:
     - send_resolved: true
       url: '${var.notification_webhook_url}'

nomad/shimmy.hcl

@@ -0,0 +1,72 @@
+variable "dc" {
+  type = string
+}
+
+variable "shimmy_hostname" {
+  type = string
+}
+
+job "[JOB_NAME]" {
+  datacenters = ["${var.dc}"]
+  type        = "service"
+  priority    = 75
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  constraint {
+    attribute = "${meta.pool_type}"
+    value     = "general"
+  }
+
+  group "shimmy" {
+    count = 1
+
+    network {
+      port "http" {
+        to = 8000
+      }
+    }
+
+    service {
+      name = "shimmy"
+      tags = ["int-urlprefix-${var.shimmy_hostname}/"]
+      port = "http"
+      check {
+        name     = "alive"
+        type     = "http"
+        path     = "/health"
+        port     = "http"
+        interval = "10s"
+        timeout  = "2s"
+      }
+    }
+
+    task "shimmy" {
+      driver = "docker"
+
+      template {
+        data = <<EOF
+#!/bin/sh
+apk add --no-cache py3-pip
+pip install --break-system-packages "fastapi[standard]"
+
+tail -f /dev/null
+EOF
+        destination = "local/custom_init.sh"
+        perms = "755"
+      }
+
+      config {
+        image = "python:alpine"
+        ports = ["http"]
+        entrypoint = ["/bin/custom_init.sh"]
+        volumes = [
+          "local/custom_init.sh:/bin/custom_init.sh"
+        ]
+      }
+    }
+  }
+}

scripts/deploy-nomad-alertmanager.sh

@@ -33,7 +33,7 @@ if [ -z "$NOMAD_ADDR" ]; then
 fi
 
 if [ -z "$NOTIFICATION_WEBHOOK_URL" ]; then
-    export NOTIFICATION_WEBHOOK_URL="https://$ENVIRONMENT-$LOCAL_REGION-shimmy.$TOP_LEVEL_DNS_ZONE_NAME"
+    export NOTIFICATION_WEBHOOK_URL="https://$ENVIRONMENT-$LOCAL_REGION-shimmy.$TOP_LEVEL_DNS_ZONE_NAME/alerts"
 fi
 
 export RESOURCE_NAME_ROOT="${ENVIRONMENT}-${ORACLE_REGION}-alertmanager"

scripts/deploy-nomad-shimmy.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+if [ -z "$ENVIRONMENT" ]; then
+    echo "No ENVIRONMENT set, exiting"
+    exit 2
+fi
+
+LOCAL_PATH=$(dirname "${BASH_SOURCE[0]}")
+
+[ -e "$LOCAL_PATH/../sites/$ENVIRONMENT/stack-env.sh" ] && . "$LOCAL_PATH/../sites/$ENVIRONMENT/stack-env.sh"
+[ -e "$LOCAL_PATH/../clouds/all.sh" ] && . "$LOCAL_PATH/../clouds/all.sh"
+[ -e "$LOCAL_PATH/../clouds/oracle.sh" ] && . "$LOCAL_PATH/../clouds/oracle.sh"
+[ -z "$ENVIRONMENT_CONFIGURATION_FILE" ] && ENVIRONMENT_CONFIGURATION_FILE="$LOCAL_PATH/../sites/$ENVIRONMENT/vars.yml"
+
+if [ -z "$ORACLE_REGION" ]; then
+    echo "No ORACLE_REGION set, exiting"
+    exit 2
+fi
+
+[ -z "$LOCAL_REGION" ] && LOCAL_REGION="$OCI_LOCAL_REGION"
+[ -z "$LOCAL_REGION" ] && LOCAL_REGION="us-phoenix-1"
+
+if [ -z "$NOMAD_ADDR" ]; then
+    export NOMAD_ADDR="https://$ENVIRONMENT-$LOCAL_REGION-nomad.$TOP_LEVEL_DNS_ZONE_NAME"
+fi
+
+export RESOURCE_NAME_ROOT="${ENVIRONMENT}-${ORACLE_REGION}-shimmy"
+
+NOMAD_JOB_PATH="$LOCAL_PATH/../nomad"
+NOMAD_DC="$ENVIRONMENT-$ORACLE_REGION"
+export NOMAD_VAR_dc="$NOMAD_DC"
+export NOMAD_VAR_shimmy_hostname="${RESOURCE_NAME_ROOT}.${TOP_LEVEL_DNS_ZONE_NAME}"
+JOB_NAME="shimmy-$ORACLE_REGION"
+
+sed -e "s/\[JOB_NAME\]/$JOB_NAME/" "$NOMAD_JOB_PATH/shimmy.hcl" | nomad job run -var="dc=$NOMAD_DC" -
+RET=$?
+
+export CNAME_VALUE="$RESOURCE_NAME_ROOT"
+export STACK_NAME="${RESOURCE_NAME_ROOT}-cname"
+export UNIQUE_ID="${RESOURCE_NAME_ROOT}"
+export CNAME_TARGET="${ENVIRONMENT}-${ORACLE_REGION}-nomad-pool-general-internal.${DEFAULT_DNS_ZONE_NAME}"
+export CNAME_VALUE="${RESOURCE_NAME_ROOT}"
+$LOCAL_PATH/create-oracle-cname-stack.sh
+
+exit $RET