WIP: Make deployment scripts handle multi-environment deployment

.env.local

@@ -0,0 +1,7 @@
+ENVIRON=local
+# Database credentials for the local development DB
+TILEGARDEN_DB_NAME=postgres
+TILEGARDEN_DB_HOST=database.internal.tilegarden
+TILEGARDEN_DB_PORT=5432
+TILEGARDEN_DB_USER=postgres
+TILEGARDEN_DB_PASSWORD=postgres

.env.template

@@ -1,17 +1,20 @@
+# Name of this deployment environment (e.g. 'dev', 'staging', 'production')
+ENVIRON=dev
+
 # Import values from local environment
 # If you want to set them specifically, change them to assignments
 USER
 AWS_PROFILE
-# $USER will be appended to the project name specified here
+# $USER and environment will be appended to the project name specified here
 PROJECT_NAME=
 
 # Database credentials
 # These get set on the Lambda function and templated into your map-config.xml file at runtime.
-#TILEGARDEN_DB_NAME=
-#TILEGARDEN_DB_HOST=
-#TILEGARDEN_DB_PORT=
-#TILEGARDEN_DB_USER=
-#TILEGARDEN_DB_PASSWORD=
+TILEGARDEN_DB_NAME=
+TILEGARDEN_DB_HOST=
+TILEGARDEN_DB_PORT=
+TILEGARDEN_DB_USER=
+TILEGARDEN_DB_PASSWORD=
 
 # Function config information
 ## REQUIRED ##

.gitignore

@@ -55,7 +55,8 @@ typings/
 .yarn-integrity
 
 # dotenv environment variables file
-.env
+.env*
+!.env.local
 
 # next.js build output
 .next

docker-compose.yml

@@ -18,7 +18,23 @@ services:
     ports:
       - "3000:3000"
       - "9229:9229"
-    env_file: .env
+    environment:
+      # Variables to pass through from environment-specific .env files
+      - ENVIRON
+      - USER
+      - AWS_PROFILE
+      - PROJECT_NAME
+      - TILEGARDEN_DB_NAME
+      - TILEGARDEN_DB_HOST
+      - TILEGARDEN_DB_PORT
+      - TILEGARDEN_DB_USER
+      - TILEGARDEN_DB_PASSWORD
+      - LAMBDA_REGION
+      - LAMBDA_ROLE
+      - LAMBDA_TIMEOUT
+      - LAMBDA_MEMORY
+      - LAMBDA_SUBNETS
+      - LAMBDA_SECURITY_GROUPS
     links:
       - database:database.internal.tilegarden
     volumes:
@@ -29,10 +45,12 @@ services:
 
   terraform:
     image: hashicorp/terraform:0.11.14
-    env_file: .env
     environment:
-      - TF_VAR_region=${LAMBDA_REGION}
-      - TF_VAR_source_name=${PROJECT_NAME}-${USER}
+      - AWS_PROFILE
+      # Providing placeholder defaults here stops docker-compose from warning about missing
+      # variables every time you run it.
+      - TF_VAR_region=${LAMBDA_REGION:-us-east-1}
+      - TF_VAR_source_name=${PROJECT_NAME:-tilegarden}-${USER}-${ENVIRON:-unknown}
     working_dir: /home/terraform
     volumes:
       - ./src/terraform:/home/terraform

scripts/deploy

@@ -4,6 +4,7 @@ set -e
 
 DEPLOY_NEW=""
 NO_CACHE=""
+ENVIRON="dev"
 
 function usage() {
     echo -n "Usage: $(basename "${0}") [OPTION]
@@ -17,17 +18,19 @@ Options:
 }
 
 function main() {
-    if [[ $DEPLOY_NEW != "--new" && ! -f ./src/tiler/claudia.json ]]; then
-        echo "No existing deployment found to update! (did you mean to use '--new'?)"
+    if [[ $DEPLOY_NEW != "--new" && ! -f "./src/tiler/claudia/claudia-${ENVIRON}.json" ]]; then
+        echo "No existing '$ENVIRON' deployment found to update! (did you mean to use '--new'?)"
         exit 1
     fi
 
     docker-compose build --pull terraform
 
+    echo "Building tiler for environment '$ENVIRON'"
     docker-compose -f docker-compose.yml \
+                   --env-file ".env.${ENVIRON}" \
                    build --pull ${NO_CACHE:+$NO_CACHE} tiler
 
-    ./scripts/publish ${DEPLOY_NEW:+$DEPLOY_NEW}
+    ./scripts/publish ${DEPLOY_NEW:+$DEPLOY_NEW} "${ENVIRON}"
 }
 
 
@@ -45,12 +48,14 @@ then
                 NO_CACHE='--no-cache'
                 shift
                 ;;
-            -h| --help| *)
+            -h| --help)
                 usage
                 ;;
+            *)
+                ENVIRON=$1
+                shift
         esac
     done
 
-    main
+    main $ENVIRON
 fi
-

scripts/destroy

@@ -3,44 +3,47 @@
 set -e
 
 function usage() {
-    echo -n "Usage: $(basename "${0}")
-Removes all published resources from AWS
+    echo -n "Usage: $(basename "${0}") ENVIRON
+Removes all published resources from AWS for the specified deployment.
 Options:
     --help      Display this help text
 "
 }
 
 function main() {
-    echo "WARNING: you are about to destroy all your deployed resources!"
+    ENVIRON=${1}
+    echo "WARNING: you are about to destroy all your deployed resources for '${ENVIRON}'!"
     printf "That includes:\n - Associated AWS Lambda functions\n - Associated API Gateways\n - Associated IAM roles (if automatically generated)\n - Associated CloudFront distributions\n"
     read -p "If this is what you want, type 'yes' to continue: " -r
     echo
     if [[ $REPLY =~ ^[yY][eE][sS]$ ]]
     then
-
         # Destroy deployed resources
         # If any part of this fails, the parts that didn't fail
         # remain destroyed, so it's best to just carry on and
         # follow through.
-        docker-compose run --no-deps tiler yarn destroy || true
-        docker-compose run -e TF_VAR_source_id=$(<src/tiler/.api-id) terraform destroy -auto-approve
+        docker-compose --env-file ".env.${ENVIRON}" run --rm --no-deps \
+            tiler yarn destroy || true
+        docker-compose --env-file ".env.${ENVIRON}" run --rm \
+            -e TF_VAR_source_id=$(<src/tiler/.api-id-${ENVIRON}) \
+            terraform destroy -auto-approve
 
         # Remove local config files
-        docker-compose run --no-deps tiler rm claudia.json .api-id
-        docker-compose run --entrypoint rm terraform -rf .terraform terraform.tfstate terraform.tfstate.backup
-
+        docker-compose run --rm --no-deps tiler \
+            rm claudia/claudia-${ENVIRON}.json .api-id-${ENVIRON}
+        docker-compose run --rm --entrypoint rm terraform \
+            -rf .terraform "${PROJECT_NAME}-${USER}-${ENVIRON}.tfstate*"
     else
         echo "Aborting destroy"
     fi
 }
 
 if [ "${BASH_SOURCE[0]}" = "${0}" ]
 then
-    if [ "${1:-}" = "--help" ]
-    then
+    if [ "${1:-}" = "--help" ] || [ -z "${1}" ]; then
         usage
     else
-        main
+        main ${1}
     fi
     exit
 fi

scripts/publish

@@ -5,7 +5,7 @@ set -e
 DEPLOY_NEW=""
 
 function usage() {
-    echo -n "Usage: $(basename "${0}") [OPTION]
+    echo -n "Usage: $(basename "${0}") [OPTION] ENVIRON
 Publish project in development to AWS Lambda
 Options:
     --new     Deploy as a new lambda
@@ -14,25 +14,61 @@ Options:
 }
 
 function main() {
+    ENVIRON="${1}"
+
     docker-compose -f docker-compose.yml \
+                   --env-file ".env.${ENVIRON}" \
                    run --rm --no-deps -e NODE_ENV=production tiler yarn deploy${DEPLOY_NEW:+-new}
 
     # Deploy CloudFront proxy
-    docker-compose run terraform init
-    docker-compose run --rm -e TF_VAR_source_id=$(<src/tiler/.api-id) terraform apply -auto-approve && \
+    # First clear any existing terraform initialization, which could be for a different deployment
+    docker-compose --env-file ".env.${ENVIRON}" run --rm --entrypoint rm terraform -rf .terraform/
+    # Then initialize terraform with the target tfstate file (which might not exist)
+    docker-compose --env-file ".env.${ENVIRON}" run --rm terraform init \
+        -backend-config="path=${ENVIRON}.tfstate"
+    # Then plan and apply
+    docker-compose --env-file ".env.${ENVIRON}" run --rm \
+            -e TF_VAR_source_id=$(<src/tiler/.api-id-${ENVIRON}) \
+            terraform plan -out="${ENVIRON}.tfplan"
+
+    # A compromise between 'apply -auto-approve' and requiring confirmation or separate plan
+    # and apply commands: show a countdown before applying to provide a chance to cancel out.
+    countdown=10
+    while [[ ${countdown} -gt 0 ]];
+    do
+        printf "\rApplying in ${countdown}s"
+        sleep 1
+        ((countdown--))
+    done
+
+    docker-compose --env-file ".env.${ENVIRON}" run --rm \
+            terraform apply "${ENVIRON}.tfplan" && \
     echo "Success! Your tiles will be available at the above URL shortly."
 }
 
 if [ "${BASH_SOURCE[0]}" = "${0}" ]
 then
-    case "$1" in
-        -n| --new)
-            DEPLOY_NEW='--new'
-            shift
-            ;;
-        -h| --help)
-            usage
-            exit
-    esac
-    main
+    while [ $# -gt 0 ]; do
+        case "$1" in
+            -n| --new)
+                DEPLOY_NEW='--new'
+                shift
+                ;;
+            -h| --help)
+                usage
+                exit
+                ;;
+            *)
+                ENVIRON=$1
+                shift
+        esac
+    done
+    if [ -z "${ENVIRON}" ]; then
+        echo "Required ENVIRON argument is missing."
+        usage
+        exit 1
+    else
+        echo "Deploying to environment '${ENVIRON}'"
+    fi
+    main $ENVIRON
 fi

scripts/server

@@ -15,18 +15,19 @@ COMMAND can be one of [start, stop, restart, status]. Default is start.
 
 function main() {
     SERVICES='tiler database'
+    DC_COMMAND='docker-compose --env-file .env.local'
     if [ "${1}" = "start" ]
     then
-        docker-compose up ${SERVICES}
+        ${DC_COMMAND} up ${SERVICES}
     elif [ "${1}" = "stop" ]
     then
-        docker-compose stop ${SERVICES}
+        ${DC_COMMAND} stop ${SERVICES}
     elif [ "${1}" = "restart" ]
     then
-        docker-compose restart ${SERVICES}
+        ${DC_COMMAND} restart ${SERVICES}
     elif [ "${1}" = "status" ]
     then
-        docker-compose ps
+        ${DC_COMMAND} ps
     else
         echo "Unknown option ${1}! Must be one of [start, stop, restart, status ]."
         exit 1

src/terraform/cloudfront.tf

@@ -6,6 +6,10 @@ provider "aws" {
   region	= "${var.region}"
 }
 
+terraform {
+  backend "local" { }
+}
+
 output "domain" {
   value = "${aws_cloudfront_distribution.tilegarden_test.domain_name}"
 }

src/tiler/.gitignore

@@ -1,7 +1,8 @@
 # Ignore lambda settings to avoid blocking new users
-claudia.json
-.api-id
+.api-id*
 
 #Ignore coverage data from jest
 /coverage/
 
+# Claudia configs
+claudia/

src/tiler/package.json

@@ -26,11 +26,11 @@
     "build-all-xml": "./scripts/build-all-xml.sh src/config src/config",
     "deploy": "./scripts/deploy",
     "deploy-new": "./scripts/deploy-new",
-    "destroy": "claudia destroy",
+    "destroy": "claudia destroy --config claudia/claudia-${ENVIRON}.json",
     "dev": "nodemon -e js,mss,json,mml,mss --ignore '*.temp.mml' --exec yarn local",
     "lint": "eslint src",
     "local": "node --inspect=0.0.0.0:9229 -- node_modules/claudia-local-api/bin/claudia-local-api --abbrev 300 --api-module src/api | bunyan -o short",
-    "parse-id": "jq -r '.api.id' claudia.json > .api-id",
+    "parse-id": "jq -r '.api.id' claudia/claudia-${ENVIRON}.json > .api-id-${ENVIRON}",
     "test": "eslint src && jest --coverage"
   },
   "devDependencies": {

src/tiler/scripts/deploy

@@ -1,6 +1,13 @@
 #!/bin/bash
 
-yarn claudia update --no-optional-dependencies \
+if [ -z "${ENVIRON}" ]; then
+   echo 'Required $ENVIRON variable not set.'
+   exit 1
+fi
+
+yarn claudia update \
+   --config claudia/claudia-${ENVIRON}.json \
+   --no-optional-dependencies \
    ${LAMBDA_TIMEOUT:+--timeout ${LAMBDA_TIMEOUT}} \
    ${LAMBDA_MEMORY:+--memory ${LAMBDA_MEMORY}} \
    ${LAMBDA_SECURITY_GROUPS:+--security-group-ids ${LAMBDA_SECURITY_GROUPS}} \

src/tiler/scripts/deploy-new

@@ -1,11 +1,21 @@
 #!/bin/bash
 
-yarn claudia create --no-optional-dependencies --api-module src/api \
-    --name ${PROJECT_NAME} --region ${LAMBDA_REGION} \
+if [ -z "${ENVIRON}" ]; then
+   echo 'Required $ENVIRON variable not set.'
+   exit 1
+elif [ -z "${PROJECT_NAME}" ]; then
+   echo 'Required $PROJECT_NAME variable not set.'
+   exit 1
+fi
+
+yarn claudia create \
+    --config claudia/claudia-${ENVIRON}.json \
+    --no-optional-dependencies --api-module src/api \
+    --name "${PROJECT_NAME}-${USER}-${ENVIRON}" --region ${LAMBDA_REGION} \
     ${LAMBDA_ROLE:+--role ${LAMBDA_ROLE}} \
     ${LAMBDA_TIMEOUT:+--timeout ${LAMBDA_TIMEOUT}} \
     ${LAMBDA_MEMORY:+--memory ${LAMBDA_MEMORY}} \
     ${LAMBDA_SECURITY_GROUPS:+--security-group-ids ${LAMBDA_SECURITY_GROUPS}} \
     ${LAMBDA_SUBNETS:+--subnet-ids ${LAMBDA_SUBNETS}} \
     --set-env TILEGARDEN_DB_HOST=${TILEGARDEN_DB_HOST},TILEGARDEN_DB_NAME=${TILEGARDEN_DB_NAME},TILEGARDEN_DB_PASSWORD=${TILEGARDEN_DB_PASSWORD},TILEGARDEN_DB_PORT=${TILEGARDEN_DB_PORT},TILEGARDEN_DB_USER=${TILEGARDEN_DB_USER} \
-&& yarn parse-id
+&& jq -r '.api.id' claudia/claudia-${ENVIRON}.json > .api-id-${ENVIRON}