From ce6b6ec73d8ffa8d78faab686814d086b15a6948 Mon Sep 17 00:00:00 2001 From: Klaas Hoekema Date: Tue, 18 Jun 2019 00:30:41 -0400 Subject: [PATCH 1/4] Set up docker-compose and deploy scripts to use env-specific env files Sets up the docker-compose config and deployment scripts to take advantage of the new (or impending) docker-compose `--env-file` option. Creating a .env.ENVIRON file then passing the name of the environment to `scripts/deploy` should read that environment file into the project environment. All the variables needed inside the tiler container have to be listed in it's `environment` so they get passed through. Adds a checked-in .env.local file, with credentials for the test database. It doesn't need any of the Lambda or project variables, since it's local. --- .env.local | 6 ++++++ .env.template | 10 +++++----- .gitignore | 3 ++- docker-compose.yml | 17 ++++++++++++++++- scripts/deploy | 13 +++++++++---- scripts/publish | 9 ++++++++- scripts/server | 9 +++++---- 7 files changed, 51 insertions(+), 16 deletions(-) create mode 100644 .env.local diff --git a/.env.local b/.env.local new file mode 100644 index 00000000..738965a2 --- /dev/null +++ b/.env.local @@ -0,0 +1,6 @@ +# Database credentials for the local development DB +TILEGARDEN_DB_NAME=postgres +TILEGARDEN_DB_HOST=database.internal.tilegarden +TILEGARDEN_DB_PORT=5432 +TILEGARDEN_DB_USER=postgres +TILEGARDEN_DB_PASSWORD=postgres diff --git a/.env.template b/.env.template index cc0f2e18..ac3729c2 100644 --- a/.env.template +++ b/.env.template @@ -7,11 +7,11 @@ PROJECT_NAME= # Database credentials # These get set on the Lambda function and templated into your map-config.xml file at runtime. -#TILEGARDEN_DB_NAME= -#TILEGARDEN_DB_HOST= -#TILEGARDEN_DB_PORT= -#TILEGARDEN_DB_USER= -#TILEGARDEN_DB_PASSWORD= +TILEGARDEN_DB_NAME= +TILEGARDEN_DB_HOST= +TILEGARDEN_DB_PORT= +TILEGARDEN_DB_USER= +TILEGARDEN_DB_PASSWORD= # Function config information ## REQUIRED ## diff --git a/.gitignore b/.gitignore index 22c3120c..446340af 100644 --- a/.gitignore +++ b/.gitignore @@ -55,7 +55,8 @@ typings/ .yarn-integrity # dotenv environment variables file -.env +.env* +!.env.local # next.js build output .next diff --git a/docker-compose.yml b/docker-compose.yml index 0a7f7e58..c7942a1b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,7 +18,22 @@ services: ports: - "3000:3000" - "9229:9229" - env_file: .env + environment: + # Variables to pass through from environment-specific .env files + - USER + - AWS_PROFILE + - PROJECT_NAME + - TILEGARDEN_DB_NAME + - TILEGARDEN_DB_HOST + - TILEGARDEN_DB_PORT + - TILEGARDEN_DB_USER + - TILEGARDEN_DB_PASSWORD + - LAMBDA_REGION + - LAMBDA_ROLE + - LAMBDA_TIMEOUT + - LAMBDA_MEMORY + - LAMBDA_SUBNETS + - LAMBDA_SECURITY_GROUPS links: - database:database.internal.tilegarden volumes: diff --git a/scripts/deploy b/scripts/deploy index a6f3023c..0977adc9 100755 --- a/scripts/deploy +++ b/scripts/deploy @@ -4,6 +4,7 @@ set -e DEPLOY_NEW="" NO_CACHE="" +ENVIRON="dev" function usage() { echo -n "Usage: $(basename "${0}") [OPTION] @@ -24,10 +25,12 @@ function main() { docker-compose build --pull terraform + echo "Building tiler for environment '$ENVIRON'" docker-compose -f docker-compose.yml \ + --env-file ".env.${ENVIRON}" \ build --pull ${NO_CACHE:+$NO_CACHE} tiler - ./scripts/publish ${DEPLOY_NEW:+$DEPLOY_NEW} + ./scripts/publish ${DEPLOY_NEW:+$DEPLOY_NEW} "${ENVIRON}" } @@ -45,12 +48,14 @@ then NO_CACHE='--no-cache' shift ;; - -h| --help| *) + -h| --help) usage ;; + *) + ENVIRON=$1 + shift esac done - main + main ENVIRON fi - diff --git a/scripts/publish b/scripts/publish index 88bd93e0..f9febc6b 100755 --- a/scripts/publish +++ b/scripts/publish @@ -3,6 +3,7 @@ set -e DEPLOY_NEW="" +ENVIRON="dev" function usage() { echo -n "Usage: $(basename "${0}") [OPTION] @@ -15,6 +16,7 @@ Options: function main() { docker-compose -f docker-compose.yml \ + --env-file ".env.${ENVIRON}" \ run --rm --no-deps -e NODE_ENV=production tiler yarn deploy${DEPLOY_NEW:+-new} # Deploy CloudFront proxy @@ -33,6 +35,11 @@ then -h| --help) usage exit + ;; + *) + ENVIRON=$1 + shift esac - main + echo "Deploying to environment '${ENVIRON}'" + main ENVIRON fi diff --git a/scripts/server b/scripts/server index a4115350..5c0f57c9 100755 --- a/scripts/server +++ b/scripts/server @@ -15,18 +15,19 @@ COMMAND can be one of [start, stop, restart, status]. Default is start. function main() { SERVICES='tiler database' + DC_COMMAND='docker-compose --env-file .env.local' if [ "${1}" = "start" ] then - docker-compose up ${SERVICES} + ${DC_COMMAND} up ${SERVICES} elif [ "${1}" = "stop" ] then - docker-compose stop ${SERVICES} + ${DC_COMMAND} stop ${SERVICES} elif [ "${1}" = "restart" ] then - docker-compose restart ${SERVICES} + ${DC_COMMAND} restart ${SERVICES} elif [ "${1}" = "status" ] then - docker-compose ps + ${DC_COMMAND} ps else echo "Unknown option ${1}! Must be one of [start, stop, restart, status ]." exit 1 From 2fda8cf777258a6d4ff6ca674a1f0b090334e6e6 Mon Sep 17 00:00:00 2001 From: Klaas Hoekema Date: Wed, 3 Jul 2019 23:06:53 -0400 Subject: [PATCH 2/4] Remove default env file from docker-compose --- docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index c7942a1b..140c9f9a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,7 +44,6 @@ services: terraform: image: hashicorp/terraform:0.11.14 - env_file: .env environment: - TF_VAR_region=${LAMBDA_REGION} - TF_VAR_source_name=${PROJECT_NAME}-${USER} From ea1081b1f8945a5e7020acde5da4fe54ebf59fcd Mon Sep 17 00:00:00 2001 From: Klaas Hoekema Date: Wed, 3 Jul 2019 23:07:26 -0400 Subject: [PATCH 3/4] Add an ENVIRON variable and use it to set deployment name Changes the deployment-related scripts to expect an argument for which environment to deploy. They'll look for a `.env.ENVIRON` file to set the environment variables and will create claudia-ENVIRON.json and .api-id-ENVIRON files rather than always using the same unqualified names. Using separate claudia.json files means the config file is now given as an argument, rather than using the default, and they go in a subdirectory. --- .env.local | 1 + .env.template | 5 +++- docker-compose.yml | 8 ++++-- scripts/deploy | 6 ++--- scripts/destroy | 27 ++++++++++--------- scripts/publish | 51 ++++++++++++++++++++++-------------- src/tiler/.gitignore | 5 ++-- src/tiler/claudia/.gitkeep | 0 src/tiler/package.json | 4 +-- src/tiler/scripts/deploy | 9 ++++++- src/tiler/scripts/deploy-new | 16 ++++++++--- 11 files changed, 86 insertions(+), 46 deletions(-) create mode 100644 src/tiler/claudia/.gitkeep diff --git a/.env.local b/.env.local index 738965a2..381a4e4f 100644 --- a/.env.local +++ b/.env.local @@ -1,3 +1,4 @@ +ENVIRON=local # Database credentials for the local development DB TILEGARDEN_DB_NAME=postgres TILEGARDEN_DB_HOST=database.internal.tilegarden diff --git a/.env.template b/.env.template index ac3729c2..d1a99c44 100644 --- a/.env.template +++ b/.env.template @@ -1,8 +1,11 @@ +# Name of this deployment environment (e.g. 'dev', 'staging', 'production') +ENVIRON=dev + # Import values from local environment # If you want to set them specifically, change them to assignments USER AWS_PROFILE -# $USER will be appended to the project name specified here +# $USER and environment will be appended to the project name specified here PROJECT_NAME= # Database credentials diff --git a/docker-compose.yml b/docker-compose.yml index 140c9f9a..860b4c7b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,7 @@ services: - "9229:9229" environment: # Variables to pass through from environment-specific .env files + - ENVIRON - USER - AWS_PROFILE - PROJECT_NAME @@ -45,8 +46,11 @@ services: terraform: image: hashicorp/terraform:0.11.14 environment: - - TF_VAR_region=${LAMBDA_REGION} - - TF_VAR_source_name=${PROJECT_NAME}-${USER} + - AWS_PROFILE + # Providing placeholder defaults here stops docker-compose from warning about missing + # variables every time you run it. + - TF_VAR_region=${LAMBDA_REGION:-us-east-1} + - TF_VAR_source_name=${PROJECT_NAME:-tilegarden}-${USER}-${ENVIRON:-unknown} working_dir: /home/terraform volumes: - ./src/terraform:/home/terraform diff --git a/scripts/deploy b/scripts/deploy index 0977adc9..d29dc4e2 100755 --- a/scripts/deploy +++ b/scripts/deploy @@ -18,8 +18,8 @@ Options: } function main() { - if [[ $DEPLOY_NEW != "--new" && ! -f ./src/tiler/claudia.json ]]; then - echo "No existing deployment found to update! (did you mean to use '--new'?)" + if [[ $DEPLOY_NEW != "--new" && ! -f "./src/tiler/claudia/claudia-${ENVIRON}.json" ]]; then + echo "No existing '$ENVIRON' deployment found to update! (did you mean to use '--new'?)" exit 1 fi @@ -57,5 +57,5 @@ then esac done - main ENVIRON + main $ENVIRON fi diff --git a/scripts/destroy b/scripts/destroy index 0cc2e60d..3f8c80e2 100755 --- a/scripts/destroy +++ b/scripts/destroy @@ -3,32 +3,36 @@ set -e function usage() { - echo -n "Usage: $(basename "${0}") -Removes all published resources from AWS + echo -n "Usage: $(basename "${0}") ENVIRON +Removes all published resources from AWS for the specified deployment. Options: --help Display this help text " } function main() { - echo "WARNING: you are about to destroy all your deployed resources!" + ENVIRON=${1} + echo "WARNING: you are about to destroy all your deployed resources for '${ENVIRON}'!" printf "That includes:\n - Associated AWS Lambda functions\n - Associated API Gateways\n - Associated IAM roles (if automatically generated)\n - Associated CloudFront distributions\n" read -p "If this is what you want, type 'yes' to continue: " -r echo if [[ $REPLY =~ ^[yY][eE][sS]$ ]] then - # Destroy deployed resources # If any part of this fails, the parts that didn't fail # remain destroyed, so it's best to just carry on and # follow through. - docker-compose run --no-deps tiler yarn destroy || true - docker-compose run -e TF_VAR_source_id=$( .api-id", + "parse-id": "jq -r '.api.id' claudia/claudia-${ENVIRON}.json > .api-id-${ENVIRON}", "test": "eslint src && jest --coverage" }, "devDependencies": { diff --git a/src/tiler/scripts/deploy b/src/tiler/scripts/deploy index f8c45638..6f9681ab 100755 --- a/src/tiler/scripts/deploy +++ b/src/tiler/scripts/deploy @@ -1,6 +1,13 @@ #!/bin/bash -yarn claudia update --no-optional-dependencies \ +if [ -z "${ENVIRON}" ]; then + echo 'Required $ENVIRON variable not set.' + exit 1 +fi + +yarn claudia update \ + --config claudia/claudia-${ENVIRON}.json \ + --no-optional-dependencies \ ${LAMBDA_TIMEOUT:+--timeout ${LAMBDA_TIMEOUT}} \ ${LAMBDA_MEMORY:+--memory ${LAMBDA_MEMORY}} \ ${LAMBDA_SECURITY_GROUPS:+--security-group-ids ${LAMBDA_SECURITY_GROUPS}} \ diff --git a/src/tiler/scripts/deploy-new b/src/tiler/scripts/deploy-new index 334451ce..a4cf15f1 100755 --- a/src/tiler/scripts/deploy-new +++ b/src/tiler/scripts/deploy-new @@ -1,11 +1,21 @@ #!/bin/bash -yarn claudia create --no-optional-dependencies --api-module src/api \ - --name ${PROJECT_NAME} --region ${LAMBDA_REGION} \ +if [ -z "${ENVIRON}" ]; then + echo 'Required $ENVIRON variable not set.' + exit 1 +elif [ -z "${PROJECT_NAME}" ]; then + echo 'Required $PROJECT_NAME variable not set.' + exit 1 +fi + +yarn claudia create \ + --config claudia/claudia-${ENVIRON}.json \ + --no-optional-dependencies --api-module src/api \ + --name "${PROJECT_NAME}-${USER}-${ENVIRON}" --region ${LAMBDA_REGION} \ ${LAMBDA_ROLE:+--role ${LAMBDA_ROLE}} \ ${LAMBDA_TIMEOUT:+--timeout ${LAMBDA_TIMEOUT}} \ ${LAMBDA_MEMORY:+--memory ${LAMBDA_MEMORY}} \ ${LAMBDA_SECURITY_GROUPS:+--security-group-ids ${LAMBDA_SECURITY_GROUPS}} \ ${LAMBDA_SUBNETS:+--subnet-ids ${LAMBDA_SUBNETS}} \ --set-env TILEGARDEN_DB_HOST=${TILEGARDEN_DB_HOST},TILEGARDEN_DB_NAME=${TILEGARDEN_DB_NAME},TILEGARDEN_DB_PASSWORD=${TILEGARDEN_DB_PASSWORD},TILEGARDEN_DB_PORT=${TILEGARDEN_DB_PORT},TILEGARDEN_DB_USER=${TILEGARDEN_DB_USER} \ -&& yarn parse-id +&& jq -r '.api.id' claudia/claudia-${ENVIRON}.json > .api-id-${ENVIRON} From 31dbca091281896bd332a0b6d79cfde55bb3a440 Mon Sep 17 00:00:00 2001 From: Klaas Hoekema Date: Sat, 3 Aug 2019 22:03:25 -0400 Subject: [PATCH 4/4] Adjust Terraform for multi-env setup Changes the terraform deployment steps to use local tfplan and tfstate files to keep track of separate environments. The "local" backend is the default, but specifying it explicitly enables specifying parameters for it, i.e. the ENVIRON-based filenames to use. This also switches from using `plan -auto-apply` to doing separate plan and apply commands, though they're still within the same `publish` script. The behavior will end up the same, but it now gives you 10 seconds to bail out if the plan looks weird, rather than applying it as soon as it's calculated. --- scripts/publish | 28 +++++++++++++++++++++++----- src/terraform/cloudfront.tf | 4 ++++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/scripts/publish b/scripts/publish index 8ebefc3a..c7f79da6 100755 --- a/scripts/publish +++ b/scripts/publish @@ -21,11 +21,29 @@ function main() { run --rm --no-deps -e NODE_ENV=production tiler yarn deploy${DEPLOY_NEW:+-new} # Deploy CloudFront proxy - docker-compose --env-file ".env.${ENVIRON}" run terraform init - docker-compose --env-file ".env.${ENVIRON}" \ - run --rm -e TF_VAR_source_id=$(