Setup CICD pipeline

lists-service/.gitignore

@@ -29,6 +29,8 @@ build/
 !**/src/main/**/build/
 !**/src/test/**/build/
 
+!**/lists-service/cicd/build/
+
 ### VS Code ###
 .vscode/
 /lists-ui/src/.env.json

lists-service/cicd/base/app/base.yaml

@@ -0,0 +1,85 @@
+AWSTemplateFormatVersion: '2010-09-09'
+
+Description: Base resources for the Lists service
+
+Parameters:
+ pBuild:
+ Type: String
+ Description: The build number
+ pCleanBranch:
+ Type: String
+ Description: The clean branch, can be used in resource names
+ pEnvironment:
+ Type: String
+ Description: The AWS environment this belongs to
+
+Conditions:
+
+ IsDev: !Equals
+ - !Ref pEnvironment
+ - development
+ NotProd: !Not
+ - !Equals
+ - !Ref pEnvironment
+ - production
+
+Resources:
+
+ ListsRepository:
+ Type: AWS::ECR::Repository
+ Properties:
+ EmptyOnDelete: !If [ NotProd, true, false ]
+ EncryptionConfiguration:
+ EncryptionType: AES256
+ ImageScanningConfiguration:
+ ScanOnPush: true
+ RepositoryName: !Sub
+ - lists-${ResourceName}
+ - ResourceName: !If [ IsDev, !Ref pCleanBranch, !Ref pEnvironment ]
+ RepositoryPolicyText:
+ Version: 2012-10-17
+ Statement:
+ - Sid: AllowPublicPull
+ Effect: Allow
+ Principal: '*'
+ Action:
+ - 'ecr:GetDownloadUrlForLayer'
+ - 'ecr:BatchGetImage'
+ - 'ecr:BatchCheckLayerAvailability'
+ Tags:
+ - Key: Environment
+ Value: !Ref pEnvironment
+ - Key: Branch
+ Value: !Ref pCleanBranch
+
+ ListsSecret:
+ Type: 'AWS::SecretsManager::Secret'
+ Properties:
+ Name: !Sub 
+ - lists-${ResourceName}
+ - ResourceName: !If [ IsDev, !Ref pCleanBranch, !Ref pEnvironment ]
+ Description: !Sub Lists app ${pEnvironment} secrets
+ GenerateSecretString: 
+ GenerateStringKey: db-password
+ PasswordLength: 12
+ ExcludeCharacters: "/@\" "
+ SecretStringTemplate: |
+ {
+ "db-password": ""
+ }
+
+
+
+Outputs:
+
+ ListsRepositoryArn:
+ Description: The ECR repository ARN for the Lists service
+ Value: !GetAtt ListsRepository.Arn
+
+ ListsRepositoryName:
+ Description: The ECR repository ARN for the Lists service
+ Value: !Ref ListsRepository
+
+ ListsSecret:
+ Description: The Secrets name for lists 
+ Value: !Ref ListsSecret

lists-service/cicd/base/app/base_template_config.j2

@@ -0,0 +1,25 @@
+{
+ "Parameters" : {
+ "pBuild" : "{{ codebuild_build_number }}",
+ "pCleanBranch" : "{{ clean_branch }}",
+ "pEnvironment" : "{{ environment }}"
+ },
+ "Tags" : {
+ "product" : "{{ product_name }}",
+ "component" : "{{ product_component }}",
+ "environment" : "{{ environment }}",
+ "branch" : "{{ src_branch }}",
+ "version" : "{{ commit_id }}",
+ "build" : "{{ codebuild_build_number }}"
+ },
+ "StackPolicy" : {
+ "Statement" : [
+ {
+ "Effect" : "Allow",
+ "Action" : "Update:*",
+ "Principal": "*",
+ "Resource" : "*"
+ }
+ ]
+ }
+}

lists-service/cicd/base/config.ini

@@ -0,0 +1,22 @@
+[DEFAULT]
+PRODUCT_COMPONENT = base
+PIPELINE_STACK_NAME = ala-${PRODUCT_NAME}-${PRODUCT_COMPONENT}-pipeline-${ENVIRONMENT}
+BASE_STACK_FILE_PFIX = base
+BASE_STACK_NAME = ala-${PRODUCT_NAME}-${PRODUCT_COMPONENT}-${ENVIRONMENT}
+AUTO_DEPLOY = false
+SLACK_DEPLOY_NOTIFICATION = false
+SLACK_ALERT_CHANNEL = deployments
+
+[development]
+# code pipeline
+PIPELINE_STACK_NAME = ala-${PRODUCT_NAME}-${PRODUCT_COMPONENT}-pipeline-${CLEAN_BRANCH}
+BASE_STACK_NAME = ala-${PRODUCT_NAME}-${PRODUCT_COMPONENT}-${CLEAN_BRANCH}
+SLACK_DEPLOY_NOTIFICATION = true
+SLACK_ALERT_CHANNEL = zabbix-lists
+AUTO_DEPLOY = true
+
+[develop]
+
+[staging]
+
+[production]

lists-service/cicd/base/pipeline/deploy_notification_buildspec.yaml

@@ -0,0 +1,59 @@
+version: 0.2
+###
+# This build project sends out the various deploy notifications
+
+env:
+ shell: bash
+ variables:
+ DEBIAN_FRONTEND: "noninteractive"
+ exported-variables:
+ - CODEBUILD_BUILD_NUMBER
+ secrets-manager:
+ DD_API_KEY: ala-secrets-production:datadog-api-key
+ SLACK_OAUTH_TOKEN: ala-secrets-production:slack-oauth-token
+
+phases:
+
+ install:
+ commands:
+ - echo Running on $(lsb_release -d | cut -f2)
+ - echo aws-cli version $(aws --version)
+ finally:
+ - #echo This always runs even if the update or install command fails
+
+
+ pre_build:
+ commands:
+ - echo Entered the pre_build phase...
+ - # have to get the commit message here because passing it as an env var from
+ - # the pipeline doesn't work when the commit message contains json breaking characters
+ - export PIPELINE_NAME=$(echo $CODEBUILD_INITIATOR | cut -d'/' -f2)
+ - export COMMIT_MSG=$(aws codepipeline list-pipeline-executions --pipeline-name $PIPELINE_NAME --max-items 1 --query 'pipelineExecutionSummaries[0].sourceRevisions[0].revisionSummary' | jq -r '. | fromjson | .CommitMessage')
+ - echo source branch is $SRC_BRANCH
+ - echo clean branch is $CLEAN_BRANCH
+ - echo Environment is $ENVIRONMENT
+ - echo commit msg is $COMMIT_MSG
+ - echo commit id is $COMMIT_ID
+ - echo Repo is $REPO
+ - export DEPLOY_MSG="$AUTHOR has released a $ENVIRONMENT update to $PRODUCT_NAME $PRODUCT_COMPONENT."
+ - echo $DEPLOY_MSG
+ finally:
+ - #echo This always runs
+
+ build:
+ commands:
+ - echo Datadog deploy notification
+ - lists-service/cicd/dd_notification.sh
+ - echo Slack Deploy notification
+ - |
+ if [ "$SLACK_DEPLOY_NOTIFICATION" == "true" ] ; then
+ lists-service/cicd/slack_notification.sh
+ fi
+
+ finally:
+ - #echo This always runs
+
+
+ post_build:
+ commands:
+ - #echo Entered the post_build phase...

lists-service/cicd/base/pipeline/deploy_pipeline.sh

@@ -0,0 +1,153 @@
+#!/bin/bash
+set -ueo pipefail
+
+###
+# Deploy the codepipeline for the lists app
+# You must have AWS CLI authentication for this to run. 
+
+usage() {
+ echo "Usage: $0 [OPTIONS]"
+ echo "Options:"
+ echo " -e The environment. Optional but must be \"prod\" to launch in production"
+ echo " -b Branch override. Used when we're not on a branch (detached head)"
+}
+
+ENV=nonprod
+BRANCH_OVERRIDE=
+SCRIPT_DIR=$(dirname "$(realpath "$0")")
+
+while getopts "e:b:" flag; do
+ case $flag in
+ e) # Handle the -e environment flag
+ # must be "prod" to launch in production
+ ENV=$OPTARG
+ ;;
+ b) # Handle the -b branch override flag
+ # branch override is used when we're not on a branch (detached head)
+ BRANCH_OVERRIDE=$OPTARG
+ ;;
+ \?)
+ usage
+ exit 1
+ ;;
+ esac
+done
+
+# get the branch
+branch=$(git branch --show-current)
+
+# confirm which environment we're deploying to if it wasnt explicitly set
+if [ "$branch" = "main" ] && [ "$ENV" = "nonprod" ]; then
+ echo "Deploy to production or staging?"
+ echo "1) production"
+ echo "2) staging"
+ read -p "Enter your choice (1 or 2): " choice
+
+ case $choice in
+ 1)
+ ENV="prod"
+ ;;
+ esac
+fi
+
+# check if we're on a detached head
+if [[ -n $branch ]]; then
+ real_branch=1
+elif [[ -z $branch && -n $BRANCH_OVERRIDE ]]; then 
+ real_branch=0
+ branch=$BRANCH_OVERRIDE
+else
+ echo "You must specify a branch override or run this script from a git branch"
+ exit 1;
+fi
+
+echo branch: $branch
+
+# get the commit_ids
+COMMIT_ID=$(git rev-parse HEAD)
+echo commit id: $COMMIT_ID
+LATEST_COMMIT_ID=$(git log -n 1 --pretty=format:"%H" $branch)
+
+# check if we're rolling back to a previous commit
+if [[ "$COMMIT_ID" != "$LATEST_COMMIT_ID" ]]; then
+ echo latest commit is $LATEST_COMMIT_ID
+ echo but we are releasing $COMMIT_ID
+ echo we are rolling back to a previous commit
+ RESTART_PIPELINE_ON_UPDATE=false
+else
+ RESTART_PIPELINE_ON_UPDATE=true
+fi
+
+# check that any changes are commited and pushed
+if [[ $real_branch -eq 1 && -n "$(git status --porcelain)" ]] ; then
+ echo "changes must be committed and pushed before deploying"
+ exit 1;
+fi
+
+# check the remote branch exists
+if [[ $real_branch -eq 1 ]] && ! git ls-remote --exit-code origin $branch > /dev/null 2>&1 ; then
+ echo "changes must be committed and pushed before deploying"
+ exit 1;
+fi
+
+# check there are no differences with remote
+if [[ $real_branch -eq 1 && -n "$(git diff $branch origin/$branch)" ]] ; then
+ echo "changes must be committed and pushed before deploying"
+ exit 1;
+fi
+
+# get the clean version of the branch
+clean_branch=$($SCRIPT_DIR/../../clean_branch.sh $branch)
+echo clean branch: $clean_branch
+
+# get the environment based on the branch
+environment=$($SCRIPT_DIR/../../branch_2_env.py --branch $branch --env $ENV)
+echo environment: $environment
+
+# load environment vars
+$SCRIPT_DIR/../../gen_env_vars.py --env $environment --clean-branch $clean_branch --conf $SCRIPT_DIR/../config.ini > env.txt
+source env.txt
+rm env.txt
+
+# Determine the operating system
+OS=$(uname)
+
+# Calculate pipeline MD5 based on the operating system
+case "$OS" in
+ "Darwin")
+ # macOS
+ PIPELINE_MD5=$(md5 -q pipeline.yaml)
+ ;;
+ "Linux")
+ # Linux
+ PIPELINE_MD5=$(md5sum pipeline.yaml | awk '{ print $1 }')
+ ;;
+ *)
+ echo "Unsupported OS: $OS"
+ exit 1
+ ;;
+esac
+
+# deploy/update the template
+echo "Deploying the pipeline template"
+aws cloudformation deploy \
+ --template-file pipeline.yaml \
+ --stack-name $PIPELINE_STACK_NAME \
+ --tags product=$PRODUCT_NAME component=cicd environment=$environment branch=$branch version=$COMMIT_ID \
+ --region $REGION \
+ --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM \
+ --parameter-overrides \
+ pAutoDeploy=$AUTO_DEPLOY \
+ pBootstrapStackName=$BOOTSTRAP_STACK_NAME \
+ pBucketsStackName=$BUCKETS_STACK_NAME \
+ pCleanBranch=$clean_branch \
+ pEksClusterName=$EKS_CLUSTER_NAME \
+ pEnvironment=$environment \
+ pGitHubBranch=$branch \
+ pGitHubOwner=$GITHUB_OWNER \
+ pGitHubRepositoryName=$GITHUB_REPO_NAME \
+ pPipelineFingerprint=$PIPELINE_MD5 \
+ pProductComponent=$PRODUCT_COMPONENT \
+ pProductName=$PRODUCT_NAME \
+ pRestartExecutionOnUpdate=$RESTART_PIPELINE_ON_UPDATE \
+