ci: add amiize CI build harness & supporting tools

This adds a CI specific harness for creating AMIs from built disk
images. To accomplish the task at hand, the script "create-ami-image"
manages the use of build artifacts and kicks off the amiize process
according to its build environment. "ensure-key-pair" validates and/or
creates an EC2 key pair for its use during automated builds. This key
may be rotated (by way of deletion) as needed with additional
straightforward & well scoped permissions needed for the build task to
manage its own key pair (aside from the overlapping EC2 permissions
needed for amiizing):

- ssm:PutParameter
- ssm:GetParameter
- ec2:ImportKey
- ec2:DescribeKeyPairs
- kms:Encrypt
- kms:Decrypt

The KMS documentation page regarding SSM Parameter Store has much more
outlined on restricting the usage of SSM' AWS-Managed CMK to the SSM
Parameters involved as well.

Signed-off-by: Jacob Vallejo <[email protected]>

bin/amiize.sh

@@ -106,6 +106,7 @@ $(basename "${0}")
  [ --root-volume-size 1234 ]
  [ --data-volume-size 5678 ]
  [ --security-group-name default | --security-group-id sg-abcdef1234 ]
+ [ --write-output-dir output-dir ]
 
 Registers the given images as an AMI in the given EC2 region.
 
@@ -130,6 +131,9 @@ Optional:
  --security-group-id The ID of a security group name that allows SSH access from this host
  --security-group-name The name of a security group name that allows SSH access from this host
  (defaults to "default" if neither name nor ID are specified)
+ --write-output-dir The directory to write out IDs into attribute named files.
+ (not written out to anywhere other than log otherwise)
+
 EOF
 }
 
@@ -161,6 +165,7 @@ parse_args() {
  --data-volume-size ) shift; DATA_VOLUME_SIZE="${1}" ;;
  --security-group-name ) shift; SECURITY_GROUP_NAME="${1}" ;;
  --security-group-id ) shift; SECURITY_GROUP_ID="${1}" ;;
+ --write-output-dir ) shift; WRITE_OUTPUT_DIR="${1}" ;;
 
  --help ) usage; exit 0 ;;
  *)
@@ -323,6 +328,19 @@ check_return() {
  return 0
 }
 
+# Helper to conditionally write out attribute if WRITE_OUTPUT_DIR is
+# configured.
+write_output() {
+ local name="$1"
+ local value="$2"
+
+ if [[ -z "${WRITE_OUTPUT_DIR}" ]]; then
+ return
+ fi
+
+ mkdir -p "${WRITE_OUTPUT_DIR}/$(dirname "$name")"
+ echo -n "$value" > "${WRITE_OUTPUT_DIR}/${name}"
+}
 
 # =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^=
 
@@ -623,6 +641,7 @@ while true; do
  echo "* Warning: Could not delete root volume!"
  # Don't die though, we got what we want...
  fi
+ write_output "root_snapshot_id" "$root_snapshot"
 
  if aws ec2 delete-volume \
  --output text \
@@ -635,6 +654,7 @@ while true; do
  echo "* Warning: Could not delete data volume!"
  # Don't die though, we got what we want...
  fi
+ write_output "data_snapshot_id" "$data_snapshot"
 
  # =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^=
 
@@ -657,6 +677,8 @@ while true; do
  check_return ${?} "AMI registration failed!" || continue
  echo "Registered ${registered_ami}"
 
+ write_output "ami_id" "$registered_ami"
+
  echo "Waiting for the AMI to appear in a describe query"
  waits=0
  while [ ${waits} -lt 20 ]; do

tools/infra/container/runtime/bin/create-image-ami

@@ -0,0 +1,271 @@
+#!/usr/bin/env bash
+#
+# create-image-ami - Create an AMI from a build's image
+#
+# usage:
+#
+# create-image-ami
+#
+# Environment variable overrides for building, testing, and otherwise
+# using this script (other variables, undocumented here, may be
+# overridden also - see script below):
+#
+# BUILD_AMI_NAME
+#
+# Name the AMI exactly instead of calculating the image's name
+#
+# BUILD_REGION
+#
+# Region to create and register the AMI and its snapshots in.
+#
+# The region must be configured and provisioned with the
+# required resources for both EC2 AMI creation and the SSM resources
+# used for manipulating the launched image building instance.
+#
+# BUILD_IMAGE_ROOT, BUILD_IMAGE_DATA
+#
+# Paths, to the ROOT and DATA images respectively, for
+# specifying exact disk images to use instead of relying on path
+# construction.
+#
+# BUILD_INSTANCE_AMI, BUILD_INSTANCE_TYPE
+#
+# AMI ID and the EC2 Instance Type may be specified to
+# explicitly choose an AMI and the Instance Type used to launch,
+# instead of querying for the latest Amazon Linux 2 in-region
+# AMI and the default EC2 Instance Type.
+#
+# KEYPAIR_NAME
+#
+# Name of the EC2 Key Pair (as named during creation or import)
+# provisioned and accessible in the SSM Parameter (configured
+# with KEYPAIR_PARAMETER).
+#
+# KEYPAIR_PARAMETER
+#
+# SSM Parameter Name (eg: "/Prod/ami-build/builder-ssh-key")
+# that holds the Private SSH Key that corresponds to the EC2 Key
+# Pair (specified in KEYPAIR_NAME) used to access the image
+# building instance.
+#
+
+set -o pipefail
+
+# BUILD_OUTPUT is the directory in which resource data will be written
+# to as files.
+BUILD_OUTPUT="${BUILD_OUTPUT:-build/ami}"
+# BUILD_KEEP_OUTPUT can be set to non-nil to retain existing output in
+# BUILD_OUTPUT.
+BUILD_KEEP_OUTPUT="${BUILD_KEEP_OUTPUT:+yes}"
+# BUILD_REGION is the region the image should be built in.
+BUILD_REGION="${AWS_DEFAULT_REGION:-us-west-2}"
+# BUILD_INSTANCE_TYPE is the instance type chosen to run the amiize
+# build on.
+BUILD_INSTANCE_TYPE="${BUILD_INSTANCE_TYPE:-m3.xlarge}"
+# BUILD_AMI provides an override to choose an image to use, otherwise
+# the latest release of Amazon Linux 2 is used.
+BUILD_INSTANCE_AMI="${BUILD_INSTANCE_AMI:-}"
+# BUILDSYS_VARIANT specifies the image's variant to be amiized.
+BUILDSYS_VARIANT="${BUILDSYS_VARIANT:-aws-k8s}"
+# BUILD_ARCH is the image's architecture.
+BUILD_ARCH="${BUILD_ARCH:-x86_64}"
+# ami_suffix provides a the dynamic portion of an AMI name using the
+# environment of a build.
+#
+# CodeBuild based defaults use pre-defined Environment Variables as
+# documented:
+# https://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref-env-vars.html
+#
+if [[ -n "$CODEBUILD_RESOLVED_SOURCE_VERSION" ]]; then
+ ami_suffix="${CODEBUILD_RESOLVED_SOURCE_VERSION:0:8}"
+else
+ ami_suffix="$USER-$(date +'%s')"
+fi
+# BUILD_AMI_NAME is the created AMI's EC2 AMI Name.
+BUILD_AMI_NAME="${BUILD_AMI_NAME:-thar-${BUILD_ARCH}-${ami_suffix}}"
+# BUILD_IMAGE_ROOT may be specified to provide a specific root disk
+# image to write out.
+BUILD_IMAGE_ROOT="${BUILD_IMAGE_ROOT:-build/thar-$BUILD_ARCH-${BUILDSYS_VARIANT}.img.lz4}"
+# BUILD_IMAGE_DATA may be specified to provide a specific data disk
+# image to write out.
+BUILD_IMAGE_DATA="${BUILD_IMAGE_ROOT:-build/thar-$BUILD_ARCH-${BUILDSYS_VARIANT}-data.img.lz4}"
+# EC2 Key Pair used to spin up and access instance for AMIizing disk
+# images. These are created automatically if the build task is
+# configured with access to the SSM, EC2, and KMS resources involved.
+#
+# KEYPAIR_NAME is the EC2 KeyPair Name
+KEYPAIR_NAME="${KEYPAIR_NAME:-ami-build-key}"
+# KEYPAIR_PARAMETER is the SecureString SSM Parameter used to hold the
+# private key for the specified EC2 Key Pair (in KEYPAIR_NAME).
+KEYPAIR_PARAMETER="${KEYPAIR_PARAMETER:-/Prod/ami-build/$KEYPAIR_NAME}"
+
+WORK_DIR="$(mktemp -d -t create-image-ami.XXX)"
+
+# Shim for `logger` to write out to STDERR correctly throughout.
+#
+# shellcheck disable=2032
+logger() {
+ command logger --stderr --no-act "$@"
+}
+
+# Explicitly configured aws-cli for making API calls.
+aws() {
+ command aws --region "$BUILD_REGION" "$@"
+}
+
+# prepareImage massages a provided disk image into a format suitable
+# for use in the amiize process and returns that file's name.
+#
+# usage: prepareImage <disk-image.img[.lz4]>
+prepareImage() {
+ local image_file="${1:?need image file to prepare it}"
+ local out
+ case "$image_file" in
+ *.lz4 )
+ un_name="${image_file%%.lz4}"
+ out="$WORK_DIR/${un_name##*/}"
+ logger -t INFO "decompressing LZ4 disk image: $image_file to $out"
+ lz4 -dc "$image_file" > "$out" || return 1
+ ;;
+ *.img )
+ out="$image_file"
+ logger -t INFO "using provided raw disk image: $out"
+ ;;
+ * )
+ logger -t ERROR "unknown image file provided: $image_file"
+ return 1
+ ;;
+ esac
+
+ echo "$out"
+}
+
+# cleanup our files and helping processes before exiting
+cleanup() {
+ if [[ -n "$SSH_AGENT_PID" ]]; then
+ logger -t INFO "terminating key-wielding ssh-agent"
+ eval "$(ssh-agent -k | grep -v '^echo')"
+ fi
+
+ if [[ -d "$WORK_DIR" ]]; then
+ logger -t INFO "removing work directory $WORK_DIR"
+ rm -rf -- "$WORK_DIR"
+ fi
+}
+
+trap "cleanup" EXIT
+
+mkdir -p "$WORK_DIR"
+
+if ! [[ -s "$BUILD_IMAGE_ROOT" ]]; then
+ logger -t ERROR "missing root disk image"
+ exit 1
+fi
+if ! [[ -s "$BUILD_IMAGE_DATA" ]]; then
+ logger -t ERROR "missing data disk image"
+ exit 1
+fi
+
+logger -t INFO "using ssh key from SSM parameter '$KEYPAIR_PARAMETER'"
+if ! (
+ # shellcheck disable=SC2030
+ export KEYPAIR_NAME KEYPAIR_PARAMETER
+ export AWS_DEFAULT_REGION="$BUILD_REGION"
+ ensure-key-pair
+ ); then
+ logger -t ERROR "unable to setup ssh key from SSM parameter"
+ exit 1
+fi
+
+logger -t INFO "configuring ssh for SSM parameter ssh key"
+# shellcheck disable=SC2091
+eval "$(ssh-agent | grep -v '^echo')"
+# shellcheck disable=SC2031
+if ! ssh-add <(aws ssm get-parameter --name "$KEYPAIR_PARAMETER" --with-decryption --query Parameter.Value --output text) ; then
+ logger -t ERROR "unable to load ssh key from SSM Parameter $KEYPAIR_PARAMETER"
+ exit 1
+fi
+
+if [[ -z "$BUILD_AMI" ]]; then
+ logger -t INFO "querying SSM for latest Amazon Linux 2 AMI"
+ BUILD_INSTANCE_AMI="$(aws ssm get-parameter --output text --query Parameter.Value \
+ --name /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2)"
+ # shellcheck disable=2181
+ if [[ "$?" -ne 0 ]]; then
+ logger -t ERROR "AMI query failed, cannot proceed without image"
+ exit 1
+ fi
+ if [[ -z "$BUILD_INSTANCE_AMI" ]]; then
+ logger -t ERROR "AMI ID is empty, cannot proceed without image"
+ exit 1
+ fi
+fi
+logger -t INFO "using $BUILD_INSTANCE_AMI for amiizing instance"
+
+logger -t INFO "preparing disk images for amiizing"
+if ! BUILD_IMAGE_ROOT="$(prepareImage "$BUILD_IMAGE_ROOT")"; then
+ logger -t ERROR "failed to prepare root image for amiizing"
+ exit 1
+fi
+
+if ! BUILD_IMAGE_DATA="$(prepareImage "$BUILD_IMAGE_DATA")"; then
+ logger -t ERROR "failed to prepare data image for amiizing"
+ exit 1
+fi
+
+userdata="$(base64 -w 0 <<EOF
+#cloud-config
+repo_upgrade: none
+
+EOF
+)"
+
+logger -t INFO "starting amiize, may take a while"
+amiize_output="${WORK_DIR}/ami"
+
+(
+ set -x
+
+ # TODO: add security group check / provision *somewhere*
+
+ # shellcheck disable=2031
+ amiize.sh \
+ --region "$BUILD_REGION" \
+ --root-image "$BUILD_IMAGE_ROOT" \
+ --data-image "$BUILD_IMAGE_DATA" \
+ --worker-ami "$BUILD_INSTANCE_AMI" \
+ --ssh-keypair "$KEYPAIR_NAME" \
+ --instance-type "$BUILD_INSTANCE_TYPE" \
+ --name "$BUILD_AMI_NAME" \
+ --arch "$BUILD_ARCH" \
+ --user-data "$userdata" \
+ --write-output-dir "$amiize_output"
+)
+amiize_ret="$?"
+if [[ "$amiize_ret" -eq 0 ]]; then
+ logger -t INFO -- "created AMI from images"
+else
+ logger -t ERROR -- "failed to create AMI from images"
+fi
+
+# Record created image resources in $BUILD_OUTPUT
+if [[ -d "$amiize_output" ]]; then
+ echo -n "$BUILD_REGION" > "$amiize_output/region"
+
+ # When we're writing to an existing directory, we allow the caller to retain
+ # existing resources there also while (effectively) overwriting the
+ # overlapping resources.
+ if [[ -d "$BUILD_OUTPUT" ]]; then
+ if [[ -z "$BUILD_KEEP_OUTPUT" ]]; then
+ logger -t WARN -- "removing prior data (in $BUILD_OUTPUT) to record newly created resources"
+ # Remove overlapping files in the $BUILD_OUTPUT first.
+ find "$amiize_output" -mindepth 1 -printf "$BUILD_OUTPUT/%P\0" | xargs -0 -- rm -rf --
+ fi
+ fi
+
+ logger -t INFO "recording created resource IDs in $BUILD_OUTPUT"
+ mkdir -p "$BUILD_OUTPUT"
+ cp -rT "$amiize_output/" "$BUILD_OUTPUT/"
+fi
+
+exit "$amiize_ret"

tools/infra/container/runtime/bin/ensure-key-pair

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+KEYPAIR_NAME="${KEYPAIR_NAME:?Need a keypair name}"
+KEYPAIR_PARAMETER="${KEYPAIR_PARAMETER:?Need a parameter name}"
+
+logger() {
+ command logger --stderr --no-act "$@"
+}
+
+ssh_keypair="$(mktemp -u -t "ssm-key-pair.XXX")"
+# shellcheck disable=2064
+trap "rm -f -- '$ssh_keypair' '$ssh_keypair.pub'" EXIT
+
+if aws ssm get-parameter --name "$KEYPAIR_PARAMETER" &>/dev/null; then
+ : SSM Secure Parameter exists
+else
+ logger -t INFO "creating SSM parameter ssh key-pair"
+ mkfifo "$ssh_keypair" "${ssh_keypair}.pub"
+ yes | ssh-keygen -P '' -C "$KEYPAIR_NAME" -f "$ssh_keypair" &
+ aws ssm put-parameter --overwrite --name "$KEYPAIR_PARAMETER" --type SecureString --value "$(<"$ssh_keypair")" >/dev/null
+fi
+
+if aws ec2 describe-key-pairs --key-name "$KEYPAIR_NAME" &>/dev/null ; then
+ : EC2 Key Pair exists
+else
+ if [ -e "${ssh_keypair}.pub" ]; then
+ logger -t INFO "importing ssh key from newly generated pair"
+ else
+ logger -t INFO "importing ssh key from SSM parameter '$KEYPAIR_PARAMETER'"
+
+ ssh-keygen -y -f <(aws ssm get-parameter --name "$KEYPAIR_PARAMETER" \
+ --with-decryption \
+ --query Parameter.Value \
+ --output text) \
+ > "${ssh_keypair}.pub"
+ # shellcheck disable=SC2181
+ if [ "$?" -ne 0 ]; then
+ logger -t ERROR "could not import ssh key from SSM parameter"
+ exit 1
+ fi
+ fi
+
+
+ aws ec2 import-key-pair --key-name "$KEYPAIR_NAME" --public-key-material file://"${ssh_keypair}.pub"
+ logger -t INFO "imported ssh key as EC2 key pair '$KEYPAIR_NAME'"
+fi