Merge pull request #633 from amazonlinux/ci-containers

Build & CI - Containerized Environments

Makefile.toml

@@ -57,6 +57,8 @@ script = [
 '''
 set -o pipefail
 if ! docker image inspect ${BUILDSYS_SDK_IMAGE} >/dev/null 2>&1 ; then
+  # Let curl resolve the certificates instead of the tasks resolved bundle.
+  unset SSL_CERT_FILE SSL_CERT_DIR
   if ! curl https://thar-upstream-lookaside-cache.s3.us-west-2.amazonaws.com/${BUILDSYS_SDK_IMAGE}.tar.gz \
        | gunzip | docker load ; then
     echo "failed to load '${BUILDSYS_SDK_IMAGE}'" >&2

bin/amiize.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Register partitioned root and data images as an AMI in EC2.
 # Only registers with HVM virtualization type and GP2 EBS volume type.
@@ -106,6 +106,7 @@ $(basename "${0}")
                  [ --root-volume-size 1234 ]
                  [ --data-volume-size 5678 ]
                  [ --security-group-name default | --security-group-id sg-abcdef1234 ]
+                 [ --write-output-dir output-dir ]
 
 Registers the given images as an AMI in the given EC2 region.
 
@@ -130,6 +131,9 @@ Optional:
    --security-group-id        The ID of a security group name that allows SSH access from this host
    --security-group-name      The name of a security group name that allows SSH access from this host
                               (defaults to "default" if neither name nor ID are specified)
+   --write-output-dir         The directory to write out IDs into attribute named files.
+                              (not written out to anywhere other than log otherwise)
+
 EOF
 }
 
@@ -161,6 +165,7 @@ parse_args() {
          --data-volume-size ) shift; DATA_VOLUME_SIZE="${1}" ;;
          --security-group-name ) shift; SECURITY_GROUP_NAME="${1}" ;;
          --security-group-id ) shift; SECURITY_GROUP_ID="${1}" ;;
+         --write-output-dir ) shift; WRITE_OUTPUT_DIR="${1}" ;;
 
          --help ) usage; exit 0 ;;
          *)
@@ -323,6 +328,19 @@ check_return() {
    return 0
 }
 
+# Helper to conditionally write out attribute if WRITE_OUTPUT_DIR is
+# configured.
+write_output() {
+    local name="$1"
+    local value="$2"
+
+    if [[ -z "${WRITE_OUTPUT_DIR}" ]]; then
+        return
+    fi
+
+    mkdir -p "${WRITE_OUTPUT_DIR}/$(dirname "$name")"
+    echo -n "$value" > "${WRITE_OUTPUT_DIR}/${name}"
+}
 
 # =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=
 
@@ -623,6 +641,7 @@ while true; do
       echo "* Warning: Could not delete root volume!"
       # Don't die though, we got what we want...
    fi
+   write_output "root_snapshot_id" "$root_snapshot"
 
    if aws ec2 delete-volume \
       --output text \
@@ -635,6 +654,7 @@ while true; do
       echo "* Warning: Could not delete data volume!"
       # Don't die though, we got what we want...
    fi
+   write_output "data_snapshot_id" "$data_snapshot"
 
    # =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=
 
@@ -657,6 +677,8 @@ while true; do
    check_return ${?} "AMI registration failed!" || continue
    echo "Registered ${registered_ami}"
 
+   write_output "ami_id" "$registered_ami"
+
    echo "Waiting for the AMI to appear in a describe query"
    waits=0
    while [ ${waits} -lt 20 ]; do

bin/upload-sources

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -euo pipefail
 

tools/docker-go

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Helper script for running commands in a golang build/runtime environment for testing/vendoring/building a go module
 

tools/gen-docs.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 DOCS=(START.md README.md INSTALL.md CHANGELOG.md extras/dogswatch/README.md)
 EXTRAS=(extras/dogswatch/{dogswatch,dev/deployment}.yaml)
 

tools/infra/buildspec/infra-pr-build.yml

@@ -8,4 +8,4 @@ env:
 phases:
   build:
     commands:
-      - make -C "$INFRA_DIR/stacks" --keep-going validate
+      - make -C "$INFRA_DIR/stacks" --keep-going validate check

tools/infra/buildspec/thar-develop-pipeline-test.yml

@@ -1,21 +1,9 @@
 version: 0.2
 
-env:
-  variables:
-    # Path to infra tooling directory.
-    INFRA_DIR: "./tools/infra"
-
 phases:
-  install:
-    runtime-versions:
-      docker: 18
-    commands:
-      - . "${INFRA_DIR}/env/lib/environment-setup"
-      - . setup-rust-builder
-      # TODO: rely on libsystemd in build container instead, see #545
-      - apt install -y libsystemd-dev
   pre_build:
     commands:
+      - start-build-environment
       - environment-report
       - write-build-meta
   build:

tools/infra/buildspec/thar-pr-build.yml

@@ -1,31 +1,21 @@
 version: 0.2
 
-env:
-  variables:
-    # Path to infra tooling directory.
-    INFRA_DIR: "./tools/infra"
-
 phases:
-  install:
-    runtime-versions:
-      docker: 18
-    commands:
-      - . "${INFRA_DIR}/env/lib/environment-setup"
-      - . setup-rust-builder
   pre_build:
     commands:
+      - start-build-environment
       - environment-report
       - write-build-meta
   build:
     commands:
-      # Retry fetches a few times before failing
-      - retry 2 'fetch dependencies' -- cargo make fetch
       - cargo make world
 
 artifacts:
   base-directory: 'build/'
   files:
     - '*.img*'
+    - '*.ext4*'
+    - '*.verity*'
   secondary-artifacts:
     meta:
       base-directory: 'build/meta'

tools/infra/container/.dockerignore

@@ -0,0 +1 @@
+Makefile

tools/infra/container/Dockerfile.builder

@@ -0,0 +1,36 @@
+# Dockerfile.builder - Base build environment container image
+#
+# The builder image provides an environment in which packages and images may be
+# built. This includes the necessary compilers, libraries, services, and
+# executable dependencies used in the course of the build process.
+#
+# Facilitating scripts may be found in the ./runtime and ./scripts directory
+# where scripts are generally participants in the build of the environment.
+#
+FROM amazonlinux:2 as base
+RUN yum update -y \
+	&& yum groupinstall -y 'Development Tools' \
+	&& yum install -y socat procps-ng awscli jq openssh rsync systemd-devel \
+	&& amazon-linux-extras enable docker \
+	&& yum install -y docker amazon-ecr-credential-helper \
+	&& yum clean all \
+	&& rm -rf /var/cache/yum /var/cache/amzn2extras
+RUN install -D /dev/null /root/.docker/config.json \
+	&& echo '{ "credsStore": "ecr-login" }' >> /root/.docker/config.json
+
+FROM base
+ENV PATH="$PATH:/build/runtime/bin:/build/scripts:/build/.cargo/bin"
+ENV CARGO_HOME="/build/.cargo"
+ENV RUNTIME_SCRIPT_LIB="/build/runtime/lib"
+
+COPY scripts /build/scripts
+COPY runtime /build/runtime
+WORKDIR /build
+
+RUN install-rust && configure-rust && install-crates
+
+COPY builder/entrypoint.sh /build/entrypoint.sh
+
+ENTRYPOINT ["/build/entrypoint.sh"]
+
+CMD [ "bash" ]

tools/infra/container/Makefile

@@ -0,0 +1,50 @@
+# SHELL is bash, silly sh.
+SHELL = bash
+# DOCKERFILES are the detected container images that are being worked
+# with. It is expected that NAME be part of the file name, as in
+# Dockerfile.NAME, which is used throughout the infrastructure.
+DOCKERFILES = $(filter-out %~,$(wildcard Dockerfile.*))
+# NAMES are the detected NAMES given the provided Dockerfiles.
+NAMES = $(DOCKERFILES:Dockerfile.%=%)
+# IMAGE_REPO_PREFIX is prepended to the image's tag. In the case of
+# `push', the IMAGE_REPO_PREFIX provides the ECR repository URI prefix
+# for each image.
+IMAGE_REPO_PREFIX ?= infra/
+# IMAGE_TAG provides the registry/image-name:IMAGE_TAG portion of the
+# URI tagged to images.
+IMAGE_TAG ?= develop
+# IMAGE_NAME is the name that the container image is tagged with.
+IMAGE_NAME ?= $(IMAGE_REPO_PREFIX)$(NAME):$(IMAGE_TAG)
+# ECR_URI_PREFIX is the ECR URI prefix based on the resolved builder
+# image URI which, like other container images, is discoverable under
+# its in-region SSM parameter - so we can lob off the builder part and
+# use it as our model for the pushed repository name.
+ECR_URI_PREFIX = $(shell aws ssm get-parameter --name /infra/container/infra/builder --query Parameter.Value --output text | sed 's/builder$$//')
+# ECR_NAME_PREFIX provides a prefix to derive the ECR repository-name
+# (the attribute) from the images' NAME - the infra/ prefix is
+# conventional across automations' consumed images.
+ECR_NAME_PREFIX ?= infra/
+
+.DEFAULT: all
+.PHONY: force all release $(NAMES)
+force:
+
+all: $(if $(NAME),$(NAME),$(NAMES))
+
+$(NAMES) : NAME = $@
+$(NAMES): force
+	@echo "Building container image for '$(NAME)'"
+	docker build -t $(IMAGE_NAME) -f Dockerfile.$(NAME) .
+
+# Push images (must explicitly provide IMAGE_TAG=release to be pulled
+# by consumers).
+push: IMAGE_REPO_PREFIX = $(ECR_URI_PREFIX)
+push: IMAGE_TAG = staging
+push: all
+	@echo "Pushing container images with tag '$(IMAGE_TAG)'"
+	@echo "Images: $(foreach NAME,$(NAMES),$(IMAGE_NAME))"
+	@$(foreach NAME,$(NAMES),\
+		echo "Pushing '$(NAME)' to '$(IMAGE_NAME)'" && \
+		aws ecr describe-repositories --repository-names $(ECR_NAME_PREFIX)$(NAME) &> /dev/null \
+		&& docker push $(IMAGE_NAME) \
+		|| echo "Could not push $(NAME) to ECR repository as $(IMAGE_NAME)";)

tools/infra/container/README.md

@@ -0,0 +1,61 @@
+# Container Environments
+
+Container images, defined in this directory, provide environments for infra's build and automation needs.
+
+## Images
+
+Each image is defined in their own `Dockerfile` and suffixed with its name. For example the `builder` container - used in CI builds - is defined by `Dockerfile.builder`. 
+The containers copy in common resources and others as needed from this shared root context.
+
+**`builder` image**
+
+The `builder` image provides an environment in which packages and images may be built.
+`builder`'s container image is created with all required dependencies used by the build driver, `buildsys`, and the supporting tools & scripts used by it (including many of the `cargo-make` tasks' dependencies).
+
+# Building
+
+## Development Images
+
+To all build images locally, a single `make` call can be made:
+
+```bash
+make all
+```
+
+Each `Dockerfile.<name>` can be built individually with `make $name` as needed.
+
+## Release Images (using a tag)
+
+As with the development images, all images may be built at once:
+
+```bash
+make all IMAGE_TAG=release
+```
+
+To build a specific image, for instance named `builder`, `make` may be provided this name to build its release image:
+
+```bash
+make all NAME=builder IMAGE_TAG=release
+```
+
+# Releasing
+
+The `push` target is provided to build & push release container images for use, at least in the context of build and release automation.
+
+The default target will prepare to push the images using the environment's AWS profile to confirm that the ECR repositories line up and subsequently pushing with a default of `IMAGE_TAG=staging`.
+This invocation **will** push to the ECR repository, but with the image tagged as "staging".
+Doing a push this way will stage the layers in the ECR repository so that subsequent pushes update lightweight references only (pushing a tag that refers to the same layers).
+
+``` bash
+make push
+```
+
+To push a container image tagged as a release image, which is required for the CodeBuild project to use, the `IMAGE_TAG` must be set explicitly to the same tag that's configured to be pulled by projects.
+If the release tag is `release`, then the call to `push` these images would be:
+
+``` bash
+make push IMAGE_TAG=release
+```
+
+The `Makefile` target would then match the images to their respective ECR repositories, as before, and `docker push` to the images' respective repositories.
+If the `make push IMAGE_TAG=release` followed an earlier `make push` then this the `make push IMAGE_TAG=release` call will simply update the references in the remote ECR repository to point to the same layers.

tools/infra/container/builder/entrypoint.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -e
+start-build-environment
+exec -- "$@"