diff --git a/Makefile.toml b/Makefile.toml index f7b5ce06082..a536e6cc8d3 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -57,6 +57,8 @@ script = [ ''' set -o pipefail if ! docker image inspect ${BUILDSYS_SDK_IMAGE} >/dev/null 2>&1 ; then + # Let curl resolve the certificates instead of the tasks resolved bundle. + unset SSL_CERT_FILE SSL_CERT_DIR if ! curl https://thar-upstream-lookaside-cache.s3.us-west-2.amazonaws.com/${BUILDSYS_SDK_IMAGE}.tar.gz \ | gunzip | docker load ; then echo "failed to load '${BUILDSYS_SDK_IMAGE}'" >&2 diff --git a/bin/amiize.sh b/bin/amiize.sh index d69b19a55fd..89ad9b69a73 100755 --- a/bin/amiize.sh +++ b/bin/amiize.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Register partitioned root and data images as an AMI in EC2. # Only registers with HVM virtualization type and GP2 EBS volume type. diff --git a/bin/upload-sources b/bin/upload-sources index cb25185577d..e7bbf21d708 100755 --- a/bin/upload-sources +++ b/bin/upload-sources @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail diff --git a/tools/docker-go b/tools/docker-go index 25c52005a1a..15516443fc4 100755 --- a/tools/docker-go +++ b/tools/docker-go @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Helper script for running commands in a golang build/runtime environment for testing/vendoring/building a go module diff --git a/tools/gen-docs.sh b/tools/gen-docs.sh index 2ae32cea42f..a83dd20c858 100755 --- a/tools/gen-docs.sh +++ b/tools/gen-docs.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash DOCS=(START.md README.md INSTALL.md CHANGELOG.md extras/dogswatch/README.md) EXTRAS=(extras/dogswatch/{dogswatch,dev/deployment}.yaml) diff --git a/tools/infra/buildspec/thar-pr-build.yml b/tools/infra/buildspec/thar-pr-build.yml index 2c793d43bcf..2cfc7e5937f 100644 --- a/tools/infra/buildspec/thar-pr-build.yml +++ b/tools/infra/buildspec/thar-pr-build.yml @@ -6,20 +6,13 @@ env: INFRA_DIR: "./tools/infra" phases: - install: - runtime-versions: - docker: 18 - commands: - - . "${INFRA_DIR}/env/lib/environment-setup" - - . setup-rust-builder pre_build: commands: + - start-build-environment - environment-report - write-build-meta build: commands: - # Retry fetches a few times before failing - - retry 2 'fetch dependencies' -- cargo make fetch - cargo make world artifacts: diff --git a/tools/infra/container/.dockerignore b/tools/infra/container/.dockerignore new file mode 100644 index 00000000000..f3c7a7c5da6 --- /dev/null +++ b/tools/infra/container/.dockerignore @@ -0,0 +1 @@ +Makefile diff --git a/tools/infra/container/Dockerfile.builder b/tools/infra/container/Dockerfile.builder new file mode 100644 index 00000000000..56eb9dec4d3 --- /dev/null +++ b/tools/infra/container/Dockerfile.builder @@ -0,0 +1,33 @@ +# Dockerfile.builder - Base build environment container image +# +# The builder image provides an environment in which packages and images may be +# built. This includes the necessary compilers, libraries, services, and +# executable dependencies used in the course of the build process. +# +# Facilitating scripts may be found in the ./runtime and ./scripts directory +# where scripts are generally participants in the build of the environment. +# +FROM amazonlinux:2 as base +RUN yum update -y \ + && yum groupinstall -y 'Development Tools' \ + && yum install -y socat procps-ng awscli jq openssh rsync \ + && amazon-linux-extras install -y docker \ + && yum clean all \ + && rm -rf /var/cache/yum /var/cache/amzn2extras + +FROM base +ENV PATH="$PATH:/build/runtime/bin:/build/scripts:/build/.cargo/bin" +ENV CARGO_HOME="/build/.cargo" +ENV RUNTIME_SCRIPT_LIB="/build/runtime/lib" + +COPY scripts /build/scripts +COPY runtime /build/runtime +WORKDIR /build + +RUN install-rust && configure-rust && install-crates + +COPY builder/entrypoint.sh /build/entrypoint.sh + +ENTRYPOINT ["/build/entrypoint.sh"] + +CMD [ "bash" ] diff --git a/tools/infra/container/Makefile b/tools/infra/container/Makefile new file mode 100644 index 00000000000..8274d2d6c75 --- /dev/null +++ b/tools/infra/container/Makefile @@ -0,0 +1,20 @@ +DOCKERFILES = $(filter-out %~,$(wildcard Dockerfile.*)) +NAMES = $(DOCKERFILES:Dockerfile.%=%) + +IMAGE_REPO_PREFIX ?= infra/ +IMAGE_NAME ?= $(IMAGE_REPO_PREFIX)$(NAME):$(IMAGE_TAG) + +.PHONY: force all release $(NAMES) +.DEFAULT: all +force: + +all : IMAGE_TAG ?= develop +all: $(NAMES) + +release : IMAGE_TAG ?= latest +release: $(if $(NAME),$(NAME),$(NAMES)) + +$(NAMES) : NAME = $@ +$(NAMES): force + @echo "Building container image for '$(NAME)'" + docker build -t $(IMAGE_NAME) -f Dockerfile.$(NAME) . diff --git a/tools/infra/container/README.md b/tools/infra/container/README.md new file mode 100644 index 00000000000..13383be6156 --- /dev/null +++ b/tools/infra/container/README.md @@ -0,0 +1,39 @@ +# Container Environments + +Container images, defined in this directory, provide environments for infra's build and automation needs. + +## Images + +Each image is defined in their own `Dockerfile` and suffixed with its name. For example the `builder` container - used in CI builds - is defined by `Dockerfile.builder`. +The containers copy in common resources and others as needed from this shared root context. + +**`builder` image** + +The `builder` image provides an environment in which packages and images may be built. +`builder`'s container image is created with all required dependencies used by the build driver, `buildsys`, and the supporting tools & scripts used by it (including many of the `cargo-make` tasks' dependencies). + +# Building + +## Development Images + +To all build images locally, a single `make` call can be made: + +```bash +make all +``` + +Each `Dockerfile.` can be built individually with `make $name` as needed. + +## Release Images + +As with the development images, all images may be built at once: + +```bash +make release +``` + +To build a specific image, for instance named `builder`, `make` may be provided this name to build its release image: + +```bash +make release NAME=builder +``` diff --git a/tools/infra/container/builder/entrypoint.sh b/tools/infra/container/builder/entrypoint.sh new file mode 100755 index 00000000000..6c31cdf3d3d --- /dev/null +++ b/tools/infra/container/builder/entrypoint.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -e +start-build-environment +exec -- "$@" diff --git a/tools/infra/env/bin/environment-report b/tools/infra/container/runtime/bin/environment-report similarity index 100% rename from tools/infra/env/bin/environment-report rename to tools/infra/container/runtime/bin/environment-report diff --git a/tools/infra/env/bin/retry b/tools/infra/container/runtime/bin/retry similarity index 100% rename from tools/infra/env/bin/retry rename to tools/infra/container/runtime/bin/retry diff --git a/tools/infra/container/runtime/bin/start-build-environment b/tools/infra/container/runtime/bin/start-build-environment new file mode 100755 index 00000000000..99f2f1f5a3a --- /dev/null +++ b/tools/infra/container/runtime/bin/start-build-environment @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# +# start-build-environment - Setup the build environment & start supporting services +# +# Intended primarily for use in the builder container image, the +# start-build-environment provides a single call setup for the common use cases +# in the context of builds. +# +# usage: +# +# start-build-environment +# +set -xe +# Ensure we have a logging facility (or stubbed service) +start-log-listener +# Ensure the docker daemon is running. +start-docker-daemon diff --git a/tools/infra/container/runtime/bin/start-docker-daemon b/tools/infra/container/runtime/bin/start-docker-daemon new file mode 100755 index 00000000000..0a95438e6fb --- /dev/null +++ b/tools/infra/container/runtime/bin/start-docker-daemon @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# +# start-docker-daemon - Start up a docker daemon for the build environment +# +# Builds depend on a series of docker images and containers used to run builds +# for various stages of the system. In the CodeBuild task, the docker daemon +# must be started as part of the environment startup (in the buildspec) as the +# CodeBuild Agent does not execute an image's ENTRYPOINT. +# +# usage: +# +# start-docker-daemon +# + +if ! hash docker dockerd; then + logger -s --no-act -t ERROR "missing required commands" +fi + +# If not configured, we'll use the default docker daemon socket address. +export DOCKER_HOST="${DOCKER_HOST:-unix:///var/run/docker.sock}" +# Don't startup our own daemon if the environment has access configured for the +if docker version &>/dev/null; then + logger -s --no-act -t INFO "delegating to running & configured dockerd (via $DOCKER_HOST)" + exit 0 +fi + +# Also check to see if the daemon is running but we're unable to access it, the +# environment may already have docker daemon running but not fully configured +# for use. +if [[ -S "${DOCKER_HOST:+${DOCKER_HOST##unix://}}" ]]; then + logger -s --no-act -t ERROR "unable to access running dockerd (via $DOCKER_HOST)" + exit 1 +fi + +# Verify we're a user that can start the docker daemon (assuming for now that +# being root means that). +euid="$(id -u)" +if [[ "$euid" -ne 0 ]]; then + logger -s --no-act -t ERROR "unable to start dockerd as non-root user (euid: $euid != 0)" + exit 1 +fi + +# In all other cases, start the docker daemon +logger -s --no-act -t INFO "starting dockerd" +nohup dockerd \ + --host="$DOCKER_HOST" \ + &>/var/log/docker.log & +daemon_pid="$!" + +sleep 1 + +if [[ ! -e "/proc/$daemon_pid" ]]; then + logger -s --no-act -t ERROR "dockerd did not start" + exit 1 +fi + +# Starting up the daemon asynchronously may take a moment or may fail to start +# in the background, tries are made to check in on the brief time between exec +# and the daemon ready to work. +# +# sleep interval before attempting each try +try_interval="0.1s" +# maximum try attempts that will be made +try_max_count=3 +# try attempts +try_count=0 + +until docker info &>/dev/null; do + ((try_count++)) + if [[ "$try_count" -gt "$try_max_count" ]]; then + logger -s --no-act -t ERROR "dockerd start exceeded deadline (may be slow or failed to start, check logs)" + if [[ -s "/var/log/docker.log" ]]; then + sed 's/^/docker.log: /' /var/log/docker.log + else + logger -s --no-act -t WARN "dockerd logs are empty" + fi + exit 1 + fi + sleep "$try_interval" +done diff --git a/tools/infra/container/runtime/bin/start-log-listener b/tools/infra/container/runtime/bin/start-log-listener new file mode 100755 index 00000000000..ee8e85a1055 --- /dev/null +++ b/tools/infra/container/runtime/bin/start-log-listener @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# +# start-log-listener - Setup a socket at /dev/log when missing +# +# The kernel's /dev/log socket may not be present in some environments so this +# script stands up a blackhole listener, to avoid logger from complaining and +# not outputting our messages. +# +# usage: +# +# start-log-listener +# +if ! hash socat; then + echo "ERROR: missing required commands" +fi + +log_sock="/dev/log" + +# Starting up the socket asynchronously may take a moment or may fail to start +# in the background, tries are made to check in on the brief time between exec +# and the listener bound. +# +# sleep interval before attempting each try +try_interval="0.1s" +# maximum try attempts that will be made +try_max_count=3 + +# Let the existing socket be. +if [[ -S "$log_sock" ]]; then + logger -s --no-act -t INFO -- "kernel's log socket is already present /dev/log" + logger -s --no-act -t INFO -- "not replacing /dev/log" + exit 0 +fi + +# Require EUID 0 as we're mucking about in /dev +euid="$(id -u)" +if [[ "$euid" -ne 0 ]]; then + echo "ERROR: unable to start log listener socket as non-root user (euid: $euid != 0)" + exit 1 +fi + +# Listen on syslog unix socket and dump any sent datagrams. +nohup socat "UNIX-LISTEN:$log_sock,fork" - &>/dev/null & + +# Wait for the listener to start and assume the process didn't successfully bind +# to the socket otherwise. +try_count=0 +until [[ -S "$log_sock" ]]; do + ((try_count++)) + if [[ "$try_count" -gt "$try_max_count" ]]; then + echo "ERROR: unable to start log listener at $log_sock" >&2 + exit 1 + fi + sleep "$try_interval" +done diff --git a/tools/infra/env/bin/write-build-meta b/tools/infra/container/runtime/bin/write-build-meta similarity index 96% rename from tools/infra/env/bin/write-build-meta rename to tools/infra/container/runtime/bin/write-build-meta index 8636072b919..1e15292271d 100755 --- a/tools/infra/env/bin/write-build-meta +++ b/tools/infra/container/runtime/bin/write-build-meta @@ -31,7 +31,7 @@ fi write_common -if [[ -z "${CODEBUILD_BUILD_ID}" ]]; then +if [[ -n "${CODEBUILD_BUILD_ID}" ]]; then write_codebuild fi diff --git a/tools/infra/container/runtime/lib/lib.bash b/tools/infra/container/runtime/lib/lib.bash new file mode 100644 index 00000000000..431ae1bc3fe --- /dev/null +++ b/tools/infra/container/runtime/lib/lib.bash @@ -0,0 +1,6 @@ +# shellcheck shell=bash + +# Logger provides the corrected interface to log to stderr. +logger() { + command logger --no-act -s "$@" +} diff --git a/tools/infra/env/bin/setup-rust-builder b/tools/infra/container/scripts/configure-rust old mode 100644 new mode 100755 similarity index 50% rename from tools/infra/env/bin/setup-rust-builder rename to tools/infra/container/scripts/configure-rust index f242e7885b6..38e9631043e --- a/tools/infra/env/bin/setup-rust-builder +++ b/tools/infra/container/scripts/configure-rust @@ -1,15 +1,15 @@ -#!/usr/bin/env sh +#!/usr/bin/env bash # -# setup-rust-builder - install rust and configure shell for its use by a builder -# -# This script is expected to be sourced by callers in order to affect their PATH -# directly AND is used in CodeBuild environments where the SHELL is /bin/sh. +# confgure-rust - configure container environment for rust usage # # usage: # -# . setup-rust-builder +# configure-rust # +# shellcheck source=../runtime/lib/lib.bash +source "$RUNTIME_SCRIPT_LIB/lib.bash" + configure_color_output() { # Prepare Cargo for build and CI usage PATH="$HOME/.cargo/bin:$PATH" @@ -42,37 +42,5 @@ configure_color_output() { unset cargo_make_config } -cargo_dep() { - cargo_install_package="$@" - logger -s -t INFO "installing cargo dep with 'cargo install $cargo_install_package'" - if ! cargo install --force $cargo_install_package; then - logger -s -t ERROR "failed to install dep with 'cargo install $cargo_install_package'" - exit 1 - fi -} - -install_rust_toolchain() { - test -f rustup-init.sh || rm -f rustup-init.sh - if ! curl -o rustup-init.sh --proto '=https' --tlsv1.2 -sS "https://sh.rustup.rs"; then - logger -s -t ERROR "could not fetch rustup, needed for managing rust toolchain" - exit 1 - fi - if ! bash rustup-init.sh -y --no-modify-path --default-toolchain stable ; then - logger -s -t ERROR "could not setup rustup & rust toolchain for build" - exit 1 - fi - rm -f rustup-init.sh -} - configure_color_output -install_rust_toolchain - -# Install build tooling dependencies -cargo_dep --version 0.23.0 cargo-make -cargo_dep --version 0.2.6 cargo-deny - -unset -f configure_color_output -unset -f install_rust_toolchain -unset -f cargo_dep - diff --git a/tools/infra/container/scripts/install-crates b/tools/infra/container/scripts/install-crates new file mode 100755 index 00000000000..145460bc366 --- /dev/null +++ b/tools/infra/container/scripts/install-crates @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# +# install-crates - install binary crate dependencies +# +# usage: +# +# install-crates +# + +# shellcheck source=../runtime/lib/lib.bash +source "$RUNTIME_SCRIPT_LIB/lib.bash" + +cargo_dep() { + local cargo_install_package=( "$@" ) + logger -s -t INFO "installing cargo dep with 'cargo install ${cargo_install_package[*]}'" + if ! cargo install --force "${cargo_install_package[@]}"; then + logger -s -t ERROR "failed to install dep with 'cargo install ${cargo_install_package[*]}'" + exit 1 + fi +} + +# Install build tooling dependencies +cargo_dep --version 0.23.0 cargo-make +cargo_dep --version 0.2.6 cargo-deny + diff --git a/tools/infra/container/scripts/install-rust b/tools/infra/container/scripts/install-rust new file mode 100755 index 00000000000..e056fd08ff1 --- /dev/null +++ b/tools/infra/container/scripts/install-rust @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# +# install-rust - install rust for use in builder containers +# +# usage: +# +# install-rust +# + +# shellcheck source=../runtime/lib/lib.bash +source "$RUNTIME_SCRIPT_LIB/lib.bash" + +install_rust_toolchain() { + local toolchain="${1:-stable}" + test -f rustup-init.sh || rm -f rustup-init.sh + if ! curl -o rustup-init.sh --proto '=https' --tlsv1.2 -sS "https://sh.rustup.rs"; then + logger -s -t ERROR "could not fetch rustup, needed for managing rust toolchain" + exit 1 + fi + if ! bash rustup-init.sh -y --profile minimal --no-modify-path --default-toolchain "$toolchain" ; then + logger -s -t ERROR "could not setup rustup & rust toolchain for build" + exit 1 + fi + rm -f rustup-init.sh +} + +install_rust_toolchain "stable" diff --git a/tools/infra/env/README.md b/tools/infra/env/README.md deleted file mode 100644 index 97189cf3585..00000000000 --- a/tools/infra/env/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# CI Builder Environment - -This directory provides the working environment within the context of a CI build. -The `lib/` directory contains bootstrap and support resources that are not otherwise not directly executed. -The `bin/` directory contains scripts and executables that are directly used in the build process. -Scripts named `setup-$NAME` are intended to be sourced into the build environment; scripts sourced into the build environment must be compatible with the bourne shell - `/bin/sh`. - diff --git a/tools/infra/env/lib/environment-setup b/tools/infra/env/lib/environment-setup deleted file mode 100644 index 5397068b3d3..00000000000 --- a/tools/infra/env/lib/environment-setup +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env sh -# -# environment-setup - setup shell for building in -# -if [ -d "${INFRA_DIR:-./tools/infra}/env/bin" ]; then - PATH="$(pwd -P)/tools/infra/env/bin:$PATH" -fi - -for src_path in $(env | awk -F'=' '/^CODEBUILD_SRC_DIR/ { print $2; }'); do - for src_bin in "$src_path/bin" "$src_path/ci/bin"; do - if [ -d "$src_bin" ]; then - PATH="$src_bin:$PATH" - fi - done -done diff --git a/tools/rpm2img b/tools/rpm2img index 3c92cbb69b6..65411776d0b 100755 --- a/tools/rpm2img +++ b/tools/rpm2img @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eu -o pipefail shopt -qs failglob