ci: add doc strings and clarify conditionals

tools/infra/container/Dockerfile.builder

@@ -1,3 +1,12 @@
+# Dockerfile.builder - Base build environment container image
+#
+# The builder image provides an environment in which packages and images may be
+# built. This includes the necessary compilers, libraries, services, and
+# executable dependencies used in the course of the build process.
+#
+# Facilitating scripts may be found in the ./runtime and ./scripts directory
+# where scripts are generally participants in the build of the environment.
+#
 FROM amazonlinux:2 as base
 RUN yum update -y \
  && yum groupinstall -y 'Development Tools' \

tools/infra/container/runtime/bin/start-build-environment

@@ -1,5 +1,17 @@
 #!/usr/bin/env bash
+#
+# start-build-environment - Setup the build environment & start supporting services
+#
+# Intended primarily for use in the builder container image, the
+# start-build-environment provides a single call setup for the common use cases
+# in the context of builds.
+#
+# usage:
+#
+# start-build-environment
+#
 set -xe
+# Ensure we have a logging facility (or stubbed service)
 start-log-listener
-logger -s -t INFO "starting docker daemon"
+# Ensure the docker daemon is running.
 start-docker-daemon

tools/infra/container/runtime/bin/start-docker-daemon

@@ -1,23 +1,80 @@
 #!/usr/bin/env bash
-set -e
+#
+# start-docker-daemon - Start up a docker daemon for the build environment
+#
+# Builds depend on a series of docker images and containers used to run builds
+# for various stages of the system. In the CodeBuild task, the docker daemon
+# must be started as part of the environment startup (in the buildspec) as the
+# CodeBuild Agent does not execute an image's ENTRYPOINT.
+#
+# usage:
+#
+# start-docker-daemon
+#
 
+if ! hash docker dockerd; then
+ logger -s --no-act -t ERROR "missing required commands"
+fi
+
+# If not configured, we'll use the default docker daemon socket address.
+export DOCKER_HOST="${DOCKER_HOST:-unix:///var/run/docker.sock}"
+# Don't startup our own daemon if the environment has access configured for the
+if docker version &>/dev/null; then
+ logger -s --no-act -t INFO "delegating to running & configured dockerd (via $DOCKER_HOST)"
+ exit 0
+fi
+
+# Also check to see if the daemon is running but we're unable to access it, the
+# environment may already have docker daemon running but not fully configured
+# for use.
+if [[ -S "${DOCKER_HOST:+${DOCKER_HOST##unix://}}" ]]; then
+ logger -s --no-act -t ERROR "unable to access running dockerd (via $DOCKER_HOST)"
+ exit 1
+fi
+
+# Verify we're a user that can start the docker daemon (assuming for now that
+# being root means that).
+euid="$(id -u)"
+if [[ "$euid" -ne 0 ]]; then
+ logger -s --no-act -t ERROR "unable to start dockerd as non-root user (euid: $euid != 0)"
+ exit 1
+fi
+
+# In all other cases, start the docker daemon
+logger -s --no-act -t INFO "starting dockerd"
 nohup dockerd \
- --host=unix:///var/run/docker.sock \
+ --host="$DOCKER_HOST" \
  &>/var/log/docker.log &
 daemon_pid="$!"
+
 sleep 1
+
 if [[ ! -e "/proc/$daemon_pid" ]]; then
- echo "ERROR: docker daemon did not start up"
+ logger -s --no-act -t ERROR "dockerd did not start"
  exit 1
 fi
 
-tries=0
-max_tries=5
+# Starting up the daemon asynchronously may take a moment or may fail to start
+# in the background, tries are made to check in on the brief time between exec
+# and the daemon ready to work.
+#
+# sleep interval before attempting each try
+try_interval="0.1s"
+# maximum try attempts that will be made
+try_max_count=3
+# try attempts
+try_count=0
+
 until docker info &>/dev/null; do
- ((tries++))
- if [[ "$tries" -gt "$max_tries" ]]; then
- echo "ERROR: unable to start dockerd for build" >&2
+ ((try_count++))
+ if [[ "$try_count" -gt "$try_max_count" ]]; then
+ logger -s --no-act -t ERROR "dockerd start exceeded deadline (may be slow or failed to start, check logs)"
+ if [[ -s "/var/log/docker.log" ]]; then
+ sed 's/^/docker.log: /' /var/log/docker.log
+ else
+ logger -s --no-act -t WARN "dockerd logs are empty"
+ fi
  exit 1
  fi
- sleep 1
+ sleep "$try_interval"
 done

tools/infra/container/runtime/bin/start-log-listener

@@ -1,13 +1,55 @@
 #!/usr/bin/env bash
-hash socat
-# Listen on syslog domain socket to make for cleaner output.
-nohup socat UNIX-LISTEN:/dev/log,fork - &>/dev/null &
-tries=0
-until test -S /dev/log; do
- ((tries++))
- if [[ "$tries" -gt 3 ]]; then
- echo "ERROR: unable to start log listener" >&2
+#
+# start-log-listener - Setup a socket at /dev/log when missing
+#
+# The kernel's /dev/log socket may not be present in some environments so this
+# script stands up a blackhole listener, to avoid logger from complaining and
+# not outputting our messages.
+#
+# usage:
+#
+# start-log-listener
+#
+if ! hash socat; then
+ echo "ERROR: missing required commands"
+fi
+
+log_sock="/dev/log"
+
+# Starting up the socket asynchronously may take a moment or may fail to start
+# in the background, tries are made to check in on the brief time between exec
+# and the listener bound.
+#
+# sleep interval before attempting each try
+try_interval="0.1s"
+# maximum try attempts that will be made
+try_max_count=3
+
+# Let the existing socket be.
+if [[ -S "$log_sock" ]]; then
+ logger -s --no-act -t INFO -- "kernel's log socket is already present /dev/log"
+ logger -s --no-act -t INFO -- "not replacing /dev/log"
+ exit 0
+fi
+
+# Require EUID 0 as we're mucking about in /dev
+euid="$(id -u)"
+if [[ "$euid" -ne 0 ]]; then
+ echo "ERROR: unable to start log listener socket as non-root user (euid: $euid != 0)"
+ exit 1
+fi
+
+# Listen on syslog unix socket and dump any sent datagrams.
+nohup socat "UNIX-LISTEN:$log_sock,fork" - &>/dev/null &
+
+# Wait for the listener to start and assume the process didn't successfully bind
+# to the socket otherwise.
+try_count=0
+until [[ -S "$log_sock" ]]; do
+ ((try_count++))
+ if [[ "$try_count" -gt "$try_max_count" ]]; then
+ echo "ERROR: unable to start log listener at $log_sock" >&2
  exit 1
  fi
- sleep 0.1s
+ sleep "$try_interval"
 done