Merge pull request #330 from solvedata/issue-327/upload-logs-when-dep…

…endencies-fail-to-start Upload logs when dependencies fail to start
buildkite-plugins · Sep 23, 2022 · f555794 · f555794
2 parents 8f02122 + 59bb87d
commit f555794
Show file tree

Hide file tree

Showing 4 changed files with 137 additions and 62 deletions.
diff --git a/README.md b/README.md
@@ -15,7 +15,7 @@ The following pipeline will run `test.sh` inside a `app` service container using
 steps:
   - command: test.sh
     plugins:
-      - docker-compose#v3.10.0:
+      - docker-compose#v3.11.1:
           run: app
 ```
 
@@ -26,7 +26,7 @@ through if you need:
 steps:
   - command: test.sh
     plugins:
-      - docker-compose#v3.10.0:
+      - docker-compose#v3.11.1:
           run: app
           config: docker-compose.tests.yml
           env:
@@ -39,7 +39,7 @@ or multiple config files:
 steps:
   - command: test.sh
     plugins:
-      - docker-compose#v3.10.0:
+      - docker-compose#v3.11.1:
           run: app
           config:
             - docker-compose.yml
@@ -54,7 +54,7 @@ env:
 steps:
   - command: test.sh
     plugins:
-      - docker-compose#v3.10.0:
+      - docker-compose#v3.11.1:
           run: app
 ```
 
@@ -65,15 +65,15 @@ steps:
   - plugins:
       - docker-login#v2.0.1:
           username: xyz
-      - docker-compose#v3.10.0:
+      - docker-compose#v3.11.1:
           build: app
           image-repository: index.docker.io/myorg/myrepo
   - wait
   - command: test.sh
     plugins:
       - docker-login#v2.0.1:
           username: xyz
-      - docker-compose#v3.10.0:
+      - docker-compose#v3.11.1:
           run: app
 ```
 
@@ -82,7 +82,7 @@ If you want to control how your command is passed to docker-compose, you can use
 ```yml
 steps:
   - plugins:
-      - docker-compose#v3.10.0:
+      - docker-compose#v3.11.1:
           run: app
           command: ["custom", "command", "values"]
 ```
@@ -98,7 +98,7 @@ steps:
   - command: generate-dist.sh
     artifact_paths: "dist/*"
     plugins:
-      - docker-compose#v3.10.0:
+      - docker-compose#v3.11.1:
           run: app
 ```
 
@@ -116,7 +116,7 @@ steps:
   - command: generate-dist.sh
     artifact_paths: "dist/*"
     plugins:
-      - docker-compose#v3.10.0:
+      - docker-compose#v3.11.1:
           run: app
           volumes:
             - "./dist:/app/dist"
@@ -137,7 +137,7 @@ this plugin offers a `environment` block of its own:
 steps:
   - command: generate-dist.sh
     plugins:
-      - docker-compose#v3.10.0:
+      - docker-compose#v3.11.1:
           run: app
           env:
             - BUILDKITE_BUILD_NUMBER
@@ -157,7 +157,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th
 steps:
   - command: generate-dist.sh
     plugins:
-      - docker-compose#v3.10.0:
+      - docker-compose#v3.11.1:
           build: app
           image-repository: index.docker.io/myorg/myrepo
           args:
@@ -174,7 +174,7 @@ To speed up run steps that use the same service/image (such as steps that run in
 steps:
   - label: ":docker: Build"
     plugins:
-      - docker-compose#v3.10.0:
+      - docker-compose#v3.11.1:
           build: app
           image-repository: index.docker.io/myorg/myrepo
 
@@ -184,7 +184,7 @@ steps:
     command: test.sh
     parallelism: 25
     plugins:
-      - docker-compose#v3.10.0:
+      - docker-compose#v3.11.1:
           run: app
 ```
 
@@ -200,7 +200,7 @@ steps:
     agents:
       queue: docker-builder
     plugins:
-      - docker-compose#v3.10.0:
+      - docker-compose#v3.11.1:
           build:
             - app
             - tests
@@ -212,7 +212,7 @@ steps:
     command: test.sh
     parallelism: 25
     plugins:
-      - docker-compose#v3.10.0:
+      - docker-compose#v3.11.1:
           run: tests
 ```
 
@@ -224,7 +224,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu
 steps:
   - label: ":docker: Push"
     plugins:
-      - docker-compose#v3.10.0:
+      - docker-compose#v3.11.1:
           push: app
 ```
 
@@ -236,7 +236,7 @@ steps:
     plugins:
       - docker-login#v2.0.1:
           username: xyz
-      - docker-compose#v3.10.0:
+      - docker-compose#v3.11.1:
           push: app
 ```
 
@@ -248,7 +248,7 @@ steps:
     plugins:
       - docker-login#v2.0.1:
           username: xyz
-      - docker-compose#v3.10.0:
+      - docker-compose#v3.11.1:
           push:
             - first-service
             - second-service
@@ -262,7 +262,7 @@ steps:
     plugins:
       - docker-login#v2.0.1:
           username: xyz
-      - docker-compose#v3.10.0:
+      - docker-compose#v3.11.1:
           push:
             - app:index.docker.io/myorg/myrepo/myapp
             - app:index.docker.io/myorg/myrepo/myapp:latest
@@ -276,14 +276,14 @@ A newly spawned agent won't contain any of the docker caches for the first run w
 steps:
   - label: ":docker: Build an image"
     plugins:
-      - docker-compose#v3.10.0:
+      - docker-compose#v3.11.1:
           build: app
           image-repository: index.docker.io/myorg/myrepo
           cache-from: app:index.docker.io/myorg/myrepo/myapp:latest
   - wait
   - label: ":docker: Push to final repository"
     plugins:
-      - docker-compose#v3.10.0:
+      - docker-compose#v3.11.1:
           push:
             - app:index.docker.io/myorg/myrepo/myapp
             - app:index.docker.io/myorg/myrepo/myapp:latest

diff --git a/commands/run.sh b/commands/run.sh
@@ -239,19 +239,35 @@ elif [[ ! -f "$override_file" ]]; then
   echo
 fi
 
-# Start up service dependencies in a different header to keep the main run with less noise
+dependency_exitcode=0
 if [[ "$(plugin_read_config DEPENDENCIES "true")" == "true" ]] ; then
+
+  # Start up service dependencies in a different header to keep the main run with less noise
   echo "~~~ :docker: Starting dependencies"
   if [[ ${#up_params[@]} -gt 0 ]] ; then
-    run_docker_compose "${up_params[@]}" up -d --scale "${run_service}=0" "${run_service}"
+    run_docker_compose "${up_params[@]}" up -d --scale "${run_service}=0" "${run_service}" || dependency_exitcode=$?
   else
-    run_docker_compose up -d --scale "${run_service}=0" "${run_service}"
+    run_docker_compose up -d --scale "${run_service}=0" "${run_service}" || dependency_exitcode=$?
   fi
 
   # Sometimes docker-compose leaves unfinished ansi codes
   echo
 fi
 
+if [[ $dependency_exitcode -ne 0 ]] ; then
+  # Dependent services failed to start.
+  echo "^^^ +++"
+  echo "+++ 🚨 Failed to start dependencies"
+
+  if [[ -n "${BUILDKITE_AGENT_ACCESS_TOKEN:-}" ]] ; then
+    print_failed_container_information
+
+    upload_container_logs "$run_service"
+  fi
+
+  return $dependency_exitcode
+fi
+
 shell=()
 shell_disabled=1
 result=()
@@ -354,35 +370,9 @@ fi
 
 if [[ -n "${BUILDKITE_AGENT_ACCESS_TOKEN:-}" ]] ; then
   if [[ "$(plugin_read_config CHECK_LINKED_CONTAINERS "true")" != "false" ]] ; then
+    print_failed_container_information
 
-    # Get list of failed containers
-    containers=()
-    while read -r container ; do
-      [[ -n "$container" ]] && containers+=("$container")
-    done <<< "$(docker_ps_by_project -q)"
-
-    failed_containers=()
-    if [[ 0 != "${#containers[@]}" ]] ; then
-      while read -r container ; do
-        [[ -n "$container" ]] && failed_containers+=("$container")
-      done <<< "$(docker inspect -f '{{if ne 0 .State.ExitCode}}{{.Name}}.{{.State.ExitCode}}{{ end }}' \
-        "${containers[@]}")"
-    fi
-
-    if [[ 0 != "${#failed_containers[@]}" ]] ; then
-      echo "+++ :warning: Some containers had non-zero exit codes"
-      docker_ps_by_project \
-        --format 'table {{.Label "com.docker.compose.service"}}\t{{ .ID }}\t{{ .Status }}'
-    fi
-
-    check_linked_containers_and_save_logs \
-      "$run_service" "docker-compose-logs" \
-      "$(plugin_read_config UPLOAD_CONTAINER_LOGS "on-error")"
-
-    if [[ -d "docker-compose-logs" ]] && test -n "$(find docker-compose-logs/ -maxdepth 1 -name '*.log' -print)"; then
-      echo "~~~ Uploading linked container logs"
-      buildkite-agent artifact upload "docker-compose-logs/*.log"
-    fi
+    upload_container_logs "$run_service"
   fi
 fi
 

diff --git a/lib/run.bash b/lib/run.bash
@@ -88,3 +88,42 @@ expand_relative_volume_path() {
 
   echo "${path/.\//$pwd/}"
 }
+
+# Prints information about the failed containers.
+function print_failed_container_information() {
+  # Get list of failed containers
+  containers=()
+  while read -r container ; do
+    [[ -n "$container" ]] && containers+=("$container")
+  done <<< "$(docker_ps_by_project -q)"
+
+  failed_containers=()
+  if [[ 0 != "${#containers[@]}" ]] ; then
+    while read -r container ; do
+      [[ -n "$container" ]] && failed_containers+=("$container")
+    done <<< "$(docker inspect -f '{{if ne 0 .State.ExitCode}}{{.Name}}.{{.State.ExitCode}}{{ end }}' \
+      "${containers[@]}")"
+  fi
+
+  if [[ 0 != "${#failed_containers[@]}" ]] ; then
+    echo "+++ :warning: Some containers had non-zero exit codes"
+    docker_ps_by_project \
+      --format 'table {{.Label "com.docker.compose.service"}}\t{{ .ID }}\t{{ .Status }}'
+  fi
+}
+
+# Uploads the container's logs, respecting the `UPLOAD_CONTAINER_LOGS` option
+function upload_container_logs() {
+  run_service="$1"
+
+  if [[ -n "${BUILDKITE_AGENT_ACCESS_TOKEN:-}" ]] ; then
+    check_linked_containers_and_save_logs \
+      "$run_service" "docker-compose-logs" \
+      "$(plugin_read_config UPLOAD_CONTAINER_LOGS "on-error")"
+
+    if [[ -d "docker-compose-logs" ]] && test -n "$(find docker-compose-logs/ -maxdepth 1 -name '*.log' -print)"; then
+      echo "~~~ Uploading linked container logs"
+      buildkite-agent artifact upload "docker-compose-logs/*.log"
+    fi
+  fi
+}