[tests] Add retries for build_test_image.

Sometimes we get a failure to read file.
shizunge · Dec 3, 2024 · 054107b · 054107b
1 parent 13f3b64
commit 054107b
Showing 1 changed file with 20 additions and 8 deletions.
diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh
@@ -642,14 +642,26 @@ build_test_image() {
   if [ -n "${EXIT_SECONDS}" ] && [ "${EXIT_SECONDS}" -gt "0" ]; then
     EXIT_CMD="sleep ${EXIT_SECONDS};"
   fi
-  local FILE=
-  FILE=$(make_test_temp_file)
-  echo "FROM $(_get_test_service_image)" > "${FILE}"
-  echo "ENTRYPOINT [\"sh\", \"-c\", \"echo $(unique_id); trap \\\"${EXIT_CMD}\\\" HUP INT TERM; ${TASK_CMD}\"]" >> "${FILE}"
-  pull_image_if_not_exist "$(_get_test_service_image)"
-  echo "Building image ${IMAGE_WITH_TAG} from ${FILE}"
-  docker build --quiet --tag "${IMAGE_WITH_TAG}" --file "${FILE}" .
-  rm "${FILE}"
+  local RETURN_VALUE=1
+  local TRIES=0
+  local MAX_RETRIES=60
+  while [ "${RETURN_VALUE}" != "0" ]; do
+    if [ "${TRIES}" -ge "${MAX_RETRIES}" ]; then
+      echo "build_test_image Reach MAX_RETRIES ${MAX_RETRIES}" >&2
+      return 1
+    fi
+    TRIES=$((TRIES+1))
+    local FILE=
+    FILE=$(make_test_temp_file)
+    echo "FROM $(_get_test_service_image)" > "${FILE}"
+    echo "ENTRYPOINT [\"sh\", \"-c\", \"echo $(unique_id); trap \\\"${EXIT_CMD}\\\" HUP INT TERM; ${TASK_CMD}\"]" >> "${FILE}"
+    pull_image_if_not_exist "$(_get_test_service_image)"
+    echo "Building image ${IMAGE_WITH_TAG} from ${FILE}"
+    docker build --quiet --tag "${IMAGE_WITH_TAG}" --file "${FILE}" . 2>&1
+    RETURN_VALUE=$?
+    rm "${FILE}"
+    [ "${RETURN_VALUE}" != "0" ] && sleep 1
+  done
 }
 
 build_and_push_test_image() {