ci: attempt to re-run failed jobs

Due to instabilities with Docker, sometimes jobs fail when running in parallel. Just retry them a couple of times before failing definitively. Signed-off-by: Leonardo Graboski Veiga <[email protected]>
toradex · May 30, 2023 · f29cc8e · f29cc8e
1 parent 02d92f5
commit f29cc8e
Show file tree

Hide file tree

Showing 23 changed files with 95 additions and 0 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -8,6 +8,10 @@ variables:
   # see https://docs.gitlab.com/ee/ci/runners/configure_runners.html#job-stages-attempts
   GET_SOURCES_ATTEMPTS: 5
 
+default:
+  # retry failed jobs, since Docker is not very stable when running many jobs in parallel
+  retry: 2
+
 # Only run branch pipelines
 include:
   - template: 'Workflows/Branch-Pipelines.gitlab-ci.yml'

diff --git a/adc/.gitlab-ci.yml b/adc/.gitlab-ci.yml
@@ -16,6 +16,10 @@ include:
 
 # Jobs
 
+default:
+  # retry failed jobs, since Docker is not very stable when running many jobs in parallel
+  retry: 2
+
 dockerfile-lint-adc:
   extends: .dockerfile-lint
   stage: lint

diff --git a/crank/.gitlab-ci.yml b/crank/.gitlab-ci.yml
@@ -13,6 +13,11 @@ variables:
   APP_NAME: "crank-storyboard"
 
 # Jobs
+
+default:
+  # retry failed jobs, since Docker is not very stable when running many jobs in parallel
+  retry: 2
+
 push-arm32v7:
   extends: .tcb-push-to-platform-services
   before_script:

diff --git a/dbus/python/.gitlab-ci.yml b/dbus/python/.gitlab-ci.yml
@@ -14,6 +14,10 @@ include:
 
 # Jobs
 
+default:
+  # retry failed jobs, since Docker is not very stable when running many jobs in parallel
+  retry: 2
+
 dockerfile-lint-dbus-python:
   extends: .dockerfile-lint
   stage: lint

diff --git a/dbus/tools/.gitlab-ci.yml b/dbus/tools/.gitlab-ci.yml
@@ -14,6 +14,10 @@ include:
 
 # Jobs
 
+default:
+  # retry failed jobs, since Docker is not very stable when running many jobs in parallel
+  retry: 2
+
 dockerfile-lint-dbus-tools:
   extends: .dockerfile-lint
   stage: lint

diff --git a/debian-container/.gitlab-ci.yml b/debian-container/.gitlab-ci.yml
@@ -14,6 +14,11 @@ variables:
   APP_NAME: "portainer"
 
 # Jobs
+
+default:
+  # retry failed jobs, since Docker is not very stable when running many jobs in parallel
+  retry: 2
+
 push-nogpu-arm32v7:
   extends: .tcb-push-to-platform-services
   before_script:

diff --git a/dlr-gstreamer/.gitlab-ci.yml b/dlr-gstreamer/.gitlab-ci.yml
@@ -15,6 +15,10 @@ include:
 # Jobs
 # Only for 64-bit vivante arch
 
+default:
+  # retry failed jobs, since Docker is not very stable when running many jobs in parallel
+  retry: 2
+
 dockerfile-lint-dlr-gstreamer:
   extends: .dockerfile-lint
   stage: lint

diff --git a/gpio/bash/.gitlab-ci.yml b/gpio/bash/.gitlab-ci.yml
@@ -16,6 +16,10 @@ include:
 
 # Jobs
 
+default:
+  # retry failed jobs, since Docker is not very stable when running many jobs in parallel
+  retry: 2
+
 dockerfile-lint-gpio-bash:
   extends: .dockerfile-lint
   stage: lint

diff --git a/gpio/c/.gitlab-ci.yml b/gpio/c/.gitlab-ci.yml
@@ -14,6 +14,10 @@ include:
 
 # Jobs
 
+default:
+  # retry failed jobs, since Docker is not very stable when running many jobs in parallel
+  retry: 2
+
 dockerfile-lint-gpio-c:
   extends: .dockerfile-lint
   stage: lint

diff --git a/gps/python/.gitlab-ci.yml b/gps/python/.gitlab-ci.yml
@@ -14,6 +14,10 @@ include:
 
 # Jobs
 
+default:
+  # retry failed jobs, since Docker is not very stable when running many jobs in parallel
+  retry: 2
+
 dockerfile-lint-uart-gps-python:
   extends: .dockerfile-lint
   stage: lint

diff --git a/grafana/.gitlab-ci.yml b/grafana/.gitlab-ci.yml
@@ -14,6 +14,10 @@ include:
 
 # Jobs
 
+default:
+  # retry failed jobs, since Docker is not very stable when running many jobs in parallel
+  retry: 2
+
 dockerfile-lint-collectd:
   extends: .dockerfile-lint
   stage: lint

diff --git a/gstreamer/bash/simple-pipeline/.gitlab-ci.yml b/gstreamer/bash/simple-pipeline/.gitlab-ci.yml
@@ -14,6 +14,10 @@ include:
 
 # Jobs
 
+default:
+  # retry failed jobs, since Docker is not very stable when running many jobs in parallel
+  retry: 2
+
 dockerfile-lint-gstreamer-simple-pipeline:
   extends: .dockerfile-lint
   stage: lint

diff --git a/nodejs/.gitlab-ci.yml b/nodejs/.gitlab-ci.yml
@@ -15,6 +15,11 @@ include:
   - /.gitlab-ci-scripts.yml
 
 # Jobs
+
+default:
+  # retry failed jobs, since Docker is not very stable when running many jobs in parallel
+  retry: 2
+
 dockerfile-lint-nodejs:
   extends: .dockerfile-lint
   stage: lint

diff --git a/opencl/.gitlab-ci.yml b/opencl/.gitlab-ci.yml
@@ -15,6 +15,10 @@ include:
 # Jobs
 # Only for 64-bit vivante arch
 
+default:
+  # retry failed jobs, since Docker is not very stable when running many jobs in parallel
+  retry: 2
+
 dockerfile-lint-opencl:
   extends: .dockerfile-lint
   stage: lint

diff --git a/opencv/.gitlab-ci.yml b/opencv/.gitlab-ci.yml
@@ -15,6 +15,10 @@ include:
 # Jobs
 # Only for 64-bit vivante arch
 
+default:
+  # retry failed jobs, since Docker is not very stable when running many jobs in parallel
+  retry: 2
+
 dockerfile-lint-opencv:
   extends: .dockerfile-lint
   stage: lint

diff --git a/pwm/.gitlab-ci.yml b/pwm/.gitlab-ci.yml
@@ -14,6 +14,10 @@ include:
 
 # Jobs
 
+default:
+  # retry failed jobs, since Docker is not very stable when running many jobs in parallel
+  retry: 2
+
 dockerfile-lint-pwm:
   extends: .dockerfile-lint
   stage: lint

diff --git a/python-qml/.gitlab-ci.yml b/python-qml/.gitlab-ci.yml
@@ -14,6 +14,10 @@ include:
 
 # Jobs
 
+default:
+  # retry failed jobs, since Docker is not very stable when running many jobs in parallel
+  retry: 2
+
 dockerfile-lint-qt5-python-qml:
   extends: .dockerfile-lint
   stage: lint

diff --git a/sht31/flaskapp/.gitlab-ci.yml b/sht31/flaskapp/.gitlab-ci.yml
@@ -14,6 +14,10 @@ include:
 
 # Jobs
 
+default:
+  # retry failed jobs, since Docker is not very stable when running many jobs in parallel
+  retry: 2
+
 dockerfile-lint-uart-sht31-flask-app:
   extends: .dockerfile-lint
   stage: lint

diff --git a/sht31/kiosk/.gitlab-ci.yml b/sht31/kiosk/.gitlab-ci.yml
@@ -14,6 +14,10 @@ include:
 
 # Jobs
 
+default:
+  # retry failed jobs, since Docker is not very stable when running many jobs in parallel
+  retry: 2
+
 dockerfile-lint-uart-sht31-kiosk:
   extends: .dockerfile-lint
   stage: lint

diff --git a/sht31/sht31-influxdb/.gitlab-ci.yml b/sht31/sht31-influxdb/.gitlab-ci.yml
@@ -14,6 +14,10 @@ include:
 
 # Jobs
 
+default:
+  # retry failed jobs, since Docker is not very stable when running many jobs in parallel
+  retry: 2
+
 dockerfile-lint-uart-sht31-influxdb:
   extends: .dockerfile-lint
   stage: lint

diff --git a/tflite/tflite-basic/.gitlab-ci.yml b/tflite/tflite-basic/.gitlab-ci.yml
@@ -16,6 +16,10 @@ include:
 
 # Jobs
 
+default:
+  # retry failed jobs, since Docker is not very stable when running many jobs in parallel
+  retry: 2
+
 dockerfile-lint-tflite:
   extends: .dockerfile-lint
   stage: lint

diff --git a/tflite/tflite-rtsp/.gitlab-ci.yml b/tflite/tflite-rtsp/.gitlab-ci.yml
@@ -15,6 +15,10 @@ include:
 # Jobs
 # Only for 64-bit vivante arch
 
+default:
+  # retry failed jobs, since Docker is not very stable when running many jobs in parallel
+  retry: 2
+
 dockerfile-lint-tflite:
   extends: .dockerfile-lint
   stage: lint

diff --git a/weather/.gitlab-ci.yml b/weather/.gitlab-ci.yml
@@ -14,6 +14,10 @@ include:
 
 # Jobs
 
+default:
+  # retry failed jobs, since Docker is not very stable when running many jobs in parallel
+  retry: 2
+
 dockerfile-lint-weather:
   extends: .dockerfile-lint
   stage: lint