diff --git a/.buildkite/it/run.sh b/.buildkite/it/run.sh index 4c02dce31..6d01e6ef7 100644 --- a/.buildkite/it/run.sh +++ b/.buildkite/it/run.sh @@ -2,6 +2,8 @@ set -eo pipefail +source .buildkite/retry.sh + export TERM=dumb export LC_ALL=en_US.UTF-8 export TZ=Etc/UTC @@ -10,37 +12,6 @@ export DEBIAN_FRONTEND=noninteractive sudo mkdir -p /etc/needrestart echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null -# based on https://gist.github.com/sj26/88e1c6584397bb7c13bd11108a579746?permalink_comment_id=4155247#gistcomment-4155247 -function retry { - local retries=$1 - shift - local cmd=($@) - local cmd_string="${@}" - local count=0 - - # be lenient with non-zero exit codes, to allow retries - set +o errexit - set +o pipefail - until "${cmd[@]}"; do - retcode=$? - wait=$(( 2 ** count )) - count=$(( count + 1)) - if [[ $count -le $retries ]]; then - printf "Command [%s] failed. Retry [%d/%d] in [%d] seconds.\n" "$cmd_string" $count $retries $wait - sleep $wait - else - printf "Exhausted all [%s] retries for command [%s]. Exiting.\n" "$cmd_string" $retries - # restore settings to fail immediately on error - set -o errexit - set -o pipefail - return $retcode - fi - done - # restore settings to fail immediately on error - set -o errexit - set -o pipefail - return 0 -} echo "--- System dependencies" PYTHON_VERSION="$1" diff --git a/.buildkite/it/run_serverless.sh b/.buildkite/it/run_serverless.sh index 7c90d012a..b7110690d 100644 --- a/.buildkite/it/run_serverless.sh +++ b/.buildkite/it/run_serverless.sh @@ -2,6 +2,8 @@ set -eo pipefail +source .buildkite/retry.sh + export TERM=dumb export LC_ALL=en_US.UTF-8 export TZ=Etc/UTC @@ -10,37 +12,6 @@ export DEBIAN_FRONTEND=noninteractive sudo mkdir -p /etc/needrestart echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null -# based on https://gist.github.com/sj26/88e1c6584397bb7c13bd11108a579746?permalink_comment_id=4155247#gistcomment-4155247 -function retry { - local retries=$1 - shift - local cmd=($@) - local cmd_string="${@}" - local count=0 - - # be lenient with non-zero exit codes, to allow retries - set +o errexit - set +o pipefail - until "${cmd[@]}"; do - retcode=$? - wait=$(( 2 ** count )) - count=$(( count + 1)) - if [[ $count -le $retries ]]; then - printf "Command [%s] failed. Retry [%d/%d] in [%d] seconds.\n" "$cmd_string" $count $retries $wait - sleep $wait - else - printf "Exhausted all [%s] retries for command [%s]. Exiting.\n" "$cmd_string" $retries - # restore settings to fail immediately on error - set -o errexit - set -o pipefail - return $retcode - fi - done - # restore settings to fail immediately on error - set -o errexit - set -o pipefail - return 0 -} echo "--- System dependencies" PYTHON_VERSION="$1" diff --git a/.buildkite/release-docker/run.sh b/.buildkite/release-docker/run.sh index 2c69e5b03..87b6c5a34 100644 --- a/.buildkite/release-docker/run.sh +++ b/.buildkite/release-docker/run.sh @@ -2,37 +2,7 @@ set -eo pipefail -# based on https://gist.github.com/sj26/88e1c6584397bb7c13bd11108a579746?permalink_comment_id=4155247#gistcomment-4155247 -function retry { - local retries=$1 - shift - local cmd=($@) - local cmd_string="${@}" - local count=0 - - # be lenient with non-zero exit codes, to allow retries - set +o errexit - set +o pipefail - until "${cmd[@]}"; do - retcode=$? - wait=$(( 2 ** count )) - count=$(( count + 1)) - if [[ $count -le $retries ]]; then - printf "Command [%s] failed. Retry [%d/%d] in [%d] seconds.\n" "$cmd_string" $count $retries $wait - sleep $wait - else - printf "Exhausted all [%s] retries for command [%s]. Exiting.\n" "$cmd_string" $retries - # restore settings to fail immediately on error - set -o errexit - set -o pipefail - return $retcode - fi - done - # restore settings to fail immediately on error - set -o errexit - set -o pipefail - return 0 -} +source .buildkite/retry.sh set +x RELEASE_VERSION=$(buildkite-agent meta-data get RELEASE_VERSION) diff --git a/.buildkite/retry.sh b/.buildkite/retry.sh new file mode 100644 index 000000000..0e08a1671 --- /dev/null +++ b/.buildkite/retry.sh @@ -0,0 +1,31 @@ +# based on https://gist.github.com/sj26/88e1c6584397bb7c13bd11108a579746?permalink_comment_id=4155247#gistcomment-4155247 +function retry { + local retries=$1 + shift + local cmd=($@) + local cmd_string="${@}" + local count=0 + + # be lenient with non-zero exit codes, to allow retries + set +o errexit + set +o pipefail + until "${cmd[@]}"; do + retcode=$? + wait=$(( 2 ** count )) + count=$(( count + 1)) + if [[ $count -le $retries ]]; then + printf "Command [%s] failed. Retry [%d/%d] in [%d] seconds.\n" "$cmd_string" $count $retries $wait + sleep $wait + else + printf "Exhausted all [%s] retries for command [%s]. Exiting.\n" "$cmd_string" $retries + # restore settings to fail immediately on error + set -o errexit + set -o pipefail + return $retcode + fi + done + # restore settings to fail immediately on error + set -o errexit + set -o pipefail + return 0 +} \ No newline at end of file