jenkins: test jenkins in a loop

Jenkinsfile

@@ -164,7 +164,7 @@ node {
  'testing-closet*', 'hotfix-*']
  def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*')
 
- if (env.BRANCH_NAME != 'master') {
+ if (env.BRANCH_NAME != 'master' && env.BRANCH_NAME != '__jenkins_loop_test') {
  properties([
  disableConcurrentBuilds(abortPrevious: true)
  ])

scripts/jenkins_loop_test.sh

@@ -0,0 +1,135 @@
+#!/usr/bin/env bash
+set -e
+
+YELLOW='\033[0;33m'
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+BOLD='\033[1m'
+NC='\033[0m'
+
+COOKIE_JAR=/tmp/cookies
+CRUMB=$(curl -s --cookie-jar $COOKIE_JAR 'https://jenkins.comma.life/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')
+
+TESTING_BRANCH="__jenkins_loop_test"
+API_ROUTE="https://jenkins.comma.life/job/openpilot/job/$TESTING_BRANCH"
+N=20
+TEST_BUILDS=()
+
+# Try to find previous builds
+ALL_BUILDS=( $(curl -s $API_ROUTE/api/json | jq .builds.[].number || :) )
+
+# No builds. Create branch
+if [[ ${#ALL_BUILDS[@]} -eq 0 ]]; then
+ TEMP_DIR=$(mktemp -d)
+ GIT_LFS_SKIP_SMUDGE=1 git clone -b master --depth=1 --no-tags [email protected]:commaai/openpilot $TEMP_DIR
+ git -C $TEMP_DIR checkout -b $TESTING_BRANCH
+ echo "TESTING" >> $TEMP_DIR/testing_jenkins
+ git -C $TEMP_DIR add testing_jenkins
+ git -C $TEMP_DIR commit -m "testing"
+ git -C $TEMP_DIR push -f origin $TESTING_BRANCH
+ rm -rf $TEMP_DIR
+ FIRST_RUN=1
+ sleep 90
+else
+ # Found some builds. Check if they are still running
+ for i in ${ALL_BUILDS[@]}; do
+ running=$(curl -s $API_ROUTE/$i/api/json/ | jq .inProgress)
+ if [[ $running == "false" ]]; then
+ continue
+ fi
+ TEST_BUILDS=( ${ALL_BUILDS[@]} )
+ N=${#TEST_BUILDS[@]}
+ break
+ done
+fi
+
+# No running builds found
+if [[ ${#TEST_BUILDS[@]} -eq 0 ]]; then
+ # Delete all previous builds
+ for i in ${ALL_BUILDS[@]}; do
+ curl -s --cookie $COOKIE_JAR -H "$CRUMB" -X POST $API_ROUTE/$i/doDelete
+ done
+
+ FIRST_RUN=$(curl -s $API_ROUTE/api/json | jq .nextBuildNumber)
+ LAST_RUN=$((FIRST_RUN+N-1))
+ TEST_BUILDS=( $(seq $FIRST_RUN $LAST_RUN) )
+
+ # Start N new builds
+ for i in ${TEST_BUILDS[@]};
+ do
+ echo "Starting build $i"
+ curl -s --output /dev/null --cookie $COOKIE_JAR -H "$CRUMB" -X POST $API_ROUTE/build?delay=0sec
+ sleep 5
+ done
+fi
+
+echo "Testing Jenkins with $N builds:"
+
+while true; do
+ #sleep 60
+ sleep 5
+
+ count=0
+ for i in ${TEST_BUILDS[@]};
+ do
+ RES=$(curl -s -w "\n%{http_code}" --cookie $COOKIE_JAR -H "$CRUMB" $API_ROUTE/$i/api/json)
+ HTTP_CODE=$(tail -n1 <<< "$RES")
+ JSON=$(sed '$ d' <<< "$RES")
+
+ if [[ $HTTP_CODE == "200" ]]; then
+ STILL_RUNNING=$(echo $JSON | jq .inProgress)
+ if [[ $STILL_RUNNING == "true" ]]; then
+ echo -e "Build $i: ${YELLOW}still running${NC}"
+ continue
+ else
+ count=$((count+1))
+ echo -e "Build $i: ${GREEN}done${NC}"
+ fi
+ else
+ echo "No status for build $i"
+ fi
+ done
+ echo ""
+
+ if [[ $count -ge $N ]]; then
+ break
+ fi
+done
+
+STAGES_NAMES=()
+while read stage; do
+ STAGES_NAMES[$index]=$stage
+ index=$((index+1))
+done < <(curl -s -H "$CRUMB" $API_ROUTE/lastBuild/wfapi/ | jq .stages[].name)
+STAGES_COUNT=${#STAGES_NAMES[@]}
+
+STAGES_FAILURES=($(for i in $(seq 1 $STAGES_COUNT); do echo 0; done))
+
+for i in ${TEST_BUILDS[@]}; do
+index=0
+while read result; do
+ if [[ $result != '"SUCCESS"' ]]; then
+ STAGES_FAILURES[$index]=$((STAGES_FAILURES[$index]+1))
+ fi
+ index=$((index+1))
+done < <(curl -s $API_ROUTE/$i/wfapi/ | jq .stages[].status)
+done
+
+echo "=========================================="
+echo "===================DONE==================="
+echo "=========================================="
+
+# print results of all builds
+for i in $(seq 0 $(($STAGES_COUNT-1))); do
+ P=$((${STAGES_FAILURES[$i]}*100/$N))
+ if [[ $P -eq 0 ]]; then
+ P=${GREEN}${BOLD}${P}
+ else
+ P=${RED}${BOLD}${P}
+ fi
+ echo -e "${STAGES_NAMES[$i]} : $P% failure rate${NC}"
+done
+
+echo "=========================================="
+echo "=========================================="
+echo "=========================================="