[Response Ops][Test] Debugging Flaky Tests

.buildkite/pipelines/pull_request/base.yml

@@ -7,32 +7,32 @@ steps:
 
   - wait
 
-  - command: .buildkite/scripts/steps/build_kibana.sh
-    label: Build Kibana Distribution and Plugins
-    agents:
-      machineType: n2-standard-16
-      preemptible: true
-    key: build
-    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
-    timeout_in_minutes: 90
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-
-  - command: .buildkite/scripts/steps/quick_checks.sh
-    label: 'Quick Checks'
-    agents:
-      machineType: n2-highcpu-8
-      preemptible: true
-    key: quick_checks
-    timeout_in_minutes: 60
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
+  # - command: .buildkite/scripts/steps/build_kibana.sh
+  #   label: Build Kibana Distribution and Plugins
+  #   agents:
+  #     machineType: n2-standard-16
+  #     preemptible: true
+  #   key: build
+  #   if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+  #   timeout_in_minutes: 90
+  #   retry:
+  #     automatic:
+  #       - exit_status: '-1'
+  #         limit: 3
 
-  - wait
+  # - command: .buildkite/scripts/steps/quick_checks.sh
+  #   label: 'Quick Checks'
+  #   agents:
+  #     machineType: n2-highcpu-8
+  #     preemptible: true
+  #   key: quick_checks
+  #   timeout_in_minutes: 60
+  #   retry:
+  #     automatic:
+  #       - exit_status: '-1'
+  #         limit: 3
+  #
+  # - wait
 
   - command: .buildkite/scripts/steps/ci_stats_ready.sh
     label: Mark CI Stats as ready
@@ -54,69 +54,69 @@ steps:
     timeout_in_minutes: 10
     env:
       JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh'
-      JEST_INTEGRATION_SCRIPT: '.buildkite/scripts/steps/test/jest_integration.sh'
-      FTR_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/ftr_configs.sh'
+      # JEST_INTEGRATION_SCRIPT: '.buildkite/scripts/steps/test/jest_integration.sh'
+      # FTR_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/ftr_configs.sh'
     retry:
       automatic:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/lint.sh
-    label: 'Linting'
-    agents:
-      machineType: n2-standard-8
-      preemptible: true
-    key: linting
-    timeout_in_minutes: 60
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-
-  - command: .buildkite/scripts/steps/check_types.sh
-    label: 'Check Types'
-    agents:
-      machineType: n2-standard-4
-      preemptible: true
-    key: check_types
-    timeout_in_minutes: 70
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-
-  - command: .buildkite/scripts/steps/lint_with_types.sh
-    label: 'Linting (with types)'
-    agents:
-      machineType: n2-standard-16
-      preemptible: true
-    key: linting_with_types
-    timeout_in_minutes: 90
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-
-  - command: .buildkite/scripts/steps/checks.sh
-    label: 'Checks'
-    key: checks
-    agents:
-      machineType: n2-standard-2
-      preemptible: true
-    timeout_in_minutes: 60
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-
-  - command: .buildkite/scripts/steps/api_docs/build_api_docs.sh
-    label: 'Build API Docs'
-    agents:
-      machineType: n2-standard-4
-      preemptible: true
-    key: build_api_docs
-    timeout_in_minutes: 90
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
+  # - command: .buildkite/scripts/steps/lint.sh
+  #   label: 'Linting'
+  #   agents:
+  #     machineType: n2-standard-8
+  #     preemptible: true
+  #   key: linting
+  #   timeout_in_minutes: 60
+  #   retry:
+  #     automatic:
+  #       - exit_status: '-1'
+  #         limit: 3
+  #
+  # - command: .buildkite/scripts/steps/check_types.sh
+  #   label: 'Check Types'
+  #   agents:
+  #     machineType: n2-standard-4
+  #     preemptible: true
+  #   key: check_types
+  #   timeout_in_minutes: 70
+  #   retry:
+  #     automatic:
+  #       - exit_status: '-1'
+  #         limit: 3
+  #
+  # - command: .buildkite/scripts/steps/lint_with_types.sh
+  #   label: 'Linting (with types)'
+  #   agents:
+  #     machineType: n2-standard-16
+  #     preemptible: true
+  #   key: linting_with_types
+  #   timeout_in_minutes: 90
+  #   retry:
+  #     automatic:
+  #       - exit_status: '-1'
+  #         limit: 3
+  #
+  # - command: .buildkite/scripts/steps/checks.sh
+  #   label: 'Checks'
+  #   key: checks
+  #   agents:
+  #     machineType: n2-standard-2
+  #     preemptible: true
+  #   timeout_in_minutes: 60
+  #   retry:
+  #     automatic:
+  #       - exit_status: '-1'
+  #         limit: 3
+  #
+  # - command: .buildkite/scripts/steps/api_docs/build_api_docs.sh
+  #   label: 'Build API Docs'
+  #   agents:
+  #     machineType: n2-standard-4
+  #     preemptible: true
+  #   key: build_api_docs
+  #   timeout_in_minutes: 90
+  #   retry:
+  #     automatic:
+  #       - exit_status: '-1'
+  #         limit: 3

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -46,26 +46,26 @@ const getPipeline = (filename: string, removeSteps = true) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/kbn_handlebars.yml'));
     }
 
-    if (
-      (await doAnyChangesMatch([
-        /^src\/plugins\/data/,
-        /^x-pack\/plugins\/actions/,
-        /^x-pack\/plugins\/alerting/,
-        /^x-pack\/plugins\/event_log/,
-        /^x-pack\/plugins\/rule_registry/,
-        /^x-pack\/plugins\/task_manager/,
-      ])) ||
-      GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
-    ) {
-      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/response_ops.yml'));
-    }
-
-    if (
-      (await doAnyChangesMatch([/^x-pack\/plugins\/cases/])) ||
-      GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
-    ) {
-      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/response_ops_cases.yml'));
-    }
+    // if (
+    //   (await doAnyChangesMatch([
+    //     /^src\/plugins\/data/,
+    //     /^x-pack\/plugins\/actions/,
+    //     /^x-pack\/plugins\/alerting/,
+    //     /^x-pack\/plugins\/event_log/,
+    //     /^x-pack\/plugins\/rule_registry/,
+    //     /^x-pack\/plugins\/task_manager/,
+    //   ])) ||
+    //   GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
+    // ) {
+    //   pipeline.push(getPipeline('.buildkite/pipelines/pull_request/response_ops.yml'));
+    // }
+
+    // if (
+    //   (await doAnyChangesMatch([/^x-pack\/plugins\/cases/])) ||
+    //   GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
+    // ) {
+    //   pipeline.push(getPipeline('.buildkite/pipelines/pull_request/response_ops_cases.yml'));
+    // }
 
     if (
       (await doAnyChangesMatch([

.buildkite/scripts/steps/quick_checks.sh

@@ -2,9 +2,10 @@
 
 set -euo pipefail
 
-if [[ "${CI:-}" =~ ^(1|true)$ ]]; then
-  export DISABLE_BOOTSTRAP_VALIDATION=false
-  .buildkite/scripts/bootstrap.sh
-fi
-
-node scripts/quick_checks --file .buildkite/scripts/steps/checks/quick_checks.txt
+# if [[ "${CI:-}" =~ ^(1|true)$ ]]; then
+#   export DISABLE_BOOTSTRAP_VALIDATION=false
+#   .buildkite/scripts/bootstrap.sh
+# fi
+#
+# node scripts/quick_checks --file .buildkite/scripts/steps/checks/quick_checks.txt
+exit 0

.buildkite/scripts/steps/test/jest_parallel.sh

@@ -53,51 +53,51 @@ echo "
   of noise on CI without any percevable benefit, so they have been disabled. If you want to log
   output in your test temporarily, you can modify 'packages/kbn-test/src/jest/setup/disable_console_logs.js'
 "
+# while read -r config; do
+config="x-pack/plugins/cases/jest.config.js"
+echo "--- $ node scripts/jest --config $config"
+
+# --trace-warnings to debug
+# Node.js process-warning detected:
+# Warning: Closing file descriptor 24 on garbage collection
+cmd="NODE_OPTIONS=\"--max-old-space-size=12288 --trace-warnings\" node ./scripts/jest --config=\"$config\" $parallelism --coverage=false --passWithNoTests --detectOpenHandles"
+echo "actual full command is:"
+echo "$cmd"
+echo ""
 
-while read -r config; do
-  echo "--- $ node scripts/jest --config $config"
-
-  # --trace-warnings to debug
-  # Node.js process-warning detected:
-  # Warning: Closing file descriptor 24 on garbage collection
-  cmd="NODE_OPTIONS=\"--max-old-space-size=12288 --trace-warnings\" node ./scripts/jest --config=\"$config\" $parallelism --coverage=false --passWithNoTests"
-  echo "actual full command is:"
-  echo "$cmd"
-  echo ""
-
-  start=$(date +%s)
-
-  # prevent non-zero exit code from breaking the loop
-  set +e;
-  eval "$cmd"
-  lastCode=$?
-  set -e;
-
-  timeSec=$(($(date +%s)-start))
-  if [[ $timeSec -gt 60 ]]; then
-    min=$((timeSec/60))
-    sec=$((timeSec-(min*60)))
-    duration="${min}m ${sec}s"
-  else
-    duration="${timeSec}s"
-  fi
+start=$(date +%s)
 
-  results+=("- $config
-    duration: ${duration}
-    result: ${lastCode}")
+# prevent non-zero exit code from breaking the loop
+set +e
+eval "$cmd"
+lastCode=$?
+set -e
 
-  if [ $lastCode -ne 0 ]; then
-    exitCode=10
-    echo "Jest exited with code $lastCode"
-    echo "^^^ +++"
+timeSec=$(($(date +%s) - start))
+if [[ $timeSec -gt 60 ]]; then
+  min=$((timeSec / 60))
+  sec=$((timeSec - (min * 60)))
+  duration="${min}m ${sec}s"
+else
+  duration="${timeSec}s"
+fi
 
-    if [[ "$failedConfigs" ]]; then
-      failedConfigs="${failedConfigs}"$'\n'"$config"
-    else
-      failedConfigs="$config"
-    fi
+results+=("- $config
+  duration: ${duration}
+  result: ${lastCode}")
+
+if [ $lastCode -ne 0 ]; then
+  exitCode=10
+  echo "Jest exited with code $lastCode"
+  echo "^^^ +++"
+
+  if [[ "$failedConfigs" ]]; then
+    failedConfigs="${failedConfigs}"$'\n'"$config"
+  else
+    failedConfigs="$config"
   fi
-done <<< "$configs"
+fi
+# done <<< "$configs"
 
 if [[ "$failedConfigs" ]]; then
   buildkite-agent meta-data set "$FAILED_CONFIGS_KEY" "$failedConfigs"

packages/kbn-test/src/jest/setup/disable_console_logs.js

@@ -8,6 +8,6 @@
  */
 
 // on CI these logs just muddy up the console and produce a ton of unnecessary noise
-console.log = () => {};
-console.error = () => {};
-console.warn = () => {};
+// console.log = () => {};
+// console.error = () => {};
+// console.warn = () => {};