ci: fix flaky image gc and mysql connect + add k3s debug log (#13660)

Signed-off-by: Mason Malone <[email protected]>

.github/workflows/ci-build.yaml

@@ -240,6 +240,13 @@ jobs:
  profile: minimal
  use-api: false
  steps:
+ - name: Free up disk space
+ run: |
+ printf "==> Available space before cleanup\n"
+ df -h
+ sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /usr/local/.ghcup /opt/hostedtoolcache/CodeQL
+ printf "==> Available space after cleanup\n"
+ df -h
  - name: Install socat (needed by Kubernetes)
  run: sudo apt-get -y install socat
  - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -267,7 +274,15 @@ jobs:
  else
  export INSTALL_K3S_VERSION=${{ matrix.install_k3s_version }}
  fi
- curl -sfL https://get.k3s.io | INSTALL_K3S_CHANNEL=stable INSTALL_K3S_EXEC=--docker K3S_KUBECONFIG_MODE=644 sh -
+
+ cat >/tmp/kubelet.config <<EOT
+ apiVersion: kubelet.config.k8s.io/v1beta1
+ kind: KubeletConfiguration
+ imageMinimumGCAge: 1h
+ imageGCHighThresholdPercent: 100
+ EOT
+
+ curl -sfL https://get.k3s.io | INSTALL_K3S_CHANNEL=stable INSTALL_K3S_EXEC='--docker --kubelet-arg=config=/tmp/kubelet.config' K3S_KUBECONFIG_MODE=644 sh -
  until kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml cluster-info ; do sleep 10s ; done
  cp /etc/rancher/k3s/k3s.yaml /home/runner/.kubeconfig
  echo "- name: fake_token_user" >> $KUBECONFIG
@@ -307,12 +322,15 @@ jobs:
  UI=false \
  POD_STATUS_CAPTURE_FINALIZER=true > /tmp/argo.log 2>&1 &
  - name: Wait for controller to be up
- run: make wait API=${{matrix.use-api}}
+ run: make wait PROFILE=${{matrix.profile}} API=${{matrix.use-api}}
  timeout-minutes: 5
  - name: Run tests ${{matrix.test}}
  run: make ${{matrix.test}} E2E_SUITE_TIMEOUT=20m STATIC_FILES=false
 
  # failure debugging below
+ - name: Failure debug - k3s logs
+ if: ${{ failure() }}
+ run: journalctl -u k3s
  - name: Failure debug - describe MinIO/MySQL deployment
  if: ${{ failure() }}
  run: |

Makefile

@@ -574,6 +574,10 @@ ifeq ($(API),true)
  # Wait for Argo Server
  until lsof -i :2746 > /dev/null ; do sleep 10s ; done
 endif
+ifeq ($(PROFILE),mysql)
+ # Wait for MySQL
+ until (: < /dev/tcp/localhost/3306) ; do sleep 10s ; done
+endif
 
 .PHONY: postgres-cli
 postgres-cli: