chore(scripts): add retry option for setup-kube commands

Signed-off-by: Artur Troian <[email protected]>
akash-network · Feb 2, 2024 · 6d57f25 · 6d57f25
1 parent 181c338
commit 6d57f25
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 8 deletions.
diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml
@@ -112,7 +112,7 @@ jobs:
         working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider
         run: |
           make -s -C _run/kube kube-deployment-rollout-operator-inventory
-          make -s -C _run/kube kube-wait-inventory-available
+          make -s -C _run/kube kube-wait-inventory-test
       - name: Run E2E Tests
         working-directory: ${{ env.GOPATH }}/src/github.com/akash-network/provider
         run: |

diff --git a/_run/common-kube.mk b/_run/common-kube.mk
@@ -190,9 +190,15 @@ akash-node-ready:
 
 .PHONY: kube-logs-operator-inventory
 kube-logs-operator-inventory:
-	kubectl -n akash-services logs -f \
+	kubectl -n akash-services logs \
 		-l app.kubernetes.io/part-of=provider,app.kubernetes.io/component=operator,app.kubernetes.io/instance=inventory-service,app.kubernetes.io/name=inventory
 
 .PHONY: kube-wait-inventory-available
 kube-wait-inventory-available:
-	$(SETUP_KUBE) wait inventory-available
+	$(SETUP_KUBE) --retries=20 wait inventory-available || exit 0
+
+
+.PHONY: kube-wait-inventory-test
+kube-wait-inventory-test: kube-wait-inventory-available kube-logs-operator-inventory
+	exit 1
+
diff --git a/script/setup-kube.sh b/script/setup-kube.sh
@@ -14,6 +14,8 @@ rootdir="$(dirname "$0")/.."
 
 CRD_FILE=$rootdir/pkg/apis/akash.network/crd.yaml
 timeout=10
+retries=10
+retrywait=1
 
 usage() {
     cat <<EOF
@@ -40,8 +42,10 @@ EOF
     exit 1
 }
 
+isuint() { [[ $1 =~ ^[0-9]+$ ]] ;}
+
 short_opts=h
-long_opts=help/crd:/timeout:   # those who take an arg END with :
+long_opts=help/crd:/retries:/retry-wait:/timeout:   # those who take an arg END with :
 
 while getopts ":$short_opts-:" o; do
     case $o in
@@ -99,6 +103,26 @@ while getopts ":$short_opts-:" o; do
             ;;
         timeout)
             timeout=$OPTARG
+            if ! isuint "$timeout" ; then
+                echo >&2 "timeout option must be positive integer"
+                exit 1
+            fi
+            ;;
+        retries)
+            retries=$OPTARG
+
+            if ! isuint "$retries" ; then
+                echo >&2 "retries option must be positive integer"
+                exit 1
+            fi
+            ;;
+        retry-wait)
+            retrywait=$OPTARG
+            if ! isuint "$retrywait" ; then
+                echo >&2 "retry-wait option must be positive integer"
+                exit 1
+            fi
+            ;;
     esac
 done
 shift "$((OPTIND - 1))"
@@ -309,14 +333,17 @@ wait_inventory_available() {
 
     timeout 10 bash -c -- 'while ! nc -vz localhost 8455 > /dev/null 2>&1 ; do sleep 0.1; done'
 
-    local retries=0
+    local r=0
 
     while ! grpcurl -plaintext localhost:8455 akash.inventory.v1.ClusterRPC.QueryCluster | jq '(.nodes | length > 0) and (.storage | length > 0)' --exit-status > /dev/null 2>&1; do
-        retries=$((retries+1))
-        if [ ${retries} -eq "${timeout}" ]; then
+        r=$((r+1))
+        if [ ${r} -eq "${retries}" ]; then
+            grpcurl -plaintext localhost:8455 akash.inventory.v1.ClusterRPC.QueryCluster
             exit 1
         fi
-        sleep 1
+
+        # shellcheck disable=SC2086
+        sleep $retrywait
     done
 }