helm test: Add retry mechanism in helm test

Some container images doesn't have a very robust health check mechanism which will cause helm test failing in connecting even with health check is passed. Add the retry mechanism in all helm test to avoid sporadic connection failure in helm test. Signed-off-by: Lianhao Lu <[email protected]>
opea-project · Aug 6, 2024 · fdad260 · fdad260
1 parent df040bf
commit fdad260
Show file tree

Hide file tree

Showing 17 changed files with 156 additions and 81 deletions.
diff --git a/helm-charts/chatqna/templates/tests/test-pod.yaml b/helm-charts/chatqna/templates/tests/test-pod.yaml
@@ -13,12 +13,17 @@ metadata:
 spec:
   containers:
     - name: curl
-      image: alpine/curl
-      #image: python:3.10.14
-      command: ['sh', '-c']
+      image: python:3.10.14
+      command: ['bash', '-c']
       args:
         - |
-          curl http://{{ include "chatqna.fullname" . }}:{{ .Values.service.port }}/v1/chatqna -sS --fail-with-body \
-            -d '{"messages": "What is the revenue of Nike in 2023?"}' \
-            -H 'Content-Type: application/json'
+          max_retry=20;
+          for ((i=1; i<=max_retry; i++)); do
+            curl http://{{ include "chatqna.fullname" . }}:{{ .Values.service.port }}/v1/chatqna -sS --fail-with-body \
+            -d '{"messages": "What is the machine learning?"}' \
+            -H 'Content-Type: application/json' && break;
+            curlcode=$?
+            if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi;
+          done;
+          if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi
   restartPolicy: Never
diff --git a/helm-charts/codegen/templates/tests/test-pod.yaml b/helm-charts/codegen/templates/tests/test-pod.yaml
@@ -13,12 +13,17 @@ metadata:
 spec:
   containers:
     - name: curl
-      image: alpine/curl
-      #image: python:3.10.14
-      command: ['sh', '-c']
+      image: python:3.10.14
+      command: ['bash', '-c']
       args:
         - |
-          curl http://{{ include "codegen.fullname" . }}:{{ .Values.service.port }}/v1/codegen -sS --fail-with-body \
+          max_retry=20;
+          for ((i=1; i<=max_retry; i++)); do
+            curl http://{{ include "codegen.fullname" . }}:{{ .Values.service.port }}/v1/codegen -sS --fail-with-body \
             -d '{"messages": "def print_hello_world():"}' \
-            -H 'Content-Type: application/json'
+            -H 'Content-Type: application/json' && break;
+            curlcode=$?
+            if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi;
+          done;
+          if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi
   restartPolicy: Never
diff --git a/helm-charts/codetrans/templates/tests/test-pod.yaml b/helm-charts/codetrans/templates/tests/test-pod.yaml
@@ -13,12 +13,17 @@ metadata:
 spec:
   containers:
     - name: curl
-      image: alpine/curl
-      #image: python:3.10.14
-      command: ['sh', '-c']
+      image: python:3.10.14
+      command: ['bash', '-c']
       args:
         - |
-          curl http://{{ include "codetrans.fullname" . }}:{{ .Values.service.port }}/v1/codetrans -sS --fail-with-body \
+          max_retry=20;
+          for ((i=1; i<=max_retry; i++)); do
+            curl http://{{ include "codetrans.fullname" . }}:{{ .Values.service.port }}/v1/codetrans -sS --fail-with-body \
             -H 'Content-Type: application/json' \
-            -d '{"language_from": "Golang","language_to": "Python","source_code": "package main\n\nimport \"fmt\"\nfunc main() {\n    fmt.Println(\"Hello, World!\");\n}"}'
+            -d '{"language_from": "Golang","language_to": "Python","source_code": "package main\n\nimport \"fmt\"\nfunc main() {\n    fmt.Println(\"Hello, World!\");\n}"}' && break;
+            curlcode=$?
+            if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi;
+          done;
+          if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi
   restartPolicy: Never
diff --git a/helm-charts/common/asr/templates/tests/test-pod.yaml b/helm-charts/common/asr/templates/tests/test-pod.yaml
@@ -13,13 +13,18 @@ metadata:
 spec:
   containers:
     - name: curl
-      image: alpine/curl
-      #image: python:3.10.14
-      command: ['sh', '-c']
+      image: python:3.10.14
+      command: ['bash', '-c']
       args:
         - |
-          curl -sS --fail-with-body http://{{ include "asr.fullname" . }}:{{ .Values.service.port }}/v1/audio/transcriptions \
+          max_retry=20;
+          for ((i=1; i<=max_retry; i++)); do
+            curl -sS --fail-with-body http://{{ include "asr.fullname" . }}:{{ .Values.service.port }}/v1/audio/transcriptions \
             -XPOST \
             -d '{"byte_str": "UklGRigAAABXQVZFZm10IBIAAAABAAEARKwAAIhYAQACABAAAABkYXRhAgAAAAEA"}' \
-            -H 'Content-Type: application/json'
+            -H 'Content-Type: application/json' && break;
+            curlcode=$?
+            if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi;
+          done;
+          if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi
   restartPolicy: Never
diff --git a/helm-charts/common/data-prep/templates/tests/test-pod.yaml b/helm-charts/common/data-prep/templates/tests/test-pod.yaml
@@ -13,7 +13,6 @@ metadata:
 spec:
   containers:
     - name: curl
-      #image: alpine/curl
       image: python:3.10.14
       command: ['bash', '-c']
       args:
@@ -28,7 +27,8 @@ spec:
             -X POST \
             -H "Content-Type: multipart/form-data" \
             -F "files=@/tmp/file1.txt" && break;
-            sleep 10;
+            curlcode=$?
+            if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi;
           done;
-          if [ $i -gt $max_retry ]; then echo "dataprep test failed."; exit 1; fi
+          if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi
   restartPolicy: Never
diff --git a/helm-charts/common/embedding-usvc/templates/tests/test-pod.yaml b/helm-charts/common/embedding-usvc/templates/tests/test-pod.yaml
@@ -13,13 +13,18 @@ metadata:
 spec:
   containers:
     - name: curl
-      image: alpine/curl
-      #image: python:3.10.14
-      command: ['sh', '-c']
+      image: python:3.10.14
+      command: ['bash', '-c']
       args:
         - |
-          curl http://{{ include "embedding-usvc.fullname" . }}:{{ .Values.service.port }}/v1/embeddings -sS --fail-with-body \
+          max_retry=20;
+          for ((i=1; i<=max_retry; i++)); do
+            curl http://{{ include "embedding-usvc.fullname" . }}:{{ .Values.service.port }}/v1/embeddings -sS --fail-with-body \
             -X POST \
             -d '{"text":"hello"}' \
-            -H 'Content-Type: application/json'
+            -H 'Content-Type: application/json' && break;
+            curlcode=$?
+            if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi;
+          done;
+          if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi
   restartPolicy: Never
diff --git a/helm-charts/common/llm-uservice/templates/tests/test-pod.yaml b/helm-charts/common/llm-uservice/templates/tests/test-pod.yaml
@@ -13,21 +13,26 @@ metadata:
 spec:
   containers:
     - name: curl
-      image: alpine/curl
-      #image: python:3.10.14
-      command: ['sh', '-c']
+      image: python:3.10.14
+      command: ['bash', '-c']
       args:
         - |
+          max_retry=20;
+          for ((i=1; i<=max_retry; i++)); do
           {{- if contains "llm-docsum-tgi" .Values.image.repository }}
           # Try with docsum endpoint
             curl http://{{ include "llm-uservice.fullname" . }}:{{ .Values.service.port }}/v1/chat/docsum -sS --fail-with-body \
               -X POST \
               -d '{"query":"Text Embeddings Inference (TEI) is a toolkit for deploying and serving open source text embeddings and sequence classification models. TEI enables high-performance extraction for the most popular models, including FlagEmbedding, Ember, GTE and E5."}' \
-              -H 'Content-Type: application/json'
+              -H 'Content-Type: application/json' && break;
           {{- else }}
             curl http://{{ include "llm-uservice.fullname" . }}:{{ .Values.service.port }}/v1/chat/completions -sS --fail-with-body \
               -X POST \
               -d '{"query":"What is Deep Learning?","max_new_tokens":17,"top_k":10,"top_p":0.95,"typical_p":0.95,"temperature":0.01,"repetition_penalty":1.03,"streaming":true}' \
-              -H 'Content-Type: application/json'
+              -H 'Content-Type: application/json' && break;
           {{- end }}
+            curlcode=$?
+            if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi;
+          done;
+          if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi
   restartPolicy: Never
diff --git a/helm-charts/common/reranking-usvc/templates/tests/test-pod.yaml b/helm-charts/common/reranking-usvc/templates/tests/test-pod.yaml
@@ -13,13 +13,18 @@ metadata:
 spec:
   containers:
     - name: curl
-      image: alpine/curl
-      #image: python:3.10.14
-      command: ['sh', '-c']
+      image: python:3.10.14
+      command: ['bash', '-c']
       args:
         - |
-          curl http://{{ include "reranking-usvc.fullname" . }}:{{ .Values.service.port }}/v1/reranking -sS --fail-with-body \
+          max_retry=20;
+          for ((i=1; i<=max_retry; i++)); do
+            curl http://{{ include "reranking-usvc.fullname" . }}:{{ .Values.service.port }}/v1/reranking -sS --fail-with-body \
             -X POST \
             -d '{"initial_query":"What is Deep Learning?", "retrieved_docs": [{"text":"Deep Learning is not..."}, {"text":"Deep learning is..."}]}' \
-            -H 'Content-Type: application/json'
+            -H 'Content-Type: application/json' && break;
+            curlcode=$?
+            if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi;
+          done;
+          if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi
   restartPolicy: Never
diff --git a/helm-charts/common/retriever-usvc/templates/tests/test-pod.yaml b/helm-charts/common/retriever-usvc/templates/tests/test-pod.yaml
@@ -13,7 +13,6 @@ metadata:
 spec:
   containers:
     - name: curl
-      #image: alpine/curl
       image: python:3.10.14
       command: ['bash', '-c']
       args:
@@ -25,7 +24,8 @@ spec:
             -X POST \
             -d "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${your_embedding}}" \
             -H 'Content-Type: application/json' && break;
-            sleep 10;
+            curlcode=$?
+            if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi;
           done;
-          if [ $i -gt $max_retry ]; then echo "retriever test failed."; exit 1; fi
+          if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi
   restartPolicy: Never
diff --git a/helm-charts/common/speecht5/templates/tests/test-pod.yaml b/helm-charts/common/speecht5/templates/tests/test-pod.yaml
@@ -13,13 +13,18 @@ metadata:
 spec:
   containers:
     - name: curl
-      image: alpine/curl
-      #image: python:3.10.14
-      command: ['sh', '-c']
+      image: python:3.10.14
+      command: ['bash', '-c']
       args:
         - |
-          curl -sS --fail-with-body http://{{ include "speecht5.fullname" . }}:{{ .Values.service.port }}/v1/tts \
+          max_retry=20;
+          for ((i=1; i<=max_retry; i++)); do
+            curl -sS --fail-with-body http://{{ include "speecht5.fullname" . }}:{{ .Values.service.port }}/v1/tts \
             -X POST \
             -d '{"text": "Who are you?"}' \
-            -H 'Content-Type: application/json'
+            -H 'Content-Type: application/json' && break;
+            curlcode=$?
+            if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi;
+          done;
+          if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi
   restartPolicy: Never
diff --git a/helm-charts/common/tei/templates/tests/test-pod.yaml b/helm-charts/common/tei/templates/tests/test-pod.yaml
@@ -13,13 +13,18 @@ metadata:
 spec:
   containers:
     - name: curl
-      image: alpine/curl
-      #image: python:3.10.14
-      command: ['sh', '-c']
+      image: python:3.10.14
+      command: ['bash', '-c']
       args:
         - |
-          curl http://{{ include "tei.fullname" . }}/embed -sS --fail-with-body \
+          max_retry=20;
+          for ((i=1; i<=max_retry; i++)); do
+            curl http://{{ include "tei.fullname" . }}/embed -sS --fail-with-body \
             -X POST \
             -d '{"inputs":"What is Deep Learning?"}' \
-            -H 'Content-Type: application/json'
+            -H 'Content-Type: application/json' && break;
+            curlcode=$?
+            if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi;
+          done;
+          if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi
   restartPolicy: Never
diff --git a/helm-charts/common/teirerank/templates/tests/test-pod.yaml b/helm-charts/common/teirerank/templates/tests/test-pod.yaml
@@ -13,13 +13,18 @@ metadata:
 spec:
   containers:
     - name: curl
-      image: alpine/curl
-      #image: python:3.10.14
-      command: ['sh', '-c']
+      image: python:3.10.14
+      command: ['bash', '-c']
       args:
         - |
-          curl http://{{ include "teirerank.fullname" . }}/rerank -sS --fail-with-body \
+          max_retry=20;
+          for ((i=1; i<=max_retry; i++)); do
+            curl http://{{ include "teirerank.fullname" . }}/rerank -sS --fail-with-body \
             -X POST \
             -d '{"query":"What is Deep Learning?", "texts": ["Deep Learning is not...", "Deep learning is..."]}' -H 'Content-Type: application/json' \
-            -H 'Content-Type: application/json'
+            -H 'Content-Type: application/json' && break;
+            curlcode=$?
+            if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi;
+          done;
+          if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi
   restartPolicy: Never
diff --git a/helm-charts/common/tgi/templates/tests/test-pod.yaml b/helm-charts/common/tgi/templates/tests/test-pod.yaml
@@ -13,13 +13,18 @@ metadata:
 spec:
   containers:
     - name: curl
-      image: alpine/curl
-      #image: python:3.10.14
-      command: ['sh', '-c']
+      image: python:3.10.14
+      command: ['bash', '-c']
       args:
         - |
-          curl http://{{ include "tgi.fullname" . }}/generate -sS --fail-with-body \
+          max_retry=20;
+          for ((i=1; i<=max_retry; i++)); do
+            curl http://{{ include "tgi.fullname" . }}/generate -sS --fail-with-body \
             -X POST \
             -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' \
-            -H 'Content-Type: application/json'
+            -H 'Content-Type: application/json' && break;
+            curlcode=$?
+            if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi;
+          done;
+          if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi
   restartPolicy: Never
diff --git a/helm-charts/common/tts/templates/tests/test-pod.yaml b/helm-charts/common/tts/templates/tests/test-pod.yaml
@@ -13,13 +13,18 @@ metadata:
 spec:
   containers:
     - name: curl
-      image: alpine/curl
-      #image: python:3.10.14
-      command: ['sh', '-c']
+      image: python:3.10.14
+      command: ['bash', '-c']
       args:
         - |
-          curl -sS --fail-with-body http://{{ include "tts.fullname" . }}:{{ .Values.service.port }}/v1/audio/speech \
+          max_retry=20;
+          for ((i=1; i<=max_retry; i++)); do
+            curl -sS --fail-with-body http://{{ include "tts.fullname" . }}:{{ .Values.service.port }}/v1/audio/speech \
             -XPOST \
             -d '{"text": "Who are you?"}' \
-            -H 'Content-Type: application/json'
+            -H 'Content-Type: application/json' && break;
+            curlcode=$?
+            if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi;
+          done;
+          if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi
   restartPolicy: Never
diff --git a/helm-charts/common/web-retriever/templates/tests/test-pod.yaml b/helm-charts/common/web-retriever/templates/tests/test-pod.yaml
@@ -13,14 +13,19 @@ metadata:
 spec:
   containers:
     - name: curl
-      #image: alpine/curl
       image: python:3.10.14
-      command: ['sh', '-c']
+      command: ['bash', '-c']
       args:
         - |
           your_embedding=$(python -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)");
-          curl -sS --fail-with-body http://{{ include "web-retriever.fullname" . }}:{{ .Values.service.port }}/v1/web_retrieval \
+          max_retry=20;
+          for ((i=1; i<=max_retry; i++)); do
+            curl -sS --fail-with-body http://{{ include "web-retriever.fullname" . }}:{{ .Values.service.port }}/v1/web_retrieval \
           -X POST \
           -d "{\"text\":\"What is OPEA?\",\"embedding\":${your_embedding}}" \
-                -H 'Content-Type: application/json'
+                -H 'Content-Type: application/json' && break;
+            curlcode=$?
+            if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi;
+          done;
+          if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi
   restartPolicy: Never
diff --git a/helm-charts/common/whisper/templates/tests/test-pod.yaml b/helm-charts/common/whisper/templates/tests/test-pod.yaml
@@ -13,13 +13,18 @@ metadata:
 spec:
   containers:
     - name: curl
-      image: alpine/curl
-      #image: python:3.10.14
-      command: ['sh', '-c']
+      image: python:3.10.14
+      command: ['bash', '-c']
       args:
         - |
-          curl -sS --fail-with-body http://{{ include "whisper.fullname" . }}:{{ .Values.service.port }}/v1/asr \
+          max_retry=20;
+          for ((i=1; i<=max_retry; i++)); do
+            curl -sS --fail-with-body http://{{ include "whisper.fullname" . }}:{{ .Values.service.port }}/v1/asr \
             -X POST \
             -d '{"audio": "UklGRigAAABXQVZFZm10IBIAAAABAAEARKwAAIhYAQACABAAAABkYXRhAgAAAAEA"}' \
-            -H 'Content-Type: application/json'
+            -H 'Content-Type: application/json' && break;
+            curlcode=$?
+            if [[ $curlcode -eq 7 ]]; then sleep 10; else echo "curl failed with code $curlcode"; exit 1; fi;
+          done;
+          if [ $i -gt $max_retry ]; then echo "test failed with maximum retry"; exit 1; fi
   restartPolicy: Never