Merge branch 'main' into webhook

opea-project · Aug 12, 2024 · 2c67857 · 2c67857
2 parents 4e8187d + 7b20273
commit 2c67857
Show file tree

Hide file tree

Showing 3 changed files with 203 additions and 33 deletions.
diff --git a/.github/workflows/scripts/e2e/gmc_gaudi_test.sh b/.github/workflows/scripts/e2e/gmc_gaudi_test.sh
@@ -11,6 +11,7 @@ USER_ID=$(whoami)
 LOG_PATH=/home/$(whoami)/logs
 CHATQNA_NAMESPACE="${APP_NAMESPACE}-chatqna"
 CHATQNA_DATAPREP_NAMESPACE="${APP_NAMESPACE}-chatqna-dataprep"
+CHATQNA_SWITCH_NAMESPACE="${APP_NAMESPACE}-chatqna-switch"
 CODEGEN_NAMESPACE="${APP_NAMESPACE}-codegen"
 CODETRANS_NAMESPACE="${APP_NAMESPACE}-codetrans"
 DOCSUM_NAMESPACE="${APP_NAMESPACE}-docsum"
@@ -22,21 +23,25 @@ function validate_gmc() {
     echo "validate chat-qna with dataprep"
     validate_chatqna_with_dataprep
 
-    echo "validate codegen"
-    validate_codegen
+    echo "validate chat-qna in switch mode"
+    validate_chatqna_in_switch
 
-    echo "validate codetrans"
-    validate_codetrans
+    # echo "validate codegen"
+    # validate_codegen
 
-    echo "validate docsum"
-    validate_docsum
+    # echo "validate codetrans"
+    # validate_codetrans
+
+    # echo "validate docsum"
+    # validate_docsum
 
     get_gmc_controller_logs
 }
 
 function cleanup_apps() {
     echo "clean up microservice-connector"
-    namespaces=("$CHATQNA_NAMESPACE" "$CHATQNA_DATAPREP_NAMESPACE" "$CODEGEN_NAMESPACE" "$CODETRANS_NAMESPACE" "$DOCSUM_NAMESPACE")
+    # namespaces=("$CHATQNA_NAMESPACE" "$CHATQNA_DATAPREP_NAMESPACE" "$CHATQNA_SWITCH_NAMESPACE" "$CODEGEN_NAMESPACE" "$CODETRANS_NAMESPACE" "$DOCSUM_NAMESPACE")
+    namespaces=("$CHATQNA_NAMESPACE" "$CHATQNA_DATAPREP_NAMESPACE" "$CHATQNA_SWITCH_NAMESPACE")
     for ns in "${namespaces[@]}"; do
         if kubectl get namespace $ns > /dev/null 2>&1; then
             echo "Deleting namespace: $ns"
@@ -101,7 +106,7 @@ function validate_chatqna() {
 }
 
 function validate_chatqna_with_dataprep() {
-    kubectl create ns $CHATQNA_DATAPREP_NAMESPACE
+   kubectl create ns $CHATQNA_DATAPREP_NAMESPACE
    sed -i "s|namespace: chatqa|namespace: $CHATQNA_DATAPREP_NAMESPACE|g"  $(pwd)/config/samples/chatQnA_dataprep_gaudi.yaml
    # workaround for issue #268
    yq -i '(.spec.nodes.root.steps[] | select ( .name == "Tgi")).internalService.config.MODEL_ID = "bigscience/bloom-560m"' $(pwd)/config/samples/chatQnA_dataprep_gaudi.yaml
@@ -178,6 +183,87 @@ function validate_chatqna_with_dataprep() {
    fi
 }
 
+function validate_chatqna_in_switch() {
+   kubectl create ns $CHATQNA_SWITCH_NAMESPACE
+   sed -i "s|namespace: switch|namespace: $CHATQNA_SWITCH_NAMESPACE|g"  $(pwd)/config/samples/chatQnA_switch_gaudi.yaml
+   # workaround for issue #268
+   yq -i '(.spec.nodes.root.steps[] | select ( .name == "Tgi")).internalService.config.MODEL_ID = "bigscience/bloom-560m"' $(pwd)/config/samples/chatQnA_switch_gaudi.yaml
+   kubectl apply -f $(pwd)/config/samples/chatQnA_switch_gaudi.yaml
+
+   # Wait until the router service is ready
+   echo "Waiting for the chatqa router service to be ready..."
+   wait_until_pod_ready "chatqna router" $CHATQNA_SWITCH_NAMESPACE "router-service"
+   output=$(kubectl get pods -n $CHATQNA_SWITCH_NAMESPACE)
+   echo $output
+
+   # deploy client pod for testing
+   kubectl create deployment client-test -n $CHATQNA_SWITCH_NAMESPACE --image=python:3.8.13 -- sleep infinity
+
+   # Wait until all pods are ready
+   wait_until_all_pod_ready $CHATQNA_SWITCH_NAMESPACE 300s
+   if [ $? -ne 0 ]; then
+       echo "Error Some pods are not ready!"
+       exit 1
+   fi
+
+   # giving time to populating data
+   sleep 90
+
+   kubectl get pods -n $CHATQNA_SWITCH_NAMESPACE
+   # send request to chatqnA
+   export CLIENT_POD=$(kubectl get pod -n $CHATQNA_SWITCH_NAMESPACE -l app=client-test -o jsonpath={.items..metadata.name})
+   echo "$CLIENT_POD"
+   accessUrl=$(kubectl get gmc -n $CHATQNA_SWITCH_NAMESPACE -o jsonpath="{.items[?(@.metadata.name=='switch')].status.accessUrl}")
+
+   # test the chatqna with model condition: "model-id":"intel" and "embedding-model-id":"small"
+   kubectl exec "$CLIENT_POD" -n $CHATQNA_SWITCH_NAMESPACE -- curl $accessUrl  -X POST  -d '{"text":"What is the revenue of Nike in 2023?", "model-id":"intel", "embedding-model-id":"small", "parameters":{"max_new_tokens":17, "do_sample": true}}' -H 'Content-Type: application/json' > $LOG_PATH/curl_chatqna_switch_intel.log
+   exit_code=$?
+   if [ $exit_code -ne 0 ]; then
+       echo "chatqna failed, please check the logs in ${LOG_PATH}!"
+       exit 1
+   fi
+
+   echo "Checking response results, make sure the output is reasonable. "
+   local status=false
+   if [[ -f $LOG_PATH/curl_chatqna_switch_intel.log ]] && \
+   [[ $(grep -c "[DONE]" $LOG_PATH/curl_chatqna_switch_intel.log) != 0 ]]; then
+       status=true
+   fi
+   if [ $status == false ]; then
+       if [[ -f $LOG_PATH/curl_chatqna_switch_intel.log ]]; then
+           cat $LOG_PATH/curl_chatqna_switch_intel.log
+       fi
+       echo "Response check failed, please check the logs in artifacts!"
+       exit 1
+   else
+       echo "Response check succeed!"
+   fi
+
+   # test the chatqna with model condition: "model-id":"llama" and "embedding-model-id":"large"
+   kubectl exec "$CLIENT_POD" -n $CHATQNA_SWITCH_NAMESPACE -- curl $accessUrl  -X POST  -d '{"text":"What is the revenue of Nike in 2023?", "model-id":"llama", "embedding-model-id":"large", "parameters":{"max_new_tokens":17, "do_sample": true}}' -H 'Content-Type: application/json' > $LOG_PATH/curl_chatqna_switch_llama.log
+   exit_code=$?
+   if [ $exit_code -ne 0 ]; then
+       echo "chatqna failed, please check the logs in ${LOG_PATH}!"
+       exit 1
+   fi
+
+   echo "Checking response results, make sure the output is reasonable. "
+   local status=false
+   if [[ -f $LOG_PATH/curl_chatqna_switch_llama.log ]] && \
+   [[ $(grep -c "[DONE]" $LOG_PATH/curl_chatqna_switch_llama.log) != 0 ]]; then
+       status=true
+   fi
+   if [ $status == false ]; then
+       if [[ -f $LOG_PATH/curl_chatqna_switch_llama.log ]]; then
+           cat $LOG_PATH/curl_chatqna_switch_llama.log
+       fi
+       echo "Response check failed, please check the logs in artifacts!"
+       exit 1
+   else
+       echo "Response check succeed!"
+   fi
+}
+
 function validate_codegen() {
    kubectl create ns $CODEGEN_NAMESPACE
    sed -i "s|namespace: codegen|namespace: $CODEGEN_NAMESPACE|g"  $(pwd)/config/samples/codegen_gaudi.yaml

diff --git a/.github/workflows/scripts/e2e/gmc_xeon_test.sh b/.github/workflows/scripts/e2e/gmc_xeon_test.sh
@@ -11,6 +11,7 @@ USER_ID=$(whoami)
 LOG_PATH=/home/$(whoami)/logs
 CHATQNA_NAMESPACE="${APP_NAMESPACE}-chatqna"
 CHATQNA_DATAPREP_NAMESPACE="${APP_NAMESPACE}-chatqna-dataprep"
+CHATQNA_SWITCH_NAMESPACE="${APP_NAMESPACE}-chatqna-switch"
 CODEGEN_NAMESPACE="${APP_NAMESPACE}-codegen"
 CODETRANS_NAMESPACE="${APP_NAMESPACE}-codetrans"
 DOCSUM_NAMESPACE="${APP_NAMESPACE}-docsum"
@@ -22,21 +23,25 @@ function validate_gmc() {
     echo "validate chat-qna with dataprep"
     validate_chatqna_with_dataprep
 
-    echo "validate codegen"
-    validate_codegen
+    echo "validate chat-qna in switch mode"
+    validate_chatqna_in_switch
 
-    echo "validate codetrans"
-    validate_codetrans
+    # echo "validate codegen"
+    # validate_codegen
 
-    echo "validate docsum"
-    validate_docsum
+    # echo "validate codetrans"
+    # validate_codetrans
+
+    # echo "validate docsum"
+    # validate_docsum
 
     get_gmc_controller_logs
 }
 
 function cleanup_apps() {
     echo "clean up microservice-connector"
-    namespaces=("$CHATQNA_NAMESPACE" "$CHATQNA_DATAPREP_NAMESPACE" "$CODEGEN_NAMESPACE" "$CODETRANS_NAMESPACE" "$DOCSUM_NAMESPACE")
+    # namespaces=("$CHATQNA_NAMESPACE" "$CHATQNA_DATAPREP_NAMESPACE" "$CHATQNA_SWITCH_NAMESPACE" "$CODEGEN_NAMESPACE" "$CODETRANS_NAMESPACE" "$DOCSUM_NAMESPACE")
+    namespaces=("$CHATQNA_NAMESPACE" "$CHATQNA_DATAPREP_NAMESPACE" "$CHATQNA_SWITCH_NAMESPACE")
     for ns in "${namespaces[@]}"; do
         if kubectl get namespace $ns > /dev/null 2>&1; then
             echo "Deleting namespace: $ns"
@@ -103,7 +108,7 @@ function validate_chatqna() {
 }
 
 function validate_chatqna_with_dataprep() {
-    kubectl create ns $CHATQNA_DATAPREP_NAMESPACE
+   kubectl create ns $CHATQNA_DATAPREP_NAMESPACE
    sed -i "s|namespace: chatqa|namespace: $CHATQNA_DATAPREP_NAMESPACE|g"  $(pwd)/config/samples/chatQnA_dataprep_xeon.yaml
    # workaround for issue #268
    yq -i '(.spec.nodes.root.steps[] | select ( .name == "Tgi")).internalService.config.MODEL_ID = "bigscience/bloom-560m"' $(pwd)/config/samples/chatQnA_dataprep_xeon.yaml
@@ -180,6 +185,87 @@ function validate_chatqna_with_dataprep() {
    fi
 }
 
+function validate_chatqna_in_switch() {
+   kubectl create ns $CHATQNA_SWITCH_NAMESPACE
+   sed -i "s|namespace: switch|namespace: $CHATQNA_SWITCH_NAMESPACE|g"  $(pwd)/config/samples/chatQnA_switch_xeon.yaml
+   # workaround for issue #268
+   yq -i '(.spec.nodes.root.steps[] | select ( .name == "Tgi")).internalService.config.MODEL_ID = "bigscience/bloom-560m"' $(pwd)/config/samples/chatQnA_switch_xeon.yaml
+   kubectl apply -f $(pwd)/config/samples/chatQnA_switch_xeon.yaml
+
+   # Wait until the router service is ready
+   echo "Waiting for the chatqa router service to be ready..."
+   wait_until_pod_ready "chatqna router" $CHATQNA_SWITCH_NAMESPACE "router-service"
+   output=$(kubectl get pods -n $CHATQNA_SWITCH_NAMESPACE)
+   echo $output
+
+   # deploy client pod for testing
+   kubectl create deployment client-test -n $CHATQNA_SWITCH_NAMESPACE --image=python:3.8.13 -- sleep infinity
+
+   # Wait until all pods are ready
+   wait_until_all_pod_ready $CHATQNA_SWITCH_NAMESPACE 300s
+   if [ $? -ne 0 ]; then
+       echo "Error Some pods are not ready!"
+       exit 1
+   fi
+
+   # giving time to populating data
+   sleep 90
+
+   kubectl get pods -n $CHATQNA_SWITCH_NAMESPACE
+   # send request to chatqnA
+   export CLIENT_POD=$(kubectl get pod -n $CHATQNA_SWITCH_NAMESPACE -l app=client-test -o jsonpath={.items..metadata.name})
+   echo "$CLIENT_POD"
+   accessUrl=$(kubectl get gmc -n $CHATQNA_SWITCH_NAMESPACE -o jsonpath="{.items[?(@.metadata.name=='switch')].status.accessUrl}")
+
+   # test the chatqna with model condition: "model-id":"intel" and "embedding-model-id":"small"
+   kubectl exec "$CLIENT_POD" -n $CHATQNA_SWITCH_NAMESPACE -- curl $accessUrl  -X POST  -d '{"text":"What is the revenue of Nike in 2023?", "model-id":"intel", "embedding-model-id":"small", "parameters":{"max_new_tokens":17, "do_sample": true}}' -H 'Content-Type: application/json' > $LOG_PATH/curl_chatqna_switch_intel.log
+   exit_code=$?
+   if [ $exit_code -ne 0 ]; then
+       echo "chatqna failed, please check the logs in ${LOG_PATH}!"
+       exit 1
+   fi
+
+   echo "Checking response results, make sure the output is reasonable. "
+   local status=false
+   if [[ -f $LOG_PATH/curl_chatqna_switch_intel.log ]] && \
+   [[ $(grep -c "[DONE]" $LOG_PATH/curl_chatqna_switch_intel.log) != 0 ]]; then
+       status=true
+   fi
+   if [ $status == false ]; then
+       if [[ -f $LOG_PATH/curl_chatqna_switch_intel.log ]]; then
+           cat $LOG_PATH/curl_chatqna_switch_intel.log
+       fi
+       echo "Response check failed, please check the logs in artifacts!"
+       exit 1
+   else
+       echo "Response check succeed!"
+   fi
+
+   # test the chatqna with model condition: "model-id":"llama" and "embedding-model-id":"large"
+   kubectl exec "$CLIENT_POD" -n $CHATQNA_SWITCH_NAMESPACE -- curl $accessUrl  -X POST  -d '{"text":"What is the revenue of Nike in 2023?", "model-id":"llama", "embedding-model-id":"large", "parameters":{"max_new_tokens":17, "do_sample": true}}' -H 'Content-Type: application/json' > $LOG_PATH/curl_chatqna_switch_llama.log
+   exit_code=$?
+   if [ $exit_code -ne 0 ]; then
+       echo "chatqna failed, please check the logs in ${LOG_PATH}!"
+       exit 1
+   fi
+
+   echo "Checking response results, make sure the output is reasonable. "
+   local status=false
+   if [[ -f $LOG_PATH/curl_chatqna_switch_llama.log ]] && \
+   [[ $(grep -c "[DONE]" $LOG_PATH/curl_chatqna_switch_llama.log) != 0 ]]; then
+       status=true
+   fi
+   if [ $status == false ]; then
+       if [[ -f $LOG_PATH/curl_chatqna_switch_llama.log ]]; then
+           cat $LOG_PATH/curl_chatqna_switch_llama.log
+       fi
+       echo "Response check failed, please check the logs in artifacts!"
+       exit 1
+   else
+       echo "Response check succeed!"
+   fi
+}
+
 function validate_codegen() {
    kubectl create ns $CODEGEN_NAMESPACE
    sed -i "s|namespace: codegen|namespace: $CODEGEN_NAMESPACE|g"  $(pwd)/config/samples/codegen_xeon.yaml

diff --git a/microservices-connector/cmd/router/main.go b/microservices-connector/cmd/router/main.go
@@ -273,8 +273,8 @@ func handleSwitchPipeline(nodeName string,
 ) (io.ReadCloser, int, error) {
 	currentNode := graph.Spec.Nodes[nodeName]
 	var statusCode int
-	var responseBytes []byte
 	var responseBody io.ReadCloser
+	var responseBytes []byte
 	var err error
 
 	initReqData := make(map[string]interface{})
@@ -294,6 +294,14 @@ func handleSwitchPipeline(nodeName string,
 			)
 			continue
 		}
+
+		// make sure that the process goes to the correct step
+		if route.Condition != "" {
+			if !pickupRouteByCondition(initInput, route.Condition) {
+				continue
+			}
+		}
+
 		log.Info("Current Step Information", "Node Name", nodeName, "Step Index", index)
 		request := input
 		if responseBody != nil {
@@ -315,18 +323,9 @@ func handleSwitchPipeline(nodeName string,
 			request = mergeRequests(responseBytes, initReqData)
 		}
 		log.Info("Print New Request Bytes", "Request Bytes", request)
-		if route.Condition == "" {
-			responseBody, statusCode, err = handleSwitchNode(&route, graph, initInput, request, headers)
-			if err != nil {
-				return nil, statusCode, err
-			}
-		} else {
-			if pickupRouteByCondition(initInput, route.Condition) {
-				responseBody, statusCode, err = handleSwitchNode(&route, graph, initInput, request, headers)
-				if err != nil {
-					return nil, statusCode, err
-				}
-			}
+		responseBody, statusCode, err = handleSwitchNode(&route, graph, initInput, request, headers)
+		if err != nil {
+			return nil, statusCode, err
 		}
 	}
 	return responseBody, statusCode, err
@@ -562,21 +561,20 @@ func mcGraphHandler(w http.ResponseWriter, req *http.Request) {
 				break
 			}
 
-			sliceBF := buffer[:n]
+			/*sliceBF := buffer[:n]
 			if !bytes.HasPrefix(sliceBF, DONE) {
 				sliceBF = bytes.TrimPrefix(sliceBF, Prefix)
 				sliceBF = bytes.TrimSuffix(sliceBF, Suffix)
 			} else {
 				sliceBF = bytes.Join([][]byte{Newline, sliceBF}, nil)
 			}
+			log.Info("[llm - chat_stream] chunk:", "Buffer", string(sliceBF))*/
 
-			log.Info("[llm - chat_stream] chunk:", "Buffer", string(sliceBF))
 			// Write the chunk to the ResponseWriter
-			if _, err := w.Write(sliceBF); err != nil {
+			if _, err := w.Write(buffer[:n]); err != nil {
 				log.Error(err, "failed to write to ResponseWriter")
 				return
 			}
-
 			// Flush the data to the client immediately
 			if flusher, ok := w.(http.Flusher); ok {
 				flusher.Flush()