Merge branch 'addUT' of https://github.com/KfreeZ/GenAIInfra into addUT

.github/workflows/_helm-e2e.yaml

@@ -19,6 +19,10 @@ on:
         default: "xeon"
         required: true
         type: string
+      valuefile:
+        default: "values"
+        required: true
+        type: string
 
 env:
   CHARTS_DIR: "helm-charts"
@@ -96,10 +100,7 @@ jobs:
         run: |
           set -xe
           echo "should_cleanup=true" >> $GITHUB_ENV
-          value_file="values.yaml"
-          if [ "${{ inputs.hardware }}" == "gaudi" ]; then
-            value_file="gaudi-values.yaml"
-          fi
+          value_file="${{ inputs.valuefile }}.yaml"
           if [[ ! -f ${{ env.CHART_FOLDER }}/${value_file} ]]; then
             echo "No value file found, exist test!"
             echo "skip_validate=true" >> $GITHUB_ENV

.github/workflows/manual-helm-cd-workflow.yaml

@@ -15,11 +15,11 @@ on:
         description: "Image tag to be tested"
         required: true
         type: string
-      nodes:
-        default: "xeon,gaudi"
-        required: true
+      opea_branch:
+        default: "main"
+        description: 'OPEA branch for image build'
+        required: false
         type: string
-        description: 'Hardwares used to run tests'
 
 env:
   CHARTS_DIR: "helm-charts"
@@ -28,41 +28,80 @@ jobs:
   get-build-matrix:
     runs-on: ubuntu-latest
     outputs:
-      services: ${{ steps.get-services.outputs.services }}
-      nodes: ${{ steps.get-services.outputs.nodes }}
+      run_matrix: ${{ steps.get-services.outputs.run_matrix }}
     steps:
       - name: Checkout out Repo
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          ref: ${{ inputs.opea_branch }}
 
       - name: Get test Services
         id: get-services
         run: |
           set -x
+          run_matrix="{\"include\":["
           if [[ -z "${{ inputs.workloads }}" ]]; then
-            subfolders=$(find "$CHARTS_DIR" -mindepth 1 -maxdepth 1 -type d ! -name "common" -exec basename {} \; | sed 's/^/"/; s/$/"/' | paste -sd, -)
-            common_subfolders=$(find "$CHARTS_DIR/common" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sed 's|^|"common/|; s/$/"/' | paste -sd, -)
-            services="[$subfolders,$common_subfolders]"
-            echo "services=$services" >> $GITHUB_OUTPUT
+            find "$CHARTS_DIR" -mindepth 1 -maxdepth 1 -type d ! -name "common" -exec basename {} \; | while read -r subfolder; do
+              for file in "$CHARTS_DIR/$subfolder"/*values.yaml; do
+                if [ -f "$file" ]; then
+                  if [[ "$file" == *"nv-values.yaml" ]]; then
+                    continue
+                  fi
+                  filename=$(basename "$file" .yaml)
+                  if [[ "$filename" == *"gaudi"* ]]; then
+                    run_matrix="${run_matrix}{\"example\":\"${subfolder}\",\"hardware\":\"gaudi\", \"valuefile\":\"${filename}\"},"
+                  else
+                    run_matrix="${run_matrix}{\"example\":\"${subfolder}\",\"hardware\":\"xeon\", \"valuefile\":\"${filename}\"},"
+                  fi
+                fi
+              done
+            done
+            find "$CHARTS_DIR/common" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | while read -r subfolder; do
+              for file in "$CHARTS_DIR/common/$subfolder"/*values.yaml; do
+                if [ -f "$file" ]; then
+                  if [[ "$file" == *"nv-values.yaml" ]]; then
+                    continue
+                  fi
+                  filename=$(basename "$file" .yaml)
+                  if [[ "$filename" == *"gaudi"* ]]; then
+                    run_matrix="${run_matrix}{\"example\":\"common/${subfolder}\",\"hardware\":\"gaudi\", \"valuefile\":\"${filename}\"},"
+                  else
+                    run_matrix="${run_matrix}{\"example\":\"common/${subfolder}\",\"hardware\":\"xeon\", \"valuefile\":\"${filename}\"},"
+                  fi
+                fi
+              done
+            done
           else
             service_list=($(echo ${{ github.event.inputs.workloads }} | tr ',' ' '))
-            services=$(printf '%s\n' "${service_list[@]}" | sort -u | jq -R '.' | jq -sc '.')
-            echo "services=$services" >> $GITHUB_OUTPUT
+            for service in $service_list; do
+              for file in "$CHARTS_DIR/$service"/*values.yaml; do
+                if [ -f "$file" ]; then
+                  if [[ "$file" == *"nv-values.yaml" ]]; then
+                    continue
+                  fi
+                  filename=$(basename "$file" .yaml)
+                  if [[ "$filename" == *"gaudi"* ]]; then
+                    run_matrix="${run_matrix}{\"example\":\"$service\",\"hardware\":\"gaudi\", \"valuefile\":\"${filename}\"},"
+                  else
+                    run_matrix="${run_matrix}{\"example\":\"$service\",\"hardware\":\"xeon\", \"valuefile\":\"${filename}\"},"
+                  fi
+                fi
+              done
+            done
           fi
-          node_list=($(echo ${{ github.event.inputs.nodes }} | tr ',' ' '))
-          nodes=$(printf '%s\n' "${node_list[@]}" | sort -u | jq -R '.' | jq -sc '.')
-          echo "nodes=$nodes" >> $GITHUB_OUTPUT
+          run_matrix=$run_matrix"]}"
+          echo "run_matrix=${run_matrix}"
+          echo "run_matrix=${run_matrix}" >> $GITHUB_OUTPUT
 
   helm-release:
     needs: get-build-matrix
     strategy:
-      matrix:
-        workload: ${{ fromJSON(needs.get-build-matrix.outputs.services) }}
-        node: ${{ fromJSON(needs.get-build-matrix.outputs.nodes) }}
+      matrix: ${{ fromJSON(needs.get-build-matrix.outputs.run_matrix) }}
     uses: ./.github/workflows/_helm-e2e.yaml
     with:
       tag: ${{ inputs.tag }}
-      workload: ${{ matrix.workload }}
-      hardware: ${{ matrix.node }}
+      workload: ${{ matrix.example }}
+      hardware: ${{ matrix.hardware }}
+      valuefile: ${{ matrix.valuefile }}
     secrets: inherit

.github/workflows/pr-chart-e2e.yaml

@@ -47,16 +47,34 @@ jobs:
           cut -d'/' -f3 | sort -u )
           run_matrix="{\"include\":["
           for chart in ${e2e_charts}; do
-            if [ -f $CHARTS_DIR/$chart/gaudi-values.yaml ]; then
-              run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"gaudi\"},"
-            fi
-            run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"xeon\"},"
+            for file in "$CHARTS_DIR/$chart"/*values.yaml; do
+              if [ -f "$file" ]; then
+                if [[ "$file" == *"nv-values.yaml" ]]; then
+                  continue
+                fi
+                filename=$(basename "$file" .yaml)
+                if [[ "$filename" == *"gaudi"* ]]; then
+                  run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"gaudi\", \"valuefile\":\"${filename}\"},"
+                else
+                  run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"xeon\", \"valuefile\":\"${filename}\"},"
+                fi
+              fi
+            done
           done
           for chart in ${common_charts}; do
-            if [ -f $CHARTS_DIR/common/$chart/gaudi-values.yaml ]; then
-              run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"gaudi\",\"directory\":\"common\"},"
-            fi
-            run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"xeon\",\"directory\":\"common\"},"
+            for file in "$CHARTS_DIR/common/$chart"/*values.yaml; do
+              if [ -f "$file" ]; then
+                if [[ "$file" == *"nv-values.yaml" ]]; then
+                  continue
+                fi
+                filename=$(basename "$file" .yaml)
+                if [[ "$filename" == *"gaudi"* ]]; then
+                  run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"gaudi\", \"valuefile\":\"${filename}\",\"directory\":\"common\"},"
+                else
+                  run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"xeon\", \"valuefile\":\"${filename}\",\"directory\":\"common\"},"
+                fi
+              fi
+            done
           done
           run_matrix=$run_matrix"]}"
           echo "run_matrix=${run_matrix}"
@@ -71,134 +89,5 @@ jobs:
     with:
       workload: ${{ matrix.directory }}/${{ matrix.example }}
       hardware: ${{ matrix.hardware }}
-      # tag: ${{ needs.image-build.outputs.image_tag }}
-      # opea-branch: "main"
+      valuefile: ${{ matrix.valuefile }}
     secrets: inherit
-
-    # needs: job1
-    # if: always() && ${{ needs.job1.outputs.run_matrix.example.length }} > 0
-    # strategy:
-    #   matrix: ${{ fromJSON(needs.job1.outputs.run_matrix) }}
-    # runs-on: ${{ matrix.hardware }}
-    # continue-on-error: true
-    # outputs:
-    #   should_cleanup: ${{ steps.set_boolean.outputs.should_cleanup }}
-    # steps:
-    #   - name: E2e test chart
-    #     run: |
-    #       echo "Matrix - chart: ${{ matrix.example }}"
-
-    #   - name: Clean Up Working Directory
-    #     run: sudo rm -rf ${{github.workspace}}/*
-
-    #   - name: Checkout out Repo
-    #     uses: actions/checkout@v4
-    #     with:
-    #       ref: "refs/pull/${{ github.event.number }}/merge"
-
-    #   - name: Set variables
-    #     run: |
-    #       echo "RELEASE_NAME=${{ matrix.example }}$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV
-    #       echo "NAMESPACE=${{ matrix.example }}-$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV
-    #       echo "ROLLOUT_TIMEOUT_SECONDS=600s" >> $GITHUB_ENV
-    #       echo "KUBECTL_TIMEOUT_SECONDS=60s" >> $GITHUB_ENV
-    #       echo "should_cleanup=false" >> $GITHUB_ENV
-    #       echo "skip_validate=false" >> $GITHUB_ENV
-    #       echo "RELEASENAME=$RELEASE_NAME"
-    #       echo "NAMESPACE=$NAMESPACE"
-    #       if [ -n "${{ matrix.directory }}" ]; then
-    #         echo "CHART_FOLDER=$CHARTS_DIR/${{ matrix.directory }}/${{ matrix.example }}" >> $GITHUB_ENV
-    #       else
-    #         echo "CHART_FOLDER=$CHARTS_DIR/${{ matrix.example }}" >> $GITHUB_ENV
-    #       fi
-
-    #   - name: Initialize chart testing
-    #     run: |
-    #       # Replace values for CI test environment
-    #       USER_ID=$(whoami)
-    #       CHART_MOUNT=/home/$USER_ID/.cache/huggingface/hub
-    #       HFTOKEN=$(cat /home/$USER_ID/.cache/huggingface/token)
-    #       pushd helm-charts
-    #       # insert a prefix before opea/.*, the prefix is OPEA_IMAGE_REPO
-    #       find . -name '*values.yaml' -type f -exec sed -i "s#repository: opea/*#repository: ${OPEA_IMAGE_REPO}opea/#g" {} \;
-    #       # set OPEA image tag to latest
-    #       find . -name '*values.yaml' -type f -exec sed -i 's#tag: ""#tag: latest#g' {} \;
-    #       # set huggingface token
-    #       find . -name '*values.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#${HFTOKEN}#g" {} \;
-    #       # replace the mount dir "Volume: *" with "Volume: $CHART_MOUNT"
-    #       find . -name '*values.yaml' -type f -exec sed -i "s#modelUseHostPath: .*#modelUseHostPath: $CHART_MOUNT#g" {} \;
-    #       # replace the pull policy "IfNotPresent" with "Always"
-    #       find . -name '*values.yaml' -type f -exec sed -i "s#pullPolicy: IfNotPresent#pullPolicy: Always#g" {} \;
-    #       popd
-
-    #   - name: Helm install
-    #     id: install
-    #     env:
-    #       GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }}
-    #       GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
-    #     run: |
-    #       set -xe
-    #       echo "should_cleanup=true" >> $GITHUB_ENV
-    #       helm-charts/update_dependency.sh && helm dependency update ${{ env.CHART_FOLDER}}
-    #       value_file="values.yaml"
-    #       if [ "${{ matrix.hardware }}" == "gaudi" ]; then
-    #         value_file="gaudi-values.yaml"
-    #       fi
-    #       if ! helm install --create-namespace --namespace $NAMESPACE --wait \
-    #           --timeout "$ROLLOUT_TIMEOUT_SECONDS" \
-    #           --set autodependency.enabled=true \
-    #           --set GOOGLE_API_KEY=${{ env.GOOGLE_API_KEY}} \
-    #           --set GOOGLE_CSE_ID=${{ env.GOOGLE_CSE_ID}} \
-    #           --values ${{ env.CHART_FOLDER}}/${value_file} \
-    #           $RELEASE_NAME ${{ env.CHART_FOLDER}} ; then
-    #         echo "Failed to install chart ${{ matrix.example }}"
-    #         echo "skip_validate=true" >> $GITHUB_ENV
-    #         .github/workflows/scripts/e2e/chart_test.sh dump_pods_status $NAMESPACE
-    #         exit 1
-    #       fi
-
-    #   - name: Validate e2e test
-    #     if: always()
-    #     run: |
-    #       set -xe
-    #       if $skip_validate; then
-    #         echo "Skip validate"
-    #       else
-    #         LOG_PATH=/home/$(whoami)/logs
-    #         chart=${{ matrix.example }}
-    #         helm test -n $NAMESPACE $RELEASE_NAME --logs |tee ${LOG_PATH}/charts-${chart}.log
-    #         exit_code=$?
-    #         if [ $exit_code -ne 0 ]; then
-    #             echo "Chart ${chart} test failed, please check the logs in ${LOG_PATH}!"
-    #             exit 1
-    #         fi
-
-    #         echo "Checking response results, make sure the output is reasonable. "
-    #         teststatus=false
-    #         if [[ -f $LOG_PATH/charts-${chart}.log ]] && \
-    #         [[ $(grep -c "^Phase:.*Failed" $LOG_PATH/charts-${chart}.log) != 0 ]]; then
-    #             teststatus=false
-    #             .github/workflows/scripts/e2e/chart_test.sh dump_all_pod_logs $NAMESPACE
-    #         else
-    #             teststatus=true
-    #         fi
-
-    #         if [ $teststatus == false ]; then
-    #             echo "Response check failed, please check the logs in artifacts!"
-    #             exit 1
-    #         else
-    #             echo "Response check succeed!"
-    #             exit 0
-    #         fi
-    #       fi
-
-    #   - name: Helm uninstall
-    #     if: always()
-    #     run: |
-    #       if $should_cleanup; then
-    #         helm uninstall $RELEASE_NAME --namespace $NAMESPACE
-    #         if ! kubectl delete ns $NAMESPACE --timeout=$KUBECTL_TIMEOUT_SECONDS; then
-    #           kubectl delete pods --namespace $NAMESPACE --force --grace-period=0 --all
-    #           kubectl delete ns $NAMESPACE --force --grace-period=0 --timeout=$KUBECTL_TIMEOUT_SECONDS
-    #         fi
-    #       fi

authN-authZ/auth-istio/README.md

@@ -45,7 +45,7 @@ The istio ingress gateway will be used to access the chatQnA service in differen
 
 Authentication and authorization are essential for securing microservices architectures. Using Bearer JWT tokens for these processes ensures that only authenticated users with valid tokens can access specific services, protecting sensitive data. Authentication verifies user identity, while authorization controls their permissions. This layered approach not only prevents unauthorized access but also provides detailed control over service interactions, maintaining system security and compliance. Here we leverage Istio mechanisms together with Bearer JWT tokens to fulfill that.
 
-<img src="./docs/OPEA auth flow with OIDC provider.png" width="700" height="300">
+![OPEA auth flow with OIDC provider](./docs/OPEA_auth_flow_with_OIDC_provider.png)
 
 ### Perform authentication and authorization via fake JWT tokens
 
@@ -195,7 +195,7 @@ curl -X POST $accessUrl -d '{"text":"What is the revenue of Nike in 2023?","para
 
 Another choice we have is using oauth2-proxy and OIDC providers. These two streamline authentication and authorization by handling user identity and access management. oauth2-proxy acts as a gateway, integrating with OIDC providers to authenticate users and issue tokens. This setup ensures secure access to applications by validating user credentials and managing permissions, simplifying the implementation of robust security protocols across services.
 
-<img src="./docs/OPEA auth flow with oauth2-proxy.png" width="700" height="400">
+![OPEA auth flow with oauth2-proxy](./docs/OPEA_auth_flow_with_oauth2-proxy.png)
 
 We are using a similar scenario here that only privileged users can access our chatQnA service and ask questions. In this case, user `mary` who has the role `user` can access the chatQnA pipeline. And user `bob` with the role `viewer` will not be able to access the service. Of course, the other users without valid token cannot access the service.
 
@@ -221,31 +221,31 @@ The user management is done via Keycloak and the configuration steps look like t
 
 1. Create a new realm named `chatqna` within Keycloak.
 
-<img src="./docs/create_realm.png" width="600" height="300">
+   ![create realm](./docs/create_realm.png)
 
 2. Create a new client called `chatqna` and set `Client authentication` to 'On'. Set "http://chatqna-ui.com:${INGRESS_PORT}/*" in the `Valid redirect URIs` part. Note that `INGRESS_PORT` and `INGRESS_HOST` shall be exported following the guide [here](https://istio.io/latest/docs/tasks/traffic-management/ingress/ingress-control/#determining-the-ingress-ip-and-ports). Under the Credentials tab you will now be able to locate `<your client's secret>`, which will be used in the oauth2-proxy configs.
 
-<img src="./docs/create_client_1.png" width="600" height="300">
+   ![create client 1](./docs/create_client_1.png)
 
-<img src="./docs/create_client_2.png" width="600" height="300">
+   ![create client 2](./docs/create_client_2.png)
 
-<img src="./docs/create_client_3.png" width="600" height="300">
+   ![create client 3](./docs/create_client_3.png)
 
 3. Access the dedicated mappers pane by clicking `<your client's id>-dedicated`, located under Assigned client scope to configure a new `Audience` mapper with name `aud-mapper-<your client's id>`. And include Audience in your client with `ID token` and `access token` set to `On`.
 
-<img src="./docs/add_mapper.png" width="600" height="300">
+   ![add mapper](./docs/add_mapper.png)
 
 4. Create new roles `user` and `viewer` by navigating to `<your client's id> -> Roles`.
 
 5. Create a new user name as `mary` and another user as `bob` with `Email verified` set to `On`. Set passwords for both users (set 'Temporary' to 'Off').
 
-<img src="./docs/create_user.png" width="600" height="300">
+   ![create user](./docs/create_user.png)
 
 6. Create a new Client Scope with the name `groups` in Keycloak with `Include in Token Scope` set as `On`. Include a mapper of type `Group Membership` and set the `Token Claim Name` to `groups`. If the "Full group path" option is selected, you need to include a "/" separator in the group names defined in the --allowed-group option of OAuth2 Proxy. Example: "/groupname". After creating the Client Scope named `groups` you will need to attach it to your client. Go to Clients and find `<your client's id> -> Client scopes` and add client scope and select `groups` and choose `Optional` and you should now have a client that maps group memberships into the JWT tokens so that Oauth2 Proxy may evaluate them.
 
-<img src="./docs/add_group_scope.png" width="600" height="300">
+   ![add group scope](./docs/add_group_scope.png)
 
-<img src="./docs/attach_group_scope.png" width="600" height="300">
+   ![attach group scope](./docs/attach_group_scope.png)
 
 7. Create two groups `user` and `viewer` by navigating to Groups -> Create group. Assign role `user` to group `user` and role `viewer` to group `viewer` and add user `mary` as a member of group `user` and `bob` as a member of group `viewer`.
 

helm-charts/common/data-prep/templates/configmap.yaml

@@ -20,6 +20,8 @@ data:
   REDIS_URL: "redis://{{ .Release.Name }}-redis-vector-db:6379"
   {{- end }}
   INDEX_NAME: {{ .Values.INDEX_NAME | quote }}
+  KEY_INDEX_NAME: {{ .Values.KEY_INDEX_NAME | quote }}
+  SEARCH_BATCH_SIZE: {{ .Values.SEARCH_BATCH_SIZE | quote }}
   HUGGINGFACEHUB_API_TOKEN: {{ .Values.global.HUGGINGFACEHUB_API_TOKEN | quote}}
   HF_HOME: "/tmp/.cache/huggingface"
   {{- if .Values.global.HF_ENDPOINT }}

helm-charts/common/data-prep/values.yaml

@@ -88,6 +88,8 @@ EMBED_MODEL: ""
 # redis DB service URL, e.g. redis://<service-name>:<port>
 REDIS_URL: ""
 INDEX_NAME: "rag-redis"
+KEY_INDEX_NAME: "file-keys"
+SEARCH_BATCH_SIZE: 10
 
 global:
   http_proxy: ""

microservices-connector/config/manifests/data-prep.yaml

@@ -18,6 +18,8 @@ data:
   EMBED_MODEL: ""
   REDIS_URL: "redis://data-prep-redis-vector-db:6379"
   INDEX_NAME: "rag-redis"
+  KEY_INDEX_NAME: "file-keys"
+  SEARCH_BATCH_SIZE: "10"
   HUGGINGFACEHUB_API_TOKEN: "insert-your-huggingface-token-here"
   HF_HOME: "/tmp/.cache/huggingface"
   http_proxy: ""