pr-chart-e2e.yaml

# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

name: E2E test with helm charts

on:
  pull_request_target:
    branches: [main]
    types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
    paths:
      - helm-charts/**
      - .github/workflows/chart-e2e.yaml
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  CHARTS_DIR: "helm-charts"

jobs:
  job1:
    name: Get-test-matrix
    runs-on: ubuntu-latest
    outputs:
      run_matrix: ${{ steps.get-test-matrix.outputs.run_matrix }}
    steps:
      - name: Checkout out Repo
        uses: actions/checkout@v4
        with:
          ref: "refs/pull/${{ github.event.number }}/merge"
          fetch-depth: 0

      - name: Get test matrix
        id: get-test-matrix
        run: |
          base_commit=${{ github.event.pull_request.base.sha }}
          merged_commit=$(git log -1 --format='%H')
          e2e_charts=$(git diff --name-only ${base_commit} ${merged_commit} | \
          grep "^$CHARTS_DIR/" | \
          grep -vE 'README.md|common|*.sh' | \
          cut -d'/' -f2 | sort -u )
          common_charts=$(git diff --name-only ${base_commit} ${merged_commit} | \
          grep "^$CHARTS_DIR/common" | \
          grep -vE 'README.md|*.sh' | \
          cut -d'/' -f3 | sort -u )
          run_matrix="{\"include\":["
          for chart in ${e2e_charts}; do
            if [ -f $CHARTS_DIR/$chart/gaudi-values.yaml ]; then
              run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"gaudi\"},"
            fi
            run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"xeon\"},"
          done
          for chart in ${common_charts}; do
            if [ -f $CHARTS_DIR/common/$chart/gaudi-values.yaml ]; then
              run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"gaudi\",\"directory\":\"common\"},"
            fi
            run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"xeon\",\"directory\":\"common\"},"
          done
          run_matrix=$run_matrix"]}"
          echo "run_matrix=${run_matrix}"
          echo "run_matrix=${run_matrix}" >> $GITHUB_OUTPUT

  Chart-test:
    needs: job1
    if: always() && ${{ needs.job1.outputs.run_matrix.example.length }} > 0
    strategy:
      matrix: ${{ fromJSON(needs.job1.outputs.run_matrix) }}
    runs-on: ${{ matrix.hardware }}
    continue-on-error: true
    outputs:
      should_cleanup: ${{ steps.set_boolean.outputs.should_cleanup }}
    steps:
      - name: E2e test chart
        run: |
          echo "Matrix - chart: ${{ matrix.example }}"

      - name: Clean Up Working Directory
        run: sudo rm -rf ${{github.workspace}}/*

      - name: Checkout out Repo
        uses: actions/checkout@v4
        with:
          ref: "refs/pull/${{ github.event.number }}/merge"

      - name: Set variables
        run: |
          echo "RELEASE_NAME=${{ matrix.example }}$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV
          echo "NAMESPACE=${{ matrix.example }}-$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV
          echo "ROLLOUT_TIMEOUT_SECONDS=600s" >> $GITHUB_ENV
          echo "KUBECTL_TIMEOUT_SECONDS=60s" >> $GITHUB_ENV
          echo "should_cleanup=false" >> $GITHUB_ENV
          echo "skip_validate=false" >> $GITHUB_ENV
          echo "RELEASENAME=$RELEASE_NAME"
          echo "NAMESPACE=$NAMESPACE"
          if [ -n "${{ matrix.directory }}" ]; then
            echo "CHART_FOLDER=$CHARTS_DIR/${{ matrix.directory }}/${{ matrix.example }}" >> $GITHUB_ENV
          else
            echo "CHART_FOLDER=$CHARTS_DIR/${{ matrix.example }}" >> $GITHUB_ENV
          fi

      - name: Initialize chart testing
        run: |
          # Replace values for CI test environment
          USER_ID=$(whoami)
          CHART_MOUNT=/home/$USER_ID/.cache/huggingface/hub
          HFTOKEN=$(cat /home/$USER_ID/.cache/huggingface/token)
          pushd helm-charts
          # insert a prefix before opea/.*, the prefix is OPEA_IMAGE_REPO
          find . -name '*values.yaml' -type f -exec sed -i "s#repository: opea/*#repository: ${OPEA_IMAGE_REPO}opea/#g" {} \;
          # set OPEA image tag to latest
          find . -name '*values.yaml' -type f -exec sed -i 's#tag: ""#tag: latest#g' {} \;
          # set huggingface token
          find . -name '*values.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#${HFTOKEN}#g" {} \;
          # replace the mount dir "Volume: *" with "Volume: $CHART_MOUNT"
          find . -name '*values.yaml' -type f -exec sed -i "s#modelUseHostPath: .*#modelUseHostPath: $CHART_MOUNT#g" {} \;
          # replace the pull policy "IfNotPresent" with "Always"
          find . -name '*values.yaml' -type f -exec sed -i "s#pullPolicy: IfNotPresent#pullPolicy: Always#g" {} \;
          popd

      - name: Helm install
        id: install
        env:
          GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }}
          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
        run: |
          set -xe
          echo "should_cleanup=true" >> $GITHUB_ENV
          helm-charts/update_dependency.sh && helm dependency update ${{ env.CHART_FOLDER}}
          value_file="values.yaml"
          if [ "${{ matrix.hardware }}" == "gaudi" ]; then
            value_file="gaudi-values.yaml"
          fi
          if ! helm install --create-namespace --namespace $NAMESPACE --wait \
              --timeout "$ROLLOUT_TIMEOUT_SECONDS" \
              --set autodependency.enabled=true \
              --set GOOGLE_API_KEY=${{ env.GOOGLE_API_KEY}} \
              --set GOOGLE_CSE_ID=${{ env.GOOGLE_CSE_ID}} \
              --values ${{ env.CHART_FOLDER}}/${value_file} \
              $RELEASE_NAME ${{ env.CHART_FOLDER}} ; then
            echo "Failed to install chart ${{ matrix.example }}"
            echo "skip_validate=true" >> $GITHUB_ENV
            .github/workflows/scripts/e2e/chart_test.sh dump_pods_status $NAMESPACE
            exit 1
          fi

      - name: Validate e2e test
        if: always()
        run: |
          set -xe
          if $skip_validate; then
            echo "Skip validate"
          else
            LOG_PATH=/home/$(whoami)/logs
            chart=${{ matrix.example }}
            helm test -n $NAMESPACE $RELEASE_NAME --logs |tee ${LOG_PATH}/charts-${chart}.log
            exit_code=$?
            if [ $exit_code -ne 0 ]; then
                echo "Chart ${chart} test failed, please check the logs in ${LOG_PATH}!"
                exit 1
            fi

            echo "Checking response results, make sure the output is reasonable. "
            teststatus=false
            if [[ -f $LOG_PATH/charts-${chart}.log ]] && \
            [[ $(grep -c "^Phase:.*Failed" $LOG_PATH/charts-${chart}.log) != 0 ]]; then
                teststatus=false
                .github/workflows/scripts/e2e/chart_test.sh dump_all_pod_logs $NAMESPACE
            else
                teststatus=true
            fi

            if [ $teststatus == false ]; then
                echo "Response check failed, please check the logs in artifacts!"
                exit 1
            else
                echo "Response check succeed!"
                exit 0
            fi
          fi

      - name: Helm uninstall
        if: always()
        run: |
          if $should_cleanup; then
            helm uninstall $RELEASE_NAME --namespace $NAMESPACE
            if ! kubectl delete ns $NAMESPACE --timeout=$KUBECTL_TIMEOUT_SECONDS; then
              kubectl delete pods --namespace $NAMESPACE --force --grace-period=0 --all
              kubectl delete ns $NAMESPACE --force --grace-period=0 --timeout=$KUBECTL_TIMEOUT_SECONDS
            fi
          fi