Add query enhancement cypress tests to Github CI (#8703) (#8898)

Add query enhancement cypress tests to Github CI Signed-off-by: abbyhu2000 <[email protected]>
opensearch-project · Nov 20, 2024 · 390c5fc · 390c5fc
1 parent 554b046
commit 390c5fc
Show file tree

Hide file tree

Showing 4 changed files with 156 additions and 16 deletions.
diff --git a/.github/workflows/cypress_workflow.yml b/.github/workflows/cypress_workflow.yml
@@ -3,7 +3,7 @@ name: Run cypress tests
 # trigger on every PR for all branches except feature branches
 on:
   pull_request:
-    branches: [ '**', '!feature/**' ]
+    branches: ['**', '!feature/**']
     paths-ignore:
       - '**/*.md'
       - '.lycheeignore'
@@ -32,8 +32,6 @@ env:
   TEST_REPO: ${{ inputs.test_repo != '' && inputs.test_repo || 'opensearch-project/opensearch-dashboards-functional-test' }}
   TEST_BRANCH: "${{ inputs.test_branch != '' && inputs.test_branch || github.base_ref }}"
   FTR_PATH: 'ftr'
-  START_CMD: 'node ../scripts/opensearch_dashboards --dev --no-base-path --no-watch --savedObjects.maxImportPayloadBytes=10485760 --server.maxPayloadBytes=1759977 --logging.json=false --data.search.aggs.shardDelay.enabled=true  --csp.warnLegacyBrowsers=false --uiSettings.overrides["query:enhancements:enabled"]=false'
-  OPENSEARCH_SNAPSHOT_CMD: 'node ../scripts/opensearch snapshot -E cluster.routing.allocation.disk.threshold_enabled=false'
   CYPRESS_BROWSER: 'chromium'
   CYPRESS_VISBUILDER_ENABLED: true
   CYPRESS_DATASOURCE_MANAGEMENT_ENABLED: false
@@ -42,18 +40,65 @@ env:
   COMMENT_TAG: '[MANUAL CYPRESS TEST RUN RESULTS]'
   COMMENT_SUCCESS_MSG: ':white_check_mark: Cypress test run succeeded!'
   COMMENT_FAILURE_MSG: ':x: Cypress test run failed!'
+  LATEST_VERSION: '2.17.0'
 
 jobs:
   cypress-tests:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-          group: [1, 2, 3, 4, 5, 6, 7, 8, 9]
+        include:
+          # Standard test configs (groups 1-9)
+          - group: 1
+            config: standard
+            test_location: ftr
+          - group: 2
+            config: standard
+            test_location: ftr
+          - group: 3
+            config: standard
+            test_location: ftr
+          - group: 4
+            config: standard
+            test_location: ftr
+          - group: 5
+            config: standard
+            test_location: ftr
+          - group: 6
+            config: standard
+            test_location: ftr
+          - group: 7
+            config: standard
+            test_location: ftr
+          - group: 8
+            config: standard
+            test_location: ftr
+          - group: 9
+            config: standard
+            test_location: ftr
+          # Query enhanced tests
+          - group: 10
+            config: query_enhanced
+            test_location: ftr
+          # Dashboard tests
+          - group: 11
+            config: dashboard
+            test_location: source
     container:
       image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2
       options: --user 1001
     env:
+      START_CMD: ${{ matrix.config == 'query_enhanced' &&
+        'node ../scripts/opensearch_dashboards --dev --no-base-path --no-watch --savedObjects.maxImportPayloadBytes=10485760 --server.maxPayloadBytes=1759977 --logging.json=false --data.search.aggs.shardDelay.enabled=true --csp.warnLegacyBrowsers=false --uiSettings.overrides["query:enhancements:enabled"]=true --uiSettings.overrides[''home:useNewHomePage'']=true --data_source.enabled=true --opensearch.ignoreVersionMismatch=true' ||
+        matrix.config == 'dashboard' &&
+        'node scripts/opensearch_dashboards --dev --no-base-path --no-watch --savedObjects.maxImportPayloadBytes=10485760 --server.maxPayloadBytes=1759977 --logging.json=false --data.search.aggs.shardDelay.enabled=true' ||
+        'node ../scripts/opensearch_dashboards --dev --no-base-path --no-watch --savedObjects.maxImportPayloadBytes=10485760 --server.maxPayloadBytes=1759977 --logging.json=false --data.search.aggs.shardDelay.enabled=true --csp.warnLegacyBrowsers=false --uiSettings.overrides["query:enhancements:enabled"]=false' }}
+      OPENSEARCH_SNAPSHOT_CMD: ${{ matrix.config == 'query_enhanced' &&
+        '/bin/bash -c "./opensearch-2.17.0/opensearch-tar-install.sh &"' ||
+        matrix.config == 'dashboard' &&
+        'node scripts/opensearch snapshot -E cluster.routing.allocation.disk.threshold_enabled=false' ||
+        'node ../scripts/opensearch snapshot -E cluster.routing.allocation.disk.threshold_enabled=false' }}
       # prevents extra Cypress installation progress messages
       CI: 1
       # avoid warnings like "tput: No value for $TERM and no -T specified"
@@ -116,7 +161,7 @@ jobs:
           echo "SPEC=${{ inputs.specs }}" >> $GITHUB_ENV
 
       - name: Setup spec files
-        if: ${{ inputs.specs == '' }}
+        if: ${{ inputs.specs == '' && matrix.test_location == 'ftr' }}
         working-directory: ${{ env.FTR_PATH }}
         shell: bash
         run: |
@@ -127,12 +172,26 @@ jobs:
           done
           echo "SPEC=${FORMATTED_SPEC}" >> $GITHUB_ENV
 
+      # Setup spec files for Dashboards in-house cypress tests
+      - name: Setup spec files for Dashboards tests
+        if: ${{ inputs.specs == '' && matrix.test_location == 'source' }}
+        shell: bash
+        run: |
+          IFS="," read -a SPEC_ARRAY <<< $(yarn --silent osd:ciGroup${{ matrix.group }})
+          DASHBOARDS_SPEC=''
+          for i in "${SPEC_ARRAY[@]}"; do
+          DASHBOARDS_SPEC+="cypress/integration/core_opensearch_dashboards/${i},"
+          done
+          echo "DASHBOARDS_SPEC=${DASHBOARDS_SPEC}" >> $GITHUB_ENV
+
       - name: Get Cypress version
+        if: ${{ matrix.test_location == 'ftr' }}
         id: cypress_version
         run: |
           echo "name=cypress_version::$(cat ./${{ env.FTR_PATH }}/package.json | jq '.devDependencies.cypress' | tr -d '"')" >> $GITHUB_OUTPUT
 
       - name: Cache Cypress
+        if: ${{ matrix.test_location == 'ftr' }}
         id: cache-cypress
         uses: actions/cache@v1
         with:
@@ -143,36 +202,115 @@ jobs:
       - run: npx cypress cache list
       - run: npx cypress cache path
 
-      - name: Run tests
+      # Run tests based on configuration
+      - name: Run FT repo tests with no query enhancements
+        if: matrix.test_location == 'ftr' && matrix.config == 'standard'
         uses: cypress-io/github-action@v2
         with:
           working-directory: ${{ env.FTR_PATH }}
           start: ${{ env.OPENSEARCH_SNAPSHOT_CMD }}, ${{ env.START_CMD }}
           wait-on: 'http://localhost:9200, http://localhost:5601'
           command: yarn cypress:run-without-security --browser ${{ env.CYPRESS_BROWSER }} --spec ${{ env.SPEC }}
 
+      - name: Download OpenSearch
+        if: matrix.test_location == 'ftr' && matrix.config == 'query_enhanced'
+        uses: suisei-cn/[email protected]
+        with:
+          url: https://artifacts.opensearch.org/releases/bundle/opensearch/${{ env.LATEST_VERSION }}/opensearch-${{ env.LATEST_VERSION }}-linux-x64.tar.gz
+
+      - name: Extract OpenSearch
+        if: matrix.test_location == 'ftr' && matrix.config == 'query_enhanced'
+        run: |
+          tar -xzf opensearch-*.tar.gz
+          rm -f opensearch-*.tar.gz
+        shell: bash
+
+      - name: Remove security plugin
+        if: matrix.test_location == 'ftr' && matrix.config == 'query_enhanced'
+        run: |
+          /bin/bash -c "yes | ./opensearch-${{ env.LATEST_VERSION }}/bin/opensearch-plugin remove opensearch-security"
+        shell: bash
+
+      - name: Run OpenSearch
+        if: matrix.test_location == 'ftr' && matrix.config == 'query_enhanced'
+        run: |
+          /bin/bash -c "./opensearch-2.17.0/opensearch-tar-install.sh &"
+          sleep 30
+        shell: bash
+
+      - name: Run FT repo tests with query enhancements
+        if: matrix.test_location == 'ftr' && matrix.config == 'query_enhanced'
+        uses: cypress-io/github-action@v2
+        with:
+          working-directory: ${{ env.FTR_PATH }}
+          start: ${{ env.START_CMD }}
+          wait-on: 'http://localhost:9200, http://localhost:5601'
+          command: yarn cypress:run-without-security --browser ${{ env.CYPRESS_BROWSER }} --spec ${{ env.SPEC }}
+
+      # Clear Cypress Cache before running Dashboards tests
+      - name: Clear Cypress Cache
+        if: matrix.test_location == 'source'
+        run: npx cypress cache clear
+
+      # Run Dashboards Cypress tests within the source repo
+      - name: Run Dashboards Cypress tests
+        if: matrix.test_location == 'source'
+        uses: cypress-io/github-action@v6
+        with:
+          install-command: npx cypress install --force
+          start: ${{ env.OPENSEARCH_SNAPSHOT_CMD }}, ${{ env.START_CMD }}
+          wait-on: 'http://localhost:9200, http://localhost:5601'
+          command: yarn cypress:run-without-security --browser ${{ env.CYPRESS_BROWSER }} --spec ${{ env.DASHBOARDS_SPEC }}
+
       # Screenshots are only captured on failure, will change this once we do visual regression tests
-      - uses: actions/upload-artifact@v3
-        if: failure()
+      - name: Upload FT repo screenshots
+        uses: actions/upload-artifact@v3
+        if: failure() && matrix.test_location == 'ftr'
         with:
           name: ftr-cypress-screenshots
           path: ${{ env.FTR_PATH }}/cypress/screenshots
           retention-days: 1
-
-      - uses: actions/upload-artifact@v3
-        if: always()
+
+      - name: Upload FT repo videos
+        uses: actions/upload-artifact@v3
+        if: always() && matrix.test_location == 'ftr'
         with:
           name: ftr-cypress-videos
           path: ${{ env.FTR_PATH }}/cypress/videos
           retention-days: 1
 
-      - uses: actions/upload-artifact@v3
-        if: always()
+      - name: Upload FT repo results
+        uses: actions/upload-artifact@v3
+        if: always() && matrix.test_location == 'ftr'
         with:
           name: ftr-cypress-results
           path: ${{ env.FTR_PATH }}/cypress/results
           retention-days: 1
 
+      - name: Upload Dashboards screenshots
+        if: failure() && matrix.test_location == 'source'
+        uses: actions/upload-artifact@v3
+        with:
+          name: dashboards-cypress-screenshots
+          path: cypress/screenshots
+          retention-days: 1
+
+      - name: Upload Dashboards repo videos
+        uses: actions/upload-artifact@v3
+        if: always() && matrix.test_location == 'source'
+        with:
+          name: dashboards-cypress-videos
+          path: cypress/videos
+          retention-days: 1
+
+      - name: Upload Dashboards repo results
+        uses: actions/upload-artifact@v3
+        if: always() && matrix.test_location == 'source'
+        with:
+          name: dashboards-cypress-results
+          path: cypress/results
+          retention-days: 1
+
   add-comment:
     needs: [cypress-tests]
     if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.pr_number != '' }}
@@ -186,7 +324,7 @@ jobs:
         with:
           issue-number: ${{ inputs.pr_number }}
           comment-author: 'github-actions[bot]'
-          body-includes: "${{ env.COMMENT_TAG }}"
+          body-includes: '${{ env.COMMENT_TAG }}'
 
       - name: Add comment on the PR
         uses: peter-evans/create-or-update-comment@v3

diff --git a/package.json b/package.json
@@ -84,8 +84,7 @@
     "release_note:generate": "scripts/use_node scripts/generate_release_note",
     "cypress:run-without-security": "env TZ=America/Los_Angeles NO_COLOR=1 cypress run --headless --env SECURITY_ENABLED=false",
     "cypress:run-with-security": "env TZ=America/Los_Angeles NO_COLOR=1 cypress run --headless --env SECURITY_ENABLED=true,openSearchUrl=https://localhost:9200,WAIT_FOR_LOADER_BUFFER_MS=500",
-    "osd:ciGroup10": "echo \"dashboard_sanity_test_spec.js\"",
-    "osd:ciGroup11": "echo \"apps/vis_builder/*.js\"",
+    "osd:ciGroup11": "echo \"dashboard_sanity_test_spec.js\"",
     "generate:opensearchsqlantlr": "./node_modules/antlr4ng-cli/index.js -Dlanguage=TypeScript -o ./src/plugins/data/public/antlr/opensearch_sql/.generated -visitor -no-listener -Xexact-output-dir ./src/plugins/data/public/antlr/opensearch_sql/grammar/OpenSearchSQLLexer.g4 ./src/plugins/data/public/antlr/opensearch_sql/grammar/OpenSearchSQLParser.g4",
     "generate:opensearchpplantlr": "./node_modules/antlr4ng-cli/index.js -Dlanguage=TypeScript -o ./src/plugins/data/public/antlr/opensearch_ppl/.generated -visitor -no-listener -Xexact-output-dir ./src/plugins/data/public/antlr/opensearch_ppl/grammar/OpenSearchPPLLexer.g4 ./src/plugins/data/public/antlr/opensearch_ppl/grammar/OpenSearchPPLParser.g4"
   },

diff --git a/src/plugins/data/public/ui/dataset_selector/configurator.tsx b/src/plugins/data/public/ui/dataset_selector/configurator.tsx
@@ -141,6 +141,7 @@ export const Configurator = ({
                 setLanguage(e.target.value);
                 setDataset({ ...dataset, language: e.target.value });
               }}
+              data-test-subj="advancedSelectorLanguageSelect"
             />
           </EuiFormRow>
           {!languageService.getLanguage(language)?.hideDatePicker &&

diff --git a/src/plugins/data/public/ui/dataset_selector/dataset_selector.tsx b/src/plugins/data/public/ui/dataset_selector/dataset_selector.tsx
@@ -215,6 +215,7 @@ export const DatasetSelector = ({
             appearance={appearance}
             {...buttonProps}
             className="datasetSelector__button"
+            data-test-subj="datasetSelectorButton"
             iconType="arrowDown"
             iconSide="right"
             onClick={togglePopover}
@@ -253,6 +254,7 @@ export const DatasetSelector = ({
       <EuiPopoverFooter paddingSize="none" className="datasetSelector__footer">
         <EuiButton
           className="datasetSelector__advancedButton"
+          data-test-subj="datasetSelectorAdvancedButton"
           iconType="gear"
           iconSide="right"
           iconSize="s"