Merge branch 'main' into lookup

# Conflicts: # integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java # integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java # opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchIndexTest.java
opensearch-project · Dec 2, 2024 · b0ccb79 · b0ccb79
2 parents 127f22b + 9d9730b
commit b0ccb79
Show file tree

Hide file tree

Showing 363 changed files with 18,491 additions and 5,515 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,2 +1,2 @@
 # This should match the owning team set up in https://github.com/orgs/opensearch-project/teams
-*   @pjfitzgibbons @ps48 @kavithacm @derek-ho @joshuali925 @dai-chen @YANG-DB @rupal-bq @mengweieric @vamsi-amazon @swiddis @penghuo @seankao-az @MaxKsyunz @Yury-Fridlyand @anirudha @forestmvey @acarbonetto @GumpacG @ykmr1224 @LantaoJin
+*   @pjfitzgibbons @ps48 @kavithacm @derek-ho @joshuali925 @dai-chen @YANG-DB @rupal-bq @mengweieric @vamsi-amazon @swiddis @penghuo @seankao-az @MaxKsyunz @Yury-Fridlyand @anirudha @forestmvey @acarbonetto @GumpacG @ykmr1224 @LantaoJin @noCharger
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,16 +1,18 @@
 ### Description
 [Describe what this change achieves]
-
-### Issues Resolved
-[List any issues this PR will resolve]
-
+
+### Related Issues
+Resolves #[Issue number to be closed when this PR is merged]
+<!-- List any other related issues here -->
+
 ### Check List
 - [ ] New functionality includes testing.
-  - [ ] All tests pass, including unit test, integration test and doctest
 - [ ] New functionality has been documented.
-  - [ ] New functionality has javadoc added
-  - [ ] New functionality has user manual doc added
-- [ ] Commits are signed per the DCO using --signoff 
+ - [ ] New functionality has javadoc added.
+ - [ ] New functionality has a user manual doc added.
+- [ ] API changes companion pull request [created](https://github.com/opensearch-project/opensearch-api-specification/blob/main/DEVELOPER_GUIDE.md).
+- [ ] Commits are signed per the DCO using `--signoff`.
+- [ ] Public documentation issue/PR [created](https://github.com/opensearch-project/documentation-website/issues/new/choose).
 
 By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
-For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin).
+For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/sql/blob/main/CONTRIBUTING.md#developer-certificate-of-origin).
diff --git a/.github/workflows/integ-tests-with-security.yml b/.github/workflows/integ-tests-with-security.yml
@@ -21,21 +21,21 @@ jobs:
       fail-fast: false
       matrix:
         java: [21]
-    env:
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     runs-on: ubuntu-latest
     container:
       # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
       # this image tag is subject to change as more dependencies and updates will arrive over time
       image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
-      # need to switch to root so that github actions can install runner binary on container without permission issues.
-      options: --user root
+      options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
 
     steps:
-    - uses: actions/checkout@v3
+    - name: Run start commands
+      run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
+
+    - uses: actions/checkout@v4
 
     - name: Set up JDK ${{ matrix.java }}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
         java-version: ${{ matrix.java }}
@@ -47,7 +47,7 @@ jobs:
 
     - name: Upload test reports
       if: ${{ always() }}
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       continue-on-error: true
       with:
         name: test-reports-${{ matrix.os }}-${{ matrix.java }}
@@ -60,16 +60,16 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ windows-latest ]
+        os: [ windows-latest, macos-13 ]
         java: [21]
 
     runs-on: ${{ matrix.os }}
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up JDK ${{ matrix.java }}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
         java-version: ${{ matrix.java }}
@@ -79,7 +79,7 @@ jobs:
 
     - name: Upload test reports
       if: ${{ always() }}
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       continue-on-error: true
       with:
         name: test-reports-${{ matrix.os }}-${{ matrix.java }}

diff --git a/.github/workflows/sql-pitest.yml b/.github/workflows/sql-pitest.yml
@@ -22,21 +22,22 @@ jobs:
       matrix:
         java:
           - 21
-    env:
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     runs-on: ubuntu-latest
     container:
       # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
       # this image tag is subject to change as more dependencies and updates will arrive over time
       image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
-      # need to switch to root so that github actions can install runner binary on container without permission issues.
-      options: --user root
+      options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
+
 
     steps:
-    - uses: actions/checkout@v3
+    - name: Run start commands
+      run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
+
+    - uses: actions/checkout@v4
 
     - name: Set up JDK ${{ matrix.java }}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
         java-version: ${{ matrix.java }}
@@ -48,7 +49,7 @@ jobs:
 
     - name: Upload test reports
       if: always()
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: test-reports-${{ matrix.entry.java }}
         path: |

diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml
@@ -29,21 +29,21 @@ jobs:
       fail-fast: false
       matrix:
         java: [21]
-    env:
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     runs-on: ubuntu-latest
     container:
       # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
       # this image tag is subject to change as more dependencies and updates will arrive over time
       image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
-      # need to switch to root so that github actions can install runner binary on container without permission issues.
-      options: --user root
+      options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
 
     steps:
-    - uses: actions/checkout@v3
+    - name: Run start commands
+      run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
+
+    - uses: actions/checkout@v4
 
     - name: Set up JDK ${{ matrix.java }}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
         java-version: ${{ matrix.java }}
@@ -53,11 +53,6 @@ jobs:
         chown -R 1000:1000 `pwd`
         su `id -un 1000` -c "./gradlew --continue build"
 
-    - name: Run backward compatibility tests
-      run: |
-        chown -R 1000:1000 `pwd`
-        su `id -un 1000` -c "./scripts/bwctest.sh"
-
     - name: Create Artifact Path
       run: |
         mkdir -p opensearch-sql-builds
@@ -66,22 +61,22 @@ jobs:
     # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action
     - name: Upload SQL Coverage Report
       if: ${{ always() }}
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       continue-on-error: true
       with:
         flags: sql-engine
         token: ${{ secrets.CODECOV_TOKEN }}
 
     - name: Upload Artifacts
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       continue-on-error: true
       with:
         name: opensearch-sql-ubuntu-latest-${{ matrix.java }}
         path: opensearch-sql-builds
 
     - name: Upload test reports
       if: ${{ always() }}
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       continue-on-error: true
       with:
         name: test-reports-ubuntu-latest-${{ matrix.java }}
@@ -105,13 +100,14 @@ jobs:
       matrix:
         entry:
           - { os: windows-latest, java: 21, os_build_args: -x doctest -PbuildPlatform=windows }
+          - { os: macos-13, java: 21 }
     runs-on: ${{ matrix.entry.os }}
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up JDK ${{ matrix.java }}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
         java-version: ${{ matrix.entry.java }}
@@ -127,22 +123,22 @@ jobs:
     # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action
     - name: Upload SQL Coverage Report
       if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }}
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       continue-on-error: true
       with:
         flags: sql-engine
         token: ${{ secrets.CODECOV_TOKEN }}
 
     - name: Upload Artifacts
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       continue-on-error: true
       with:
         name: opensearch-sql-${{ matrix.entry.os }}-${{ matrix.entry.java }}
         path: opensearch-sql-builds
 
     - name: Upload test reports
       if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }}
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       continue-on-error: true
       with:
         name: test-reports-${{ matrix.entry.os }}-${{ matrix.entry.java }}
@@ -158,3 +154,49 @@ jobs:
           plugin/build/reports/**
           doctest/build/testclusters/docTestCluster-0/logs/*
           integ-test/build/testclusters/*/logs/*
+
+  bwc-tests:
+    needs: Get-CI-Image-Tag
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java: [21]
+    container:
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
+
+    steps:
+    - name: Run start commands
+      run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
+
+    - uses: actions/checkout@v4
+
+    - name: Set up JDK ${{ matrix.java }}
+      uses: actions/setup-java@v4
+      with:
+        distribution: 'temurin'
+        java-version: ${{ matrix.java }}
+
+    - name: Run backward compatibility tests
+      run: |
+        chown -R 1000:1000 `pwd`
+        su `id -un 1000` -c "./scripts/bwctest.sh"
+
+    - name: Upload test reports
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v4
+      continue-on-error: true
+      with:
+        name: test-reports-ubuntu-latest-${{ matrix.java }}-bwc
+        path: |
+          sql/build/reports/**
+          ppl/build/reports/**
+          core/build/reports/**
+          common/build/reports/**
+          opensearch/build/reports/**
+          integ-test/build/reports/**
+          protocol/build/reports/**
+          legacy/build/reports/**
+          plugin/build/reports/**
+          doctest/build/testclusters/docTestCluster-0/logs/*
+          integ-test/build/testclusters/*/logs/*
diff --git a/.github/workflows/sql-test-workflow.yml b/.github/workflows/sql-test-workflow.yml
@@ -22,21 +22,21 @@ jobs:
       matrix:
         java:
           - 21
-    env:
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     runs-on: ubuntu-latest
     container:
       # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
       # this image tag is subject to change as more dependencies and updates will arrive over time
       image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
-      # need to switch to root so that github actions can install runner binary on container without permission issues.
-      options: --user root
+      options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
 
     steps:
-    - uses: actions/checkout@v3
+    - name: Run start commands
+      run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
+
+    - uses: actions/checkout@v4
 
     - name: Set up JDK ${{ matrix.java }}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
         java-version: ${{ matrix.java }}
@@ -86,7 +86,7 @@ jobs:
 
     - name: Upload test reports
       if: always()
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: test-reports-${{ matrix.entry.java }}
         path: |

diff --git a/.gitignore b/.gitignore
@@ -49,4 +49,5 @@ gen
 .worktrees
 http-client.env.json
 /doctest/sql-cli/
+/doctest/opensearch-job-scheduler/
 .factorypath
diff --git a/MAINTAINERS.md b/MAINTAINERS.md
@@ -22,6 +22,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Anirudha Jadhav   | [anirudha](https://github.com/anirudha)             | Amazon      |
 | Tomoyuki Morita   | [ykmr1224](https://github.com/ykmr1224)             | Amazon      |
 | Lantao Jin        | [LantaoJin](https://github.com/LantaoJin)           | Amazon      |
+| Louis Chu         | [noCharger](https://github.com/noCharger)           | Amazon      |
 | Max Ksyunz        | [MaxKsyunz](https://github.com/MaxKsyunz)           | Improving   |
 | Yury Fridlyand    | [Yury-Fridlyand](https://github.com/Yury-Fridlyand) | Improving   |
 | Andrew Carbonetto | [acarbonetto](https://github.com/acarbonetto)       | Improving   |

diff --git a/async-query-core/README.md b/async-query-core/README.md
@@ -32,3 +32,11 @@ Following is the list of extension points where the consumer of the library need
   - [DataSourceSparkParameterComposer](src/main/java/org/opensearch/sql/spark/parameter/DataSourceSparkParameterComposer.java)
   - [GeneralSparkParameterComposer](src/main/java/org/opensearch/sql/spark/parameter/GeneralSparkParameterComposer.java)
   - [SparkSubmitParameterModifier](src/main/java/org/opensearch/sql/spark/config/SparkSubmitParameterModifier.java) To be deprecated in favor of GeneralSparkParameterComposer
+
+## Update Grammar files
+This package uses ANTLR grammar files from `opensearch-spark` and `Spark` repositories.
+To update the grammar files, update `build.gradle` file (in `downloadG4Files` task) as needed and run:
+```
+./gradlew async-query-core:downloadG4Files
+```
+This will overwrite the files under `src/main/antlr`.