Merge branch 'main' into fix-issue-2440

Signed-off-by: Andreas Kulicke <[email protected]>

.github/workflows/integ-tests-with-security.yml

@@ -20,7 +20,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        java: [ 11, 17 ]
+        java: [ 11, 17, 21 ]
 
     runs-on: ubuntu-latest
     container:
@@ -60,7 +60,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ windows-latest, macos-latest ]
-        java: [ 11, 17 ]
+        java: [ 11, 17, 21 ]
 
     runs-on: ${{ matrix.os }}
 

.github/workflows/sql-test-and-build-workflow.yml

@@ -31,6 +31,7 @@ jobs:
         java:
           - 11
           - 17
+          - 21
     runs-on: ubuntu-latest
     container:
       # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
@@ -108,6 +109,8 @@ jobs:
           - { os: macos-latest, java: 11}
           - { os: windows-latest, java: 17, os_build_args: -x doctest -PbuildPlatform=windows }
           - { os: macos-latest, java: 17 }
+          - { os: windows-latest, java: 21, os_build_args: -x doctest -PbuildPlatform=windows }
+          - { os: macos-latest, java: 21 }
     runs-on: ${{ matrix.entry.os }}
 
     steps:

build.gradle

@@ -49,6 +49,7 @@ buildscript {
         getPrometheusBinaryLocation = { ->
             return "https://github.com/prometheus/prometheus/releases/download/v${prometheus_binary_version}/prometheus-${prometheus_binary_version}."+ getOSFamilyType() + "-" + getArchType() + ".tar.gz"
         }
+        aws_java_sdk_version = "1.12.651"
     }
 
     repositories {
@@ -116,6 +117,7 @@ allprojects {
     configurations.all {
         resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.9.10"
         resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10"
+        resolutionStrategy.force "net.bytebuddy:byte-buddy:1.14.9"
     }
 }
 

common/build.gradle

@@ -37,11 +37,12 @@ dependencies {
     api group: 'com.google.guava', name: 'guava', version: '32.0.1-jre'
     api group: 'org.apache.logging.log4j', name: 'log4j-core', version:"${versions.log4j}"
     api group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0'
+    api group: 'org.apache.commons', name: 'commons-text', version: '1.10.0'
     api group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.12.0'
     implementation 'com.github.babbel:okhttp-aws-signer:1.0.2'
-    api group: 'com.amazonaws', name: 'aws-java-sdk-core', version: '1.12.545'
-    api group: 'com.amazonaws', name: 'aws-java-sdk-sts', version: '1.12.545'
-    implementation "com.github.seancfoley:ipaddress:5.4.0"
+    api group: 'com.amazonaws', name: 'aws-java-sdk-core', version: "${aws_java_sdk_version}"
+    api group: 'com.amazonaws', name: 'aws-java-sdk-sts', version: "${aws_java_sdk_version}"
+    implementation "com.github.seancfoley:ipaddress:5.4.2"
 
     testImplementation group: 'junit', name: 'junit', version: '4.13.2'
     testImplementation group: 'org.assertj', name: 'assertj-core', version: '3.9.1'

common/src/main/java/org/opensearch/sql/common/setting/Settings.java

@@ -45,7 +45,12 @@ public enum Key {
     AUTO_INDEX_MANAGEMENT_ENABLED(
         "plugins.query.executionengine.spark.auto_index_management.enabled"),
     SESSION_INACTIVITY_TIMEOUT_MILLIS(
-        "plugins.query.executionengine.spark.session_inactivity_timeout_millis");
+        "plugins.query.executionengine.spark.session_inactivity_timeout_millis"),
+
+    /** Async query Settings * */
+    ASYNC_QUERY_ENABLED("plugins.query.executionengine.async_query.enabled"),
+    STREAMING_JOB_HOUSEKEEPER_INTERVAL(
+        "plugins.query.executionengine.spark.streamingjobs.housekeeper.interval");
 
     @Getter private final String keyValue;
 

core/build.gradle

@@ -48,6 +48,7 @@ pitest {
 dependencies {
     api group: 'com.google.guava', name: 'guava', version: '32.0.1-jre'
     api group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0'
+    api group: 'org.apache.commons', name: 'commons-text', version: '1.10.0'
     api group: 'com.facebook.presto', name: 'presto-matching', version: '0.240'
     api group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1'
     api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
@@ -108,7 +109,9 @@ jacocoTestCoverageVerification {
             excludes = [
                     'org.opensearch.sql.utils.MLCommonsConstants',
                     'org.opensearch.sql.utils.Constants',
-                    'org.opensearch.sql.datasource.model.*'
+                    'org.opensearch.sql.datasource.model.DataSource',
+                    'org.opensearch.sql.datasource.model.DataSourceStatus',
+                    'org.opensearch.sql.datasource.model.DataSourceType'
             ]
             limit {
                 counter = 'LINE'

core/src/main/java/org/opensearch/sql/datasource/DataSourceService.java

@@ -14,7 +14,8 @@
 public interface DataSourceService {
 
   /**
-   * Returns {@link DataSource} corresponding to the DataSource name.
+   * Returns {@link DataSource} corresponding to the DataSource name only if the datasource is
+   * active and authorized.
    *
    * @param dataSourceName Name of the {@link DataSource}.
    * @return {@link DataSource}.
@@ -40,15 +41,6 @@ public interface DataSourceService {
    */
   DataSourceMetadata getDataSourceMetadata(String name);
 
-  /**
-   * Returns dataSourceMetadata object with specific name. The returned objects contain all the
-   * metadata information without any filtering.
-   *
-   * @param name name of the {@link DataSource}.
-   * @return set of {@link DataSourceMetadata}.
-   */
-  DataSourceMetadata getRawDataSourceMetadata(String name);
-
   /**
    * Register {@link DataSource} defined by {@link DataSourceMetadata}.
    *
@@ -84,4 +76,12 @@ public interface DataSourceService {
    * @param dataSourceName name of the {@link DataSource}.
    */
   Boolean dataSourceExists(String dataSourceName);
+
+  /**
+   * Performs authorization and datasource status check and then returns RawDataSourceMetadata.
+   * Specifically for addressing use cases in SparkQueryDispatcher.
+   *
+   * @param dataSourceName of the {@link DataSource}
+   */
+  DataSourceMetadata verifyDataSourceAccessAndGetRawMetadata(String dataSourceName);
 }