From 943b70e36f52f88154886b16f9d7f41ae9dd6abb Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 20 Mar 2024 08:09:43 -0700 Subject: [PATCH] Upgrade opensearch-spark jars to 0.3.0 (#2568) (#2572) (cherry picked from commit 4dc83b7b8ffdab5a26742723c848fedf5af9d564) Signed-off-by: Louis Chu Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- docs/user/ppl/admin/connectors/spark_connector.rst | 2 +- .../sql/spark/data/constants/SparkConstants.java | 10 +++++----- .../opensearch/sql/spark/constants/TestConstants.java | 2 +- .../sql/spark/dispatcher/SparkQueryDispatcherTest.java | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/user/ppl/admin/connectors/spark_connector.rst b/docs/user/ppl/admin/connectors/spark_connector.rst index 8ff8dd944e..59a52998bc 100644 --- a/docs/user/ppl/admin/connectors/spark_connector.rst +++ b/docs/user/ppl/admin/connectors/spark_connector.rst @@ -35,7 +35,7 @@ Spark Connector Properties. * ``spark.datasource.flint.*`` [Optional] * This parameters provides the Opensearch domain host information for flint integration. * ``spark.datasource.flint.integration`` [Optional] - * Default value for integration jar is ``s3://spark-datasource/flint-spark-integration-assembly-0.1.0-SNAPSHOT.jar``. + * Default value for integration jar is ``s3://spark-datasource/flint-spark-integration-assembly-0.3.0-SNAPSHOT.jar``. * ``spark.datasource.flint.host`` [Optional] * Default value for host is ``localhost``. * ``spark.datasource.flint.port`` [Optional] diff --git a/spark/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java b/spark/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java index 906a0b740a..ceb1b4da54 100644 --- a/spark/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java +++ b/spark/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java @@ -19,11 +19,11 @@ public class SparkConstants { // EMR-S will download JAR to local maven public static final String SPARK_SQL_APPLICATION_JAR = - "file:///home/hadoop/.ivy2/jars/org.opensearch_opensearch-spark-sql-application_2.12-0.1.0-SNAPSHOT.jar"; + "file:///home/hadoop/.ivy2/jars/org.opensearch_opensearch-spark-sql-application_2.12-0.3.0-SNAPSHOT.jar"; public static final String SPARK_REQUEST_BUFFER_INDEX_NAME = ".query_execution_request"; // TODO should be replaced with mvn jar. public static final String FLINT_INTEGRATION_JAR = - "s3://spark-datasource/flint-spark-integration-assembly-0.1.0-SNAPSHOT.jar"; + "s3://spark-datasource/flint-spark-integration-assembly-0.3.0-SNAPSHOT.jar"; // TODO should be replaced with mvn jar. public static final String FLINT_DEFAULT_CLUSTER_NAME = "opensearch-cluster"; public static final String FLINT_DEFAULT_HOST = "localhost"; @@ -70,11 +70,11 @@ public class SparkConstants { public static final String DEFAULT_GLUE_CATALOG_CREDENTIALS_PROVIDER_FACTORY_KEY = "com.amazonaws.glue.catalog.metastore.STSAssumeRoleSessionCredentialsProviderFactory"; public static final String SPARK_STANDALONE_PACKAGE = - "org.opensearch:opensearch-spark-standalone_2.12:0.1.0-SNAPSHOT"; + "org.opensearch:opensearch-spark-standalone_2.12:0.3.0-SNAPSHOT"; public static final String SPARK_LAUNCHER_PACKAGE = - "org.opensearch:opensearch-spark-sql-application_2.12:0.1.0-SNAPSHOT"; + "org.opensearch:opensearch-spark-sql-application_2.12:0.3.0-SNAPSHOT"; public static final String PPL_STANDALONE_PACKAGE = - "org.opensearch:opensearch-spark-ppl_2.12:0.1.0-SNAPSHOT"; + "org.opensearch:opensearch-spark-ppl_2.12:0.3.0-SNAPSHOT"; public static final String AWS_SNAPSHOT_REPOSITORY = "https://aws.oss.sonatype.org/content/repositories/snapshots"; public static final String GLUE_HIVE_CATALOG_FACTORY_CLASS = diff --git a/spark/src/test/java/org/opensearch/sql/spark/constants/TestConstants.java b/spark/src/test/java/org/opensearch/sql/spark/constants/TestConstants.java index b06b2e4cc3..09a3163d98 100644 --- a/spark/src/test/java/org/opensearch/sql/spark/constants/TestConstants.java +++ b/spark/src/test/java/org/opensearch/sql/spark/constants/TestConstants.java @@ -19,7 +19,7 @@ public class TestConstants { public static final String MOCK_SESSION_ID = "s-0123456"; public static final String MOCK_STATEMENT_ID = "st-0123456"; public static final String ENTRY_POINT_START_JAR = - "file:///home/hadoop/.ivy2/jars/org.opensearch_opensearch-spark-sql-application_2.12-0.1.0-SNAPSHOT.jar"; + "file:///home/hadoop/.ivy2/jars/org.opensearch_opensearch-spark-sql-application_2.12-0.3.0-SNAPSHOT.jar"; public static final String DEFAULT_RESULT_INDEX = "query_execution_result_ds1"; public static final String US_EAST_REGION = "us-east-1"; public static final String US_WEST_REGION = "us-west-1"; diff --git a/spark/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java b/spark/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java index fc4bfb1923..9f58f7708d 100644 --- a/spark/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java +++ b/spark/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java @@ -953,7 +953,7 @@ private String constructExpectedSparkSubmitParameterString( + " --conf" + " spark.hadoop.aws.catalog.credentials.provider.factory.class=com.amazonaws.glue.catalog.metastore.STSAssumeRoleSessionCredentialsProviderFactory" + " --conf" - + " spark.jars.packages=org.opensearch:opensearch-spark-standalone_2.12:0.1.0-SNAPSHOT,org.opensearch:opensearch-spark-sql-application_2.12:0.1.0-SNAPSHOT,org.opensearch:opensearch-spark-ppl_2.12:0.1.0-SNAPSHOT" + + " spark.jars.packages=org.opensearch:opensearch-spark-standalone_2.12:0.3.0-SNAPSHOT,org.opensearch:opensearch-spark-sql-application_2.12:0.3.0-SNAPSHOT,org.opensearch:opensearch-spark-ppl_2.12:0.3.0-SNAPSHOT" + " --conf" + " spark.jars.repositories=https://aws.oss.sonatype.org/content/repositories/snapshots" + " --conf"