From f73f53abb50f50b68c083710cacb20a77a4e5053 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Tue, 23 Jul 2024 10:03:36 -0700 Subject: [PATCH 1/2] Revert OpenSearch to 2.6 Signed-off-by: Peng Huo --- build.sbt | 2 +- .../ResourceBasedAWSRequestSigningApacheInterceptor.java | 2 +- .../opensearch/flint/core/metadata/FlintJsonHelper.scala | 3 +-- .../flint/core/storage/FlintOpenSearchClient.java | 6 +++--- .../flint/core/storage/OpenSearchClientUtils.java | 2 +- .../flint/core/storage/OpenSearchScrollReader.java | 2 +- .../opensearch/flint/core/storage/OpenSearchWriter.java | 2 +- .../src/main/scala/org/apache/spark/sql/FlintREPL.scala | 2 +- .../src/main/scala/org/apache/spark/sql/OSClient.scala | 7 +++---- 9 files changed, 13 insertions(+), 15 deletions(-) diff --git a/build.sbt b/build.sbt index c190c43f3..b1d9f9b04 100644 --- a/build.sbt +++ b/build.sbt @@ -6,7 +6,7 @@ import Dependencies._ lazy val scala212 = "2.12.14" lazy val sparkVersion = "3.3.2" -lazy val opensearchVersion = "2.11.1" +lazy val opensearchVersion = "2.6.0" lazy val icebergVersion = "1.5.0" val scalaMinorVersion = scala212.split("\\.").take(2).mkString(".") diff --git a/flint-core/src/main/scala/org/opensearch/flint/core/auth/ResourceBasedAWSRequestSigningApacheInterceptor.java b/flint-core/src/main/scala/org/opensearch/flint/core/auth/ResourceBasedAWSRequestSigningApacheInterceptor.java index 66f2e359c..b69343730 100644 --- a/flint-core/src/main/scala/org/opensearch/flint/core/auth/ResourceBasedAWSRequestSigningApacheInterceptor.java +++ b/flint-core/src/main/scala/org/opensearch/flint/core/auth/ResourceBasedAWSRequestSigningApacheInterceptor.java @@ -13,7 +13,7 @@ import org.apache.http.client.utils.URIBuilder; import org.apache.http.protocol.HttpContext; import org.jetbrains.annotations.TestOnly; -import org.opensearch.core.common.Strings; +import org.opensearch.common.Strings; import software.amazon.awssdk.authcrt.signer.AwsCrtV4aSigner; import java.io.IOException; diff --git a/flint-core/src/main/scala/org/opensearch/flint/core/metadata/FlintJsonHelper.scala b/flint-core/src/main/scala/org/opensearch/flint/core/metadata/FlintJsonHelper.scala index 8cdf4a2af..4c1991edc 100644 --- a/flint-core/src/main/scala/org/opensearch/flint/core/metadata/FlintJsonHelper.scala +++ b/flint-core/src/main/scala/org/opensearch/flint/core/metadata/FlintJsonHelper.scala @@ -7,10 +7,9 @@ package org.opensearch.flint.core.metadata import java.nio.charset.StandardCharsets.UTF_8 +import org.opensearch.common.bytes.BytesReference import org.opensearch.common.xcontent._ import org.opensearch.common.xcontent.json.JsonXContent -import org.opensearch.core.common.bytes.BytesReference -import org.opensearch.core.xcontent.{DeprecationHandler, NamedXContentRegistry, XContentBuilder, XContentParser} /** * JSON parsing and building helper. diff --git a/flint-core/src/main/scala/org/opensearch/flint/core/storage/FlintOpenSearchClient.java b/flint-core/src/main/scala/org/opensearch/flint/core/storage/FlintOpenSearchClient.java index 08e606ea5..e60080e54 100644 --- a/flint-core/src/main/scala/org/opensearch/flint/core/storage/FlintOpenSearchClient.java +++ b/flint-core/src/main/scala/org/opensearch/flint/core/storage/FlintOpenSearchClient.java @@ -14,9 +14,9 @@ import org.opensearch.client.indices.PutMappingRequest; import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.NamedXContentRegistry; +import org.opensearch.common.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.xcontent.XContentParser; import org.opensearch.flint.core.FlintClient; import org.opensearch.flint.core.FlintOptions; import org.opensearch.flint.core.IRestHighLevelClient; @@ -39,7 +39,7 @@ import java.util.logging.Logger; import java.util.stream.Collectors; -import static org.opensearch.core.xcontent.DeprecationHandler.IGNORE_DEPRECATIONS; +import static org.opensearch.common.xcontent.DeprecationHandler.IGNORE_DEPRECATIONS; /** * Flint client implementation for OpenSearch storage. diff --git a/flint-core/src/main/scala/org/opensearch/flint/core/storage/OpenSearchClientUtils.java b/flint-core/src/main/scala/org/opensearch/flint/core/storage/OpenSearchClientUtils.java index e002e132b..9277a17df 100644 --- a/flint-core/src/main/scala/org/opensearch/flint/core/storage/OpenSearchClientUtils.java +++ b/flint-core/src/main/scala/org/opensearch/flint/core/storage/OpenSearchClientUtils.java @@ -18,7 +18,7 @@ import org.opensearch.client.RestClient; import org.opensearch.client.RestClientBuilder; import org.opensearch.client.RestHighLevelClient; -import org.opensearch.core.common.Strings; +import org.opensearch.common.Strings; import org.opensearch.flint.core.FlintOptions; import org.opensearch.flint.core.IRestHighLevelClient; import org.opensearch.flint.core.RestHighLevelClientWrapper; diff --git a/flint-core/src/main/scala/org/opensearch/flint/core/storage/OpenSearchScrollReader.java b/flint-core/src/main/scala/org/opensearch/flint/core/storage/OpenSearchScrollReader.java index 3c671e77d..bfcf15636 100644 --- a/flint-core/src/main/scala/org/opensearch/flint/core/storage/OpenSearchScrollReader.java +++ b/flint-core/src/main/scala/org/opensearch/flint/core/storage/OpenSearchScrollReader.java @@ -12,7 +12,7 @@ import org.opensearch.action.search.SearchScrollRequest; import org.opensearch.client.RequestOptions; import org.opensearch.common.unit.TimeValue; -import org.opensearch.core.common.Strings; +import org.opensearch.common.Strings; import org.opensearch.flint.core.FlintOptions; import org.opensearch.flint.core.IRestHighLevelClient; import org.opensearch.search.builder.SearchSourceBuilder; diff --git a/flint-core/src/main/scala/org/opensearch/flint/core/storage/OpenSearchWriter.java b/flint-core/src/main/scala/org/opensearch/flint/core/storage/OpenSearchWriter.java index f4c1cb0c9..0b71f851b 100644 --- a/flint-core/src/main/scala/org/opensearch/flint/core/storage/OpenSearchWriter.java +++ b/flint-core/src/main/scala/org/opensearch/flint/core/storage/OpenSearchWriter.java @@ -11,7 +11,7 @@ import org.opensearch.action.bulk.BulkResponse; import org.opensearch.client.RequestOptions; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.rest.RestStatus; +import org.opensearch.rest.RestStatus; import org.opensearch.flint.core.IRestHighLevelClient; import java.io.ByteArrayOutputStream; diff --git a/spark-sql-application/src/main/scala/org/apache/spark/sql/FlintREPL.scala b/spark-sql-application/src/main/scala/org/apache/spark/sql/FlintREPL.scala index 43d997671..8cad8844b 100644 --- a/spark-sql-application/src/main/scala/org/apache/spark/sql/FlintREPL.scala +++ b/spark-sql-application/src/main/scala/org/apache/spark/sql/FlintREPL.scala @@ -17,7 +17,7 @@ import scala.util.control.NonFatal import com.codahale.metrics.Timer import org.json4s.native.Serialization import org.opensearch.action.get.GetResponse -import org.opensearch.core.common.Strings +import org.opensearch.common.Strings import org.opensearch.flint.core.FlintOptions import org.opensearch.flint.core.logging.CustomLogging import org.opensearch.flint.core.metrics.MetricConstants diff --git a/spark-sql-application/src/main/scala/org/apache/spark/sql/OSClient.scala b/spark-sql-application/src/main/scala/org/apache/spark/sql/OSClient.scala index 155f43e84..ec9901b73 100644 --- a/spark-sql-application/src/main/scala/org/apache/spark/sql/OSClient.scala +++ b/spark-sql-application/src/main/scala/org/apache/spark/sql/OSClient.scala @@ -14,11 +14,10 @@ import scala.util.{Failure, Success, Try} import org.opensearch.action.get.{GetRequest, GetResponse} import org.opensearch.client.RequestOptions import org.opensearch.client.indices.{CreateIndexRequest, GetIndexRequest} +import org.opensearch.common.Strings import org.opensearch.common.settings.Settings -import org.opensearch.common.xcontent.XContentType -import org.opensearch.core.common.Strings -import org.opensearch.core.xcontent.DeprecationHandler.IGNORE_DEPRECATIONS -import org.opensearch.core.xcontent.NamedXContentRegistry +import org.opensearch.common.xcontent.{NamedXContentRegistry, XContentType} +import org.opensearch.common.xcontent.DeprecationHandler.IGNORE_DEPRECATIONS import org.opensearch.flint.core.{FlintClient, FlintClientBuilder, FlintOptions, IRestHighLevelClient} import org.opensearch.flint.core.metrics.MetricConstants import org.opensearch.flint.core.storage.{FlintReader, OpenSearchQueryReader, OpenSearchScrollReader, OpenSearchUpdater} From e60b3050d0aa43bcda42e369d4b00c10398c6ece Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Tue, 23 Jul 2024 11:56:12 -0700 Subject: [PATCH 2/2] add comments Signed-off-by: Peng Huo --- build.sbt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build.sbt b/build.sbt index b1d9f9b04..6ed0ec537 100644 --- a/build.sbt +++ b/build.sbt @@ -6,6 +6,9 @@ import Dependencies._ lazy val scala212 = "2.12.14" lazy val sparkVersion = "3.3.2" + +// The transitive opensearch jackson-databind dependency version should align with Spark jackson databind dependency version. +// Issue: https://github.com/opensearch-project/opensearch-spark/issues/442 lazy val opensearchVersion = "2.6.0" lazy val icebergVersion = "1.5.0" @@ -66,6 +69,8 @@ lazy val flintCore = (project in file("flint-core")) "org.opensearch.client" % "opensearch-rest-high-level-client" % opensearchVersion exclude ("org.apache.logging.log4j", "log4j-api"), "org.opensearch.client" % "opensearch-java" % opensearchVersion + // error: Scala module 2.13.4 requires Jackson Databind version >= 2.13.0 and < 2.14.0 - + // Found jackson-databind version 2.14. exclude ("com.fasterxml.jackson.core", "jackson-databind") exclude ("com.fasterxml.jackson.core", "jackson-core") exclude ("org.apache.httpcomponents.client5", "httpclient5"),