diff --git a/async-query-core/build.gradle b/async-query-core/build.gradle index 3673872988..e275793b13 100644 --- a/async-query-core/build.gradle +++ b/async-query-core/build.gradle @@ -28,7 +28,7 @@ tasks.register('downloadG4Files', Exec) { } generateGrammarSource { - arguments += ['-visitor', '-package', 'org.opensearch.sql.asyncquery.antlr.parser'] + arguments += ['-visitor', '-package', 'org.opensearch.sql.spark.antlr.parser'] source = sourceSets.main.antlr outputDirectory = file("build/generated-src/antlr/main/org/opensearch/sql/asyncquery/antlr/parser") } @@ -44,6 +44,11 @@ generateGrammarSource.dependsOn downloadG4Files dependencies { antlr "org.antlr:antlr4:4.7.1" + implementation project(':core') + implementation project(':spark') // TODO: dependency to spark should be eliminated + implementation project(':datasources') // TODO: dependency to datasources should be eliminated + implementation project(':legacy') // TODO: dependency to legacy should be eliminated + implementation group: 'org.json', name: 'json', version: '20231013' implementation group: 'com.google.guava', name: 'guava', version: '32.0.1-jre' implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${versions.jackson}" implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${versions.jackson_databind}" @@ -108,7 +113,19 @@ jacocoTestCoverageVerification { violationRules { rule { element = 'CLASS' - excludes = [] + //　TODO: Add unit tests in async-query-core and remove exclusions + excludes = [ + 'org.opensearch.sql.spark.asyncquery.model.*', + 'org.opensearch.sql.spark.data.constants.*', + 'org.opensearch.sql.spark.dispatcher.model.*', + 'org.opensearch.sql.spark.dispatcher.*', + 'org.opensearch.sql.spark.execution.session.*', + 'org.opensearch.sql.spark.execution.statement.*', + 'org.opensearch.sql.spark.flint.*', + 'org.opensearch.sql.spark.flint.operation.*', + 'org.opensearch.sql.spark.rest.*', + 'org.opensearch.sql.spark.utils.SQLQueryUtils.*' + ] limit { counter = 'LINE' minimum = 1.0 diff --git a/async-query-core/src/main/antlr/.gitkeep b/async-query-core/src/main/antlr/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/spark/src/main/antlr/FlintSparkSqlExtensions.g4 b/async-query-core/src/main/antlr/FlintSparkSqlExtensions.g4 similarity index 100% rename from spark/src/main/antlr/FlintSparkSqlExtensions.g4 rename to async-query-core/src/main/antlr/FlintSparkSqlExtensions.g4 diff --git a/spark/src/main/antlr/SparkSqlBase.g4 b/async-query-core/src/main/antlr/SparkSqlBase.g4 similarity index 100% rename from spark/src/main/antlr/SparkSqlBase.g4 rename to async-query-core/src/main/antlr/SparkSqlBase.g4 diff --git a/spark/src/main/antlr/SqlBaseLexer.g4 b/async-query-core/src/main/antlr/SqlBaseLexer.g4 similarity index 100% rename from spark/src/main/antlr/SqlBaseLexer.g4 rename to async-query-core/src/main/antlr/SqlBaseLexer.g4 diff --git a/spark/src/main/antlr/SqlBaseParser.g4 b/async-query-core/src/main/antlr/SqlBaseParser.g4 similarity index 100% rename from spark/src/main/antlr/SqlBaseParser.g4 rename to async-query-core/src/main/antlr/SqlBaseParser.g4 diff --git a/async-query-core/src/main/java/org/opensearch/sql/asyncquery/Dummy.java b/async-query-core/src/main/java/org/opensearch/sql/asyncquery/Dummy.java deleted file mode 100644 index b7ab572f2a..0000000000 --- a/async-query-core/src/main/java/org/opensearch/sql/asyncquery/Dummy.java +++ /dev/null @@ -1,13 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.asyncquery; - -// This is a dummy class for scaffolding and should be deleted later -public class Dummy { - public String hello() { - return "Hello!"; - } -} diff --git a/spark/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorService.java b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorService.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorService.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorService.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImpl.java b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImpl.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImpl.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImpl.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryJobMetadataStorageService.java b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryJobMetadataStorageService.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryJobMetadataStorageService.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryJobMetadataStorageService.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/asyncquery/exceptions/AsyncQueryNotFoundException.java b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/exceptions/AsyncQueryNotFoundException.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/asyncquery/exceptions/AsyncQueryNotFoundException.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/exceptions/AsyncQueryNotFoundException.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryExecutionResponse.java b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryExecutionResponse.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryExecutionResponse.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryExecutionResponse.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryId.java b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryId.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryId.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryId.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryJobMetadata.java b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryJobMetadata.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryJobMetadata.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryJobMetadata.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryRequestContext.java b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryRequestContext.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryRequestContext.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryRequestContext.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryResult.java b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryResult.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryResult.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/AsyncQueryResult.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/asyncquery/model/NullAsyncQueryRequestContext.java b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/NullAsyncQueryRequestContext.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/asyncquery/model/NullAsyncQueryRequestContext.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/NullAsyncQueryRequestContext.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/asyncquery/model/SparkSubmitParameters.java b/async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/SparkSubmitParameters.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/asyncquery/model/SparkSubmitParameters.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/asyncquery/model/SparkSubmitParameters.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/client/EMRServerlessClient.java b/async-query-core/src/main/java/org/opensearch/sql/spark/client/EMRServerlessClient.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/client/EMRServerlessClient.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/client/EMRServerlessClient.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/client/EMRServerlessClientFactory.java b/async-query-core/src/main/java/org/opensearch/sql/spark/client/EMRServerlessClientFactory.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/client/EMRServerlessClientFactory.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/client/EMRServerlessClientFactory.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/client/EMRServerlessClientFactoryImpl.java b/async-query-core/src/main/java/org/opensearch/sql/spark/client/EMRServerlessClientFactoryImpl.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/client/EMRServerlessClientFactoryImpl.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/client/EMRServerlessClientFactoryImpl.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/client/EmrServerlessClientImpl.java b/async-query-core/src/main/java/org/opensearch/sql/spark/client/EmrServerlessClientImpl.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/client/EmrServerlessClientImpl.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/client/EmrServerlessClientImpl.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/client/StartJobRequest.java b/async-query-core/src/main/java/org/opensearch/sql/spark/client/StartJobRequest.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/client/StartJobRequest.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/client/StartJobRequest.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfig.java b/async-query-core/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfig.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfig.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfig.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigClusterSetting.java b/async-query-core/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigClusterSetting.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigClusterSetting.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigClusterSetting.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigSupplier.java b/async-query-core/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigSupplier.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigSupplier.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigSupplier.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/config/SparkSubmitParameterModifier.java b/async-query-core/src/main/java/org/opensearch/sql/spark/config/SparkSubmitParameterModifier.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/config/SparkSubmitParameterModifier.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/config/SparkSubmitParameterModifier.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java b/async-query-core/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java similarity index 94% rename from spark/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java index b9436b0801..5b25bc175a 100644 --- a/spark/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java +++ b/async-query-core/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java @@ -6,8 +6,6 @@ package org.opensearch.sql.spark.data.constants; public class SparkConstants { - public static final String EMR = "emr"; - public static final String STEP_ID_FIELD = "stepId.keyword"; public static final String JOB_ID_FIELD = "jobRunId"; @@ -21,16 +19,11 @@ public class SparkConstants { public static final String SPARK_SQL_APPLICATION_JAR = "file:///home/hadoop/.ivy2/jars/org.opensearch_opensearch-spark-sql-application_2.12-0.3.0-SNAPSHOT.jar"; public static final String SPARK_REQUEST_BUFFER_INDEX_NAME = ".query_execution_request"; - // TODO should be replaced with mvn jar. - public static final String FLINT_INTEGRATION_JAR = - "s3://spark-datasource/flint-spark-integration-assembly-0.3.0-SNAPSHOT.jar"; - // TODO should be replaced with mvn jar. public static final String FLINT_DEFAULT_CLUSTER_NAME = "opensearch-cluster"; public static final String FLINT_DEFAULT_HOST = "localhost"; public static final String FLINT_DEFAULT_PORT = "9200"; public static final String FLINT_DEFAULT_SCHEME = "http"; public static final String FLINT_DEFAULT_AUTH = "noauth"; - public static final String FLINT_DEFAULT_REGION = "us-west-2"; public static final String DEFAULT_CLASS_NAME = "org.apache.spark.sql.FlintJob"; public static final String S3_AWS_CREDENTIALS_PROVIDER_KEY = "spark.hadoop.fs.s3.customAWSCredentialsProvider"; diff --git a/spark/src/main/java/org/opensearch/sql/spark/dispatcher/AsyncQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/AsyncQueryHandler.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/dispatcher/AsyncQueryHandler.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/AsyncQueryHandler.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/BatchQueryHandler.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/dispatcher/DatasourceEmbeddedQueryIdProvider.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/DatasourceEmbeddedQueryIdProvider.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/dispatcher/DatasourceEmbeddedQueryIdProvider.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/DatasourceEmbeddedQueryIdProvider.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandler.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/dispatcher/InteractiveQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/InteractiveQueryHandler.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/dispatcher/InteractiveQueryHandler.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/InteractiveQueryHandler.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/dispatcher/QueryHandlerFactory.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/QueryHandlerFactory.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/dispatcher/QueryHandlerFactory.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/QueryHandlerFactory.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/dispatcher/QueryIdProvider.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/QueryIdProvider.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/dispatcher/QueryIdProvider.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/QueryIdProvider.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/dispatcher/RefreshQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/RefreshQueryHandler.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/dispatcher/RefreshQueryHandler.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/RefreshQueryHandler.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcher.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/dispatcher/StreamingQueryHandler.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/StreamingQueryHandler.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/dispatcher/StreamingQueryHandler.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/StreamingQueryHandler.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/dispatcher/model/DispatchQueryContext.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/DispatchQueryContext.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/dispatcher/model/DispatchQueryContext.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/DispatchQueryContext.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/dispatcher/model/DispatchQueryRequest.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/DispatchQueryRequest.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/dispatcher/model/DispatchQueryRequest.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/DispatchQueryRequest.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/dispatcher/model/DispatchQueryResponse.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/DispatchQueryResponse.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/dispatcher/model/DispatchQueryResponse.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/DispatchQueryResponse.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/dispatcher/model/FlintIndexOptions.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/FlintIndexOptions.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/dispatcher/model/FlintIndexOptions.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/FlintIndexOptions.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/dispatcher/model/FullyQualifiedTableName.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/FullyQualifiedTableName.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/dispatcher/model/FullyQualifiedTableName.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/FullyQualifiedTableName.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/dispatcher/model/IndexDMLResult.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/IndexDMLResult.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/dispatcher/model/IndexDMLResult.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/IndexDMLResult.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/dispatcher/model/IndexQueryActionType.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/IndexQueryActionType.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/dispatcher/model/IndexQueryActionType.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/IndexQueryActionType.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/dispatcher/model/IndexQueryDetails.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/IndexQueryDetails.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/dispatcher/model/IndexQueryDetails.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/IndexQueryDetails.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/dispatcher/model/JobType.java b/async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/JobType.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/dispatcher/model/JobType.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/dispatcher/model/JobType.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/session/CreateSessionRequest.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/CreateSessionRequest.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/session/CreateSessionRequest.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/CreateSessionRequest.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/session/DatasourceEmbeddedSessionIdProvider.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/DatasourceEmbeddedSessionIdProvider.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/session/DatasourceEmbeddedSessionIdProvider.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/DatasourceEmbeddedSessionIdProvider.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/session/InteractiveSession.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/InteractiveSession.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/session/InteractiveSession.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/InteractiveSession.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/session/Session.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/Session.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/session/Session.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/Session.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/session/SessionConfigSupplier.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/SessionConfigSupplier.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/session/SessionConfigSupplier.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/SessionConfigSupplier.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/session/SessionIdProvider.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/SessionIdProvider.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/session/SessionIdProvider.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/SessionIdProvider.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/session/SessionManager.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/SessionManager.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/session/SessionManager.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/SessionManager.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/session/SessionModel.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/SessionModel.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/session/SessionModel.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/SessionModel.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/session/SessionState.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/SessionState.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/session/SessionState.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/SessionState.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/session/SessionType.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/SessionType.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/session/SessionType.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/execution/session/SessionType.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/statement/QueryRequest.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/statement/QueryRequest.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/statement/QueryRequest.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/execution/statement/QueryRequest.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/statement/Statement.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/statement/Statement.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/statement/Statement.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/execution/statement/Statement.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/statement/StatementId.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/statement/StatementId.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/statement/StatementId.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/execution/statement/StatementId.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/statement/StatementModel.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/statement/StatementModel.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/statement/StatementModel.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/execution/statement/StatementModel.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/statement/StatementState.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/statement/StatementState.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/statement/StatementState.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/execution/statement/StatementState.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/statestore/CopyBuilder.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/statestore/CopyBuilder.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/statestore/CopyBuilder.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/execution/statestore/CopyBuilder.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/statestore/OpenSearchStateStoreUtil.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/statestore/OpenSearchStateStoreUtil.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/statestore/OpenSearchStateStoreUtil.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/execution/statestore/OpenSearchStateStoreUtil.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/statestore/SessionStorageService.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/statestore/SessionStorageService.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/statestore/SessionStorageService.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/execution/statestore/SessionStorageService.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/statestore/StateCopyBuilder.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/statestore/StateCopyBuilder.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/statestore/StateCopyBuilder.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/execution/statestore/StateCopyBuilder.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/statestore/StateModel.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/statestore/StateModel.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/statestore/StateModel.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/execution/statestore/StateModel.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/statestore/StatementStorageService.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/statestore/StatementStorageService.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/statestore/StatementStorageService.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/execution/statestore/StatementStorageService.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/xcontent/XContentSerializerUtil.java b/async-query-core/src/main/java/org/opensearch/sql/spark/execution/xcontent/XContentSerializerUtil.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/xcontent/XContentSerializerUtil.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/execution/xcontent/XContentSerializerUtil.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadata.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadata.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadata.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadata.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataService.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataService.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataService.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataService.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/flint/FlintIndexState.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/FlintIndexState.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/flint/FlintIndexState.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/flint/FlintIndexState.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/flint/FlintIndexStateModel.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/FlintIndexStateModel.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/flint/FlintIndexStateModel.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/flint/FlintIndexStateModel.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/flint/FlintIndexStateModelService.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/FlintIndexStateModelService.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/flint/FlintIndexStateModelService.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/flint/FlintIndexStateModelService.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/flint/FlintIndexType.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/FlintIndexType.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/flint/FlintIndexType.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/flint/FlintIndexType.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/flint/IndexDMLResultStorageService.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/IndexDMLResultStorageService.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/flint/IndexDMLResultStorageService.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/flint/IndexDMLResultStorageService.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOp.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOp.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOp.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOp.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpAlter.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpAlter.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpAlter.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpAlter.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpCancel.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpCancel.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpCancel.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpCancel.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpDrop.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpDrop.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpDrop.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpDrop.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpFactory.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpFactory.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpFactory.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpFactory.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuum.java b/async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuum.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuum.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpVacuum.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/leasemanager/ConcurrencyLimitExceededException.java b/async-query-core/src/main/java/org/opensearch/sql/spark/leasemanager/ConcurrencyLimitExceededException.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/leasemanager/ConcurrencyLimitExceededException.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/leasemanager/ConcurrencyLimitExceededException.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/leasemanager/LeaseManager.java b/async-query-core/src/main/java/org/opensearch/sql/spark/leasemanager/LeaseManager.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/leasemanager/LeaseManager.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/leasemanager/LeaseManager.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/leasemanager/model/LeaseRequest.java b/async-query-core/src/main/java/org/opensearch/sql/spark/leasemanager/model/LeaseRequest.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/leasemanager/model/LeaseRequest.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/leasemanager/model/LeaseRequest.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/response/JobExecutionResponseReader.java b/async-query-core/src/main/java/org/opensearch/sql/spark/response/JobExecutionResponseReader.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/response/JobExecutionResponseReader.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/response/JobExecutionResponseReader.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/rest/model/CreateAsyncQueryRequest.java b/async-query-core/src/main/java/org/opensearch/sql/spark/rest/model/CreateAsyncQueryRequest.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/rest/model/CreateAsyncQueryRequest.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/rest/model/CreateAsyncQueryRequest.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/rest/model/CreateAsyncQueryResponse.java b/async-query-core/src/main/java/org/opensearch/sql/spark/rest/model/CreateAsyncQueryResponse.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/rest/model/CreateAsyncQueryResponse.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/rest/model/CreateAsyncQueryResponse.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/rest/model/LangType.java b/async-query-core/src/main/java/org/opensearch/sql/spark/rest/model/LangType.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/rest/model/LangType.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/rest/model/LangType.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/utils/IDUtils.java b/async-query-core/src/main/java/org/opensearch/sql/spark/utils/IDUtils.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/utils/IDUtils.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/utils/IDUtils.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/utils/RealTimeProvider.java b/async-query-core/src/main/java/org/opensearch/sql/spark/utils/RealTimeProvider.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/utils/RealTimeProvider.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/utils/RealTimeProvider.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java b/async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/utils/TimeProvider.java b/async-query-core/src/main/java/org/opensearch/sql/spark/utils/TimeProvider.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/utils/TimeProvider.java rename to async-query-core/src/main/java/org/opensearch/sql/spark/utils/TimeProvider.java diff --git a/async-query-core/src/test/java/org/opensearch/sql/asyncquery/DummyTest.java b/async-query-core/src/test/java/org/opensearch/sql/asyncquery/DummyTest.java deleted file mode 100644 index 8fa1cf49ec..0000000000 --- a/async-query-core/src/test/java/org/opensearch/sql/asyncquery/DummyTest.java +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.asyncquery; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.junit.jupiter.api.Test; - -public class DummyTest { - @Test - public void test() { - Dummy dummy = new Dummy(); - assertEquals("Hello!", dummy.hello()); - } -} diff --git a/spark/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java similarity index 97% rename from spark/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java rename to async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java index b87fb0dad7..8325a10fbc 100644 --- a/spark/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplTest.java @@ -33,7 +33,7 @@ import org.opensearch.sql.spark.asyncquery.model.AsyncQueryExecutionResponse; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryJobMetadata; import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; -import org.opensearch.sql.spark.config.OpenSearchSparkSubmitParameterModifier; +import org.opensearch.sql.spark.asyncquery.model.SparkSubmitParameters; import org.opensearch.sql.spark.config.SparkExecutionEngineConfig; import org.opensearch.sql.spark.config.SparkExecutionEngineConfigSupplier; import org.opensearch.sql.spark.config.SparkSubmitParameterModifier; @@ -114,8 +114,10 @@ void testCreateAsyncQuery() { @Test void testCreateAsyncQueryWithExtraSparkSubmitParameter() { - OpenSearchSparkSubmitParameterModifier modifier = - new OpenSearchSparkSubmitParameterModifier("--conf spark.dynamicAllocation.enabled=false"); + SparkSubmitParameterModifier modifier = + (SparkSubmitParameters parameters) -> { + parameters.setExtraParameters("--conf spark.dynamicAllocation.enabled=false"); + }; when(sparkExecutionEngineConfigSupplier.getSparkExecutionEngineConfig(any())) .thenReturn( SparkExecutionEngineConfig.builder() diff --git a/spark/src/test/java/org/opensearch/sql/spark/asyncquery/model/SparkSubmitParametersTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/model/SparkSubmitParametersTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/asyncquery/model/SparkSubmitParametersTest.java rename to async-query-core/src/test/java/org/opensearch/sql/spark/asyncquery/model/SparkSubmitParametersTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/client/EMRServerlessClientFactoryImplTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/client/EMRServerlessClientFactoryImplTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/client/EMRServerlessClientFactoryImplTest.java rename to async-query-core/src/test/java/org/opensearch/sql/spark/client/EMRServerlessClientFactoryImplTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/client/EmrServerlessClientImplTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/client/EmrServerlessClientImplTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/client/EmrServerlessClientImplTest.java rename to async-query-core/src/test/java/org/opensearch/sql/spark/client/EmrServerlessClientImplTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/client/StartJobRequestTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/client/StartJobRequestTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/client/StartJobRequestTest.java rename to async-query-core/src/test/java/org/opensearch/sql/spark/client/StartJobRequestTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigClusterSettingTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigClusterSettingTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigClusterSettingTest.java rename to async-query-core/src/test/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigClusterSettingTest.java diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/constants/TestConstants.java b/async-query-core/src/test/java/org/opensearch/sql/spark/constants/TestConstants.java new file mode 100644 index 0000000000..295c74dcee --- /dev/null +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/constants/TestConstants.java @@ -0,0 +1,23 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.constants; + +public class TestConstants { + public static final String QUERY = "select 1"; + public static final String EMR_JOB_ID = "job-123xxx"; + public static final String EMRS_APPLICATION_ID = "app-xxxxx"; + public static final String EMRS_EXECUTION_ROLE = "execution_role"; + public static final String EMRS_JOB_NAME = "job_name"; + public static final String SPARK_SUBMIT_PARAMETERS = "--conf org.flint.sql.SQLJob"; + public static final String TEST_CLUSTER_NAME = "TEST_CLUSTER"; + public static final String MOCK_SESSION_ID = "s-0123456"; + public static final String MOCK_STATEMENT_ID = "st-0123456"; + public static final String ENTRY_POINT_START_JAR = + "file:///home/hadoop/.ivy2/jars/org.opensearch_opensearch-spark-sql-application_2.12-0.3.0-SNAPSHOT.jar"; + public static final String DEFAULT_RESULT_INDEX = "query_execution_result_ds1"; + public static final String US_EAST_REGION = "us-east-1"; + public static final String US_WEST_REGION = "us-west-1"; +} diff --git a/spark/src/test/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandlerTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandlerTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandlerTest.java rename to async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/IndexDMLHandlerTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java rename to async-query-core/src/test/java/org/opensearch/sql/spark/dispatcher/SparkQueryDispatcherTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/execution/session/SessionManagerTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/execution/session/SessionManagerTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/execution/session/SessionManagerTest.java rename to async-query-core/src/test/java/org/opensearch/sql/spark/execution/session/SessionManagerTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/execution/session/SessionStateTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/execution/session/SessionStateTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/execution/session/SessionStateTest.java rename to async-query-core/src/test/java/org/opensearch/sql/spark/execution/session/SessionStateTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/execution/session/SessionTypeTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/execution/session/SessionTypeTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/execution/session/SessionTypeTest.java rename to async-query-core/src/test/java/org/opensearch/sql/spark/execution/session/SessionTypeTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/execution/statement/StatementStateTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/execution/statement/StatementStateTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/execution/statement/StatementStateTest.java rename to async-query-core/src/test/java/org/opensearch/sql/spark/execution/statement/StatementStateTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/execution/statestore/OpenSearchStateStoreUtilTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/execution/statestore/OpenSearchStateStoreUtilTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/execution/statestore/OpenSearchStateStoreUtilTest.java rename to async-query-core/src/test/java/org/opensearch/sql/spark/execution/statestore/OpenSearchStateStoreUtilTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/execution/statestore/StateModelTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/execution/statestore/StateModelTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/execution/statestore/StateModelTest.java rename to async-query-core/src/test/java/org/opensearch/sql/spark/execution/statestore/StateModelTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/execution/xcontent/XContentSerializerUtilTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/execution/xcontent/XContentSerializerUtilTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/execution/xcontent/XContentSerializerUtilTest.java rename to async-query-core/src/test/java/org/opensearch/sql/spark/execution/xcontent/XContentSerializerUtilTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/flint/FlintIndexStateTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/flint/FlintIndexStateTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/flint/FlintIndexStateTest.java rename to async-query-core/src/test/java/org/opensearch/sql/spark/flint/FlintIndexStateTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/flint/IndexQueryDetailsTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/flint/IndexQueryDetailsTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/flint/IndexQueryDetailsTest.java rename to async-query-core/src/test/java/org/opensearch/sql/spark/flint/IndexQueryDetailsTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpTest.java rename to async-query-core/src/test/java/org/opensearch/sql/spark/flint/operation/FlintIndexOpTest.java diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/leasemanager/ConcurrencyLimitExceededExceptionTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/leasemanager/ConcurrencyLimitExceededExceptionTest.java new file mode 100644 index 0000000000..c0591eaf66 --- /dev/null +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/leasemanager/ConcurrencyLimitExceededExceptionTest.java @@ -0,0 +1,19 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.leasemanager; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class ConcurrencyLimitExceededExceptionTest { + @Test + public void test() { + ConcurrencyLimitExceededException e = new ConcurrencyLimitExceededException("Test"); + + assertEquals("Test", e.getMessage()); + } +} diff --git a/spark/src/test/java/org/opensearch/sql/spark/rest/model/CreateAsyncQueryRequestTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/rest/model/CreateAsyncQueryRequestTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/rest/model/CreateAsyncQueryRequestTest.java rename to async-query-core/src/test/java/org/opensearch/sql/spark/rest/model/CreateAsyncQueryRequestTest.java diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/utils/IDUtilsTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/utils/IDUtilsTest.java new file mode 100644 index 0000000000..1893256c39 --- /dev/null +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/utils/IDUtilsTest.java @@ -0,0 +1,33 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class IDUtilsTest { + public static final String DATASOURCE_NAME = "DATASOURCE_NAME"; + + @Test + public void encodeAndDecode() { + String id = IDUtils.encode(DATASOURCE_NAME); + String decoded = IDUtils.decode(id); + + assertTrue(id.length() > IDUtils.PREFIX_LEN); + assertEquals(DATASOURCE_NAME, decoded); + } + + @Test + public void generateUniqueIds() { + String id1 = IDUtils.encode(DATASOURCE_NAME); + String id2 = IDUtils.encode(DATASOURCE_NAME); + + assertNotEquals(id1, id2); + } +} diff --git a/spark/src/test/java/org/opensearch/sql/spark/utils/MockTimeProvider.java b/async-query-core/src/test/java/org/opensearch/sql/spark/utils/MockTimeProvider.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/utils/MockTimeProvider.java rename to async-query-core/src/test/java/org/opensearch/sql/spark/utils/MockTimeProvider.java diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/utils/RealTimeProviderTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/utils/RealTimeProviderTest.java new file mode 100644 index 0000000000..7eb5a56cfe --- /dev/null +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/utils/RealTimeProviderTest.java @@ -0,0 +1,19 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.utils; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class RealTimeProviderTest { + @Test + public void testCurrentEpochMillis() { + RealTimeProvider realTimeProvider = new RealTimeProvider(); + + assertTrue(realTimeProvider.currentEpochMillis() > 0); + } +} diff --git a/spark/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java b/async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java rename to async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java diff --git a/async-query-core/src/test/java/org/opensearch/sql/spark/utils/TestUtils.java b/async-query-core/src/test/java/org/opensearch/sql/spark/utils/TestUtils.java new file mode 100644 index 0000000000..0b5b39d50d --- /dev/null +++ b/async-query-core/src/test/java/org/opensearch/sql/spark/utils/TestUtils.java @@ -0,0 +1,24 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.utils; + +import com.google.common.base.Charsets; +import com.google.common.io.Resources; +import java.io.IOException; +import java.net.URL; +import java.util.Objects; +import lombok.SneakyThrows; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.client.Client; +import org.opensearch.common.xcontent.XContentType; + +public class TestUtils { + public static String getJson(String filename) throws IOException { + ClassLoader classLoader = TestUtils.class.getClassLoader(); + return new String( + Objects.requireNonNull(classLoader.getResourceAsStream(filename)).readAllBytes()); + } +} diff --git a/spark/src/test/resources/invalid_response.json b/async-query-core/src/test/resources/invalid_response.json similarity index 100% rename from spark/src/test/resources/invalid_response.json rename to async-query-core/src/test/resources/invalid_response.json diff --git a/async-query-core/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/async-query-core/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 0000000000..ca6ee9cea8 --- /dev/null +++ b/async-query-core/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline \ No newline at end of file diff --git a/async-query-core/src/test/resources/select_query_response.json b/async-query-core/src/test/resources/select_query_response.json new file mode 100644 index 0000000000..24cb06b49e --- /dev/null +++ b/async-query-core/src/test/resources/select_query_response.json @@ -0,0 +1,12 @@ +{ + "data": { + "result": [ + "{'1':1}" + ], + "schema": [ + "{'column_name':'1','data_type':'integer'}" + ], + "stepId": "s-123456789", + "applicationId": "application-abc" + } +} diff --git a/async-query/build.gradle b/async-query/build.gradle index ee40e5b366..bf8cafe1e7 100644 --- a/async-query/build.gradle +++ b/async-query/build.gradle @@ -17,7 +17,7 @@ repositories { dependencies { api project(':core') - implementation project(':async-query-core') + api project(':async-query-core') implementation project(':protocol') implementation project(':datasources') implementation project(':legacy') diff --git a/async-query/src/main/java/org/opensearch/sql/asyncquery/DummyConsumer.java b/async-query/src/main/java/org/opensearch/sql/asyncquery/DummyConsumer.java deleted file mode 100644 index 9b1641e559..0000000000 --- a/async-query/src/main/java/org/opensearch/sql/asyncquery/DummyConsumer.java +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.asyncquery; - -import lombok.AllArgsConstructor; - -// This is a dummy class for scaffolding and should be deleted later -@AllArgsConstructor -public class DummyConsumer { - Dummy dummy; - - public String hello() { - return dummy.hello(); - } -} diff --git a/spark/src/main/java/org/opensearch/sql/spark/asyncquery/OpenSearchAsyncQueryJobMetadataStorageService.java b/async-query/src/main/java/org/opensearch/sql/spark/asyncquery/OpenSearchAsyncQueryJobMetadataStorageService.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/asyncquery/OpenSearchAsyncQueryJobMetadataStorageService.java rename to async-query/src/main/java/org/opensearch/sql/spark/asyncquery/OpenSearchAsyncQueryJobMetadataStorageService.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/cluster/ClusterManagerEventListener.java b/async-query/src/main/java/org/opensearch/sql/spark/cluster/ClusterManagerEventListener.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/cluster/ClusterManagerEventListener.java rename to async-query/src/main/java/org/opensearch/sql/spark/cluster/ClusterManagerEventListener.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/cluster/FlintIndexRetention.java b/async-query/src/main/java/org/opensearch/sql/spark/cluster/FlintIndexRetention.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/cluster/FlintIndexRetention.java rename to async-query/src/main/java/org/opensearch/sql/spark/cluster/FlintIndexRetention.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/cluster/FlintStreamingJobHouseKeeperTask.java b/async-query/src/main/java/org/opensearch/sql/spark/cluster/FlintStreamingJobHouseKeeperTask.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/cluster/FlintStreamingJobHouseKeeperTask.java rename to async-query/src/main/java/org/opensearch/sql/spark/cluster/FlintStreamingJobHouseKeeperTask.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/cluster/IndexCleanup.java b/async-query/src/main/java/org/opensearch/sql/spark/cluster/IndexCleanup.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/cluster/IndexCleanup.java rename to async-query/src/main/java/org/opensearch/sql/spark/cluster/IndexCleanup.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/config/OpenSearchSparkSubmitParameterModifier.java b/async-query/src/main/java/org/opensearch/sql/spark/config/OpenSearchSparkSubmitParameterModifier.java similarity index 84% rename from spark/src/main/java/org/opensearch/sql/spark/config/OpenSearchSparkSubmitParameterModifier.java rename to async-query/src/main/java/org/opensearch/sql/spark/config/OpenSearchSparkSubmitParameterModifier.java index f1831c9786..a034e04095 100644 --- a/spark/src/main/java/org/opensearch/sql/spark/config/OpenSearchSparkSubmitParameterModifier.java +++ b/async-query/src/main/java/org/opensearch/sql/spark/config/OpenSearchSparkSubmitParameterModifier.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.sql.spark.config; import lombok.AllArgsConstructor; diff --git a/spark/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigSupplierImpl.java b/async-query/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigSupplierImpl.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigSupplierImpl.java rename to async-query/src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigSupplierImpl.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/session/OpenSearchSessionConfigSupplier.java b/async-query/src/main/java/org/opensearch/sql/spark/execution/session/OpenSearchSessionConfigSupplier.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/session/OpenSearchSessionConfigSupplier.java rename to async-query/src/main/java/org/opensearch/sql/spark/execution/session/OpenSearchSessionConfigSupplier.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/statestore/FromXContent.java b/async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/FromXContent.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/statestore/FromXContent.java rename to async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/FromXContent.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/statestore/OpenSearchSessionStorageService.java b/async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/OpenSearchSessionStorageService.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/statestore/OpenSearchSessionStorageService.java rename to async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/OpenSearchSessionStorageService.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/statestore/OpenSearchStatementStorageService.java b/async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/OpenSearchStatementStorageService.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/statestore/OpenSearchStatementStorageService.java rename to async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/OpenSearchStatementStorageService.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/statestore/StateStore.java b/async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/StateStore.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/statestore/StateStore.java rename to async-query/src/main/java/org/opensearch/sql/spark/execution/statestore/StateStore.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/xcontent/AsyncQueryJobMetadataXContentSerializer.java b/async-query/src/main/java/org/opensearch/sql/spark/execution/xcontent/AsyncQueryJobMetadataXContentSerializer.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/xcontent/AsyncQueryJobMetadataXContentSerializer.java rename to async-query/src/main/java/org/opensearch/sql/spark/execution/xcontent/AsyncQueryJobMetadataXContentSerializer.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/xcontent/FlintIndexStateModelXContentSerializer.java b/async-query/src/main/java/org/opensearch/sql/spark/execution/xcontent/FlintIndexStateModelXContentSerializer.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/xcontent/FlintIndexStateModelXContentSerializer.java rename to async-query/src/main/java/org/opensearch/sql/spark/execution/xcontent/FlintIndexStateModelXContentSerializer.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/xcontent/IndexDMLResultXContentSerializer.java b/async-query/src/main/java/org/opensearch/sql/spark/execution/xcontent/IndexDMLResultXContentSerializer.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/xcontent/IndexDMLResultXContentSerializer.java rename to async-query/src/main/java/org/opensearch/sql/spark/execution/xcontent/IndexDMLResultXContentSerializer.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/xcontent/SessionModelXContentSerializer.java b/async-query/src/main/java/org/opensearch/sql/spark/execution/xcontent/SessionModelXContentSerializer.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/xcontent/SessionModelXContentSerializer.java rename to async-query/src/main/java/org/opensearch/sql/spark/execution/xcontent/SessionModelXContentSerializer.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/xcontent/StatementModelXContentSerializer.java b/async-query/src/main/java/org/opensearch/sql/spark/execution/xcontent/StatementModelXContentSerializer.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/xcontent/StatementModelXContentSerializer.java rename to async-query/src/main/java/org/opensearch/sql/spark/execution/xcontent/StatementModelXContentSerializer.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/xcontent/XContentCommonAttributes.java b/async-query/src/main/java/org/opensearch/sql/spark/execution/xcontent/XContentCommonAttributes.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/xcontent/XContentCommonAttributes.java rename to async-query/src/main/java/org/opensearch/sql/spark/execution/xcontent/XContentCommonAttributes.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/xcontent/XContentSerializer.java b/async-query/src/main/java/org/opensearch/sql/spark/execution/xcontent/XContentSerializer.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/execution/xcontent/XContentSerializer.java rename to async-query/src/main/java/org/opensearch/sql/spark/execution/xcontent/XContentSerializer.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataServiceImpl.java b/async-query/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataServiceImpl.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataServiceImpl.java rename to async-query/src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataServiceImpl.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexStateModelService.java b/async-query/src/main/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexStateModelService.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexStateModelService.java rename to async-query/src/main/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexStateModelService.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/flint/OpenSearchIndexDMLResultStorageService.java b/async-query/src/main/java/org/opensearch/sql/spark/flint/OpenSearchIndexDMLResultStorageService.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/flint/OpenSearchIndexDMLResultStorageService.java rename to async-query/src/main/java/org/opensearch/sql/spark/flint/OpenSearchIndexDMLResultStorageService.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/leasemanager/DefaultLeaseManager.java b/async-query/src/main/java/org/opensearch/sql/spark/leasemanager/DefaultLeaseManager.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/leasemanager/DefaultLeaseManager.java rename to async-query/src/main/java/org/opensearch/sql/spark/leasemanager/DefaultLeaseManager.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReader.java b/async-query/src/main/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReader.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReader.java rename to async-query/src/main/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReader.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/rest/RestAsyncQueryManagementAction.java b/async-query/src/main/java/org/opensearch/sql/spark/rest/RestAsyncQueryManagementAction.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/rest/RestAsyncQueryManagementAction.java rename to async-query/src/main/java/org/opensearch/sql/spark/rest/RestAsyncQueryManagementAction.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/transport/TransportCancelAsyncQueryRequestAction.java b/async-query/src/main/java/org/opensearch/sql/spark/transport/TransportCancelAsyncQueryRequestAction.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/transport/TransportCancelAsyncQueryRequestAction.java rename to async-query/src/main/java/org/opensearch/sql/spark/transport/TransportCancelAsyncQueryRequestAction.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/transport/TransportCreateAsyncQueryRequestAction.java b/async-query/src/main/java/org/opensearch/sql/spark/transport/TransportCreateAsyncQueryRequestAction.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/transport/TransportCreateAsyncQueryRequestAction.java rename to async-query/src/main/java/org/opensearch/sql/spark/transport/TransportCreateAsyncQueryRequestAction.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/transport/TransportGetAsyncQueryResultAction.java b/async-query/src/main/java/org/opensearch/sql/spark/transport/TransportGetAsyncQueryResultAction.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/transport/TransportGetAsyncQueryResultAction.java rename to async-query/src/main/java/org/opensearch/sql/spark/transport/TransportGetAsyncQueryResultAction.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java b/async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java rename to async-query/src/main/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModule.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/transport/format/AsyncQueryResultResponseFormatter.java b/async-query/src/main/java/org/opensearch/sql/spark/transport/format/AsyncQueryResultResponseFormatter.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/transport/format/AsyncQueryResultResponseFormatter.java rename to async-query/src/main/java/org/opensearch/sql/spark/transport/format/AsyncQueryResultResponseFormatter.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/transport/model/CancelAsyncQueryActionRequest.java b/async-query/src/main/java/org/opensearch/sql/spark/transport/model/CancelAsyncQueryActionRequest.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/transport/model/CancelAsyncQueryActionRequest.java rename to async-query/src/main/java/org/opensearch/sql/spark/transport/model/CancelAsyncQueryActionRequest.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/transport/model/CancelAsyncQueryActionResponse.java b/async-query/src/main/java/org/opensearch/sql/spark/transport/model/CancelAsyncQueryActionResponse.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/transport/model/CancelAsyncQueryActionResponse.java rename to async-query/src/main/java/org/opensearch/sql/spark/transport/model/CancelAsyncQueryActionResponse.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/transport/model/CreateAsyncQueryActionRequest.java b/async-query/src/main/java/org/opensearch/sql/spark/transport/model/CreateAsyncQueryActionRequest.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/transport/model/CreateAsyncQueryActionRequest.java rename to async-query/src/main/java/org/opensearch/sql/spark/transport/model/CreateAsyncQueryActionRequest.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/transport/model/CreateAsyncQueryActionResponse.java b/async-query/src/main/java/org/opensearch/sql/spark/transport/model/CreateAsyncQueryActionResponse.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/transport/model/CreateAsyncQueryActionResponse.java rename to async-query/src/main/java/org/opensearch/sql/spark/transport/model/CreateAsyncQueryActionResponse.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/transport/model/GetAsyncQueryResultActionRequest.java b/async-query/src/main/java/org/opensearch/sql/spark/transport/model/GetAsyncQueryResultActionRequest.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/transport/model/GetAsyncQueryResultActionRequest.java rename to async-query/src/main/java/org/opensearch/sql/spark/transport/model/GetAsyncQueryResultActionRequest.java diff --git a/spark/src/main/java/org/opensearch/sql/spark/transport/model/GetAsyncQueryResultActionResponse.java b/async-query/src/main/java/org/opensearch/sql/spark/transport/model/GetAsyncQueryResultActionResponse.java similarity index 100% rename from spark/src/main/java/org/opensearch/sql/spark/transport/model/GetAsyncQueryResultActionResponse.java rename to async-query/src/main/java/org/opensearch/sql/spark/transport/model/GetAsyncQueryResultActionResponse.java diff --git a/async-query/src/test/java/org/opensearch/sql/asyncquery/DummyConsumerTest.java b/async-query/src/test/java/org/opensearch/sql/asyncquery/DummyConsumerTest.java deleted file mode 100644 index a08dbae736..0000000000 --- a/async-query/src/test/java/org/opensearch/sql/asyncquery/DummyConsumerTest.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.asyncquery; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.when; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -class DummyConsumerTest { - - @Mock Dummy dummy; - - @Test - public void test() { - DummyConsumer dummyConsumer = new DummyConsumer(dummy); - when(dummy.hello()).thenReturn("Hello from mock"); - - assertEquals("Hello from mock", dummyConsumer.hello()); - } -} diff --git a/spark/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java rename to async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java rename to async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceSpec.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryGetResultSpecTest.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryGetResultSpecTest.java similarity index 98% rename from spark/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryGetResultSpecTest.java rename to async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryGetResultSpecTest.java index 12fa8043ea..ae6d357842 100644 --- a/spark/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryGetResultSpecTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryGetResultSpecTest.java @@ -56,7 +56,10 @@ public class AsyncQueryGetResultSpecTest extends AsyncQueryExecutorServiceSpec { @Before public void doSetUp() { mockIndexState = - new MockFlintSparkJob(flintIndexStateModelService, mockIndex.latestId, MYS3_DATASOURCE); + new MockFlintSparkJob( + flintIndexStateModelService, + mockIndex.latestId, + MYS3_DATASOURCE); } @Test @@ -439,7 +442,8 @@ public JSONObject getResultWithQueryId(String queryId, String resultIndex) { }); this.createQueryResponse = queryService.createAsyncQuery( - new CreateAsyncQueryRequest(query, MYS3_DATASOURCE, LangType.SQL, null), + new CreateAsyncQueryRequest( + query, MYS3_DATASOURCE, LangType.SQL, null), asyncQueryRequestContext); } @@ -515,7 +519,10 @@ void emrJobWriteResultDoc(Map resultDoc) { /** Simulate EMR-S updates query_execution_request with state */ void emrJobUpdateStatementState(StatementState newState) { - StatementModel stmt = statementStorageService.getStatement(queryId, MYS3_DATASOURCE).get(); + StatementModel stmt = + statementStorageService + .getStatement(queryId, MYS3_DATASOURCE) + .get(); statementStorageService.updateStatementState(stmt, newState); } diff --git a/spark/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecAlterTest.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecAlterTest.java similarity index 99% rename from spark/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecAlterTest.java rename to async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecAlterTest.java index 801a24922f..230853a5eb 100644 --- a/spark/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecAlterTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecAlterTest.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.sql.spark.asyncquery; import com.amazonaws.services.emrserverless.model.CancelJobRunResult; diff --git a/spark/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecTest.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecTest.java rename to async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecVacuumTest.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecVacuumTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecVacuumTest.java rename to async-query/src/test/java/org/opensearch/sql/spark/asyncquery/IndexQuerySpecVacuumTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/asyncquery/OpenSearchAsyncQueryJobMetadataStorageServiceTest.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/OpenSearchAsyncQueryJobMetadataStorageServiceTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/asyncquery/OpenSearchAsyncQueryJobMetadataStorageServiceTest.java rename to async-query/src/test/java/org/opensearch/sql/spark/asyncquery/OpenSearchAsyncQueryJobMetadataStorageServiceTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/asyncquery/model/MockFlintIndex.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/model/MockFlintIndex.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/asyncquery/model/MockFlintIndex.java rename to async-query/src/test/java/org/opensearch/sql/spark/asyncquery/model/MockFlintIndex.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/asyncquery/model/MockFlintSparkJob.java b/async-query/src/test/java/org/opensearch/sql/spark/asyncquery/model/MockFlintSparkJob.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/asyncquery/model/MockFlintSparkJob.java rename to async-query/src/test/java/org/opensearch/sql/spark/asyncquery/model/MockFlintSparkJob.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/cluster/FlintStreamingJobHouseKeeperTaskTest.java b/async-query/src/test/java/org/opensearch/sql/spark/cluster/FlintStreamingJobHouseKeeperTaskTest.java similarity index 99% rename from spark/src/test/java/org/opensearch/sql/spark/cluster/FlintStreamingJobHouseKeeperTaskTest.java rename to async-query/src/test/java/org/opensearch/sql/spark/cluster/FlintStreamingJobHouseKeeperTaskTest.java index aa4684811f..89f3ac9871 100644 --- a/spark/src/test/java/org/opensearch/sql/spark/cluster/FlintStreamingJobHouseKeeperTaskTest.java +++ b/async-query/src/test/java/org/opensearch/sql/spark/cluster/FlintStreamingJobHouseKeeperTaskTest.java @@ -6,6 +6,7 @@ package org.opensearch.sql.spark.cluster; import static org.opensearch.sql.datasource.model.DataSourceStatus.DISABLED; +import static org.opensearch.sql.spark.asyncquery.AsyncQueryExecutorServiceSpec.MYGLUE_DATASOURCE; import com.amazonaws.services.emrserverless.model.GetJobRunResult; import com.amazonaws.services.emrserverless.model.JobRun; diff --git a/spark/src/test/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigSupplierImplTest.java b/async-query/src/test/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigSupplierImplTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigSupplierImplTest.java rename to async-query/src/test/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigSupplierImplTest.java diff --git a/async-query/src/test/java/org/opensearch/sql/spark/constants/TestConstants.java b/async-query/src/test/java/org/opensearch/sql/spark/constants/TestConstants.java new file mode 100644 index 0000000000..5b4ffbea2c --- /dev/null +++ b/async-query/src/test/java/org/opensearch/sql/spark/constants/TestConstants.java @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.constants; + +public class TestConstants { + public static final String TEST_DATASOURCE_NAME = "test_datasource_name"; + public static final String EMR_JOB_ID = "job-123xxx"; + public static final String EMRS_APPLICATION_ID = "app-xxxxx"; + public static final String EMRS_EXECUTION_ROLE = "execution_role"; + public static final String SPARK_SUBMIT_PARAMETERS = "--conf org.flint.sql.SQLJob"; + public static final String TEST_CLUSTER_NAME = "TEST_CLUSTER"; + public static final String MOCK_SESSION_ID = "s-0123456"; + public static final String US_WEST_REGION = "us-west-1"; +} diff --git a/spark/src/test/java/org/opensearch/sql/spark/execution/session/InteractiveSessionTest.java b/async-query/src/test/java/org/opensearch/sql/spark/execution/session/InteractiveSessionTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/execution/session/InteractiveSessionTest.java rename to async-query/src/test/java/org/opensearch/sql/spark/execution/session/InteractiveSessionTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/execution/session/SessionTestUtil.java b/async-query/src/test/java/org/opensearch/sql/spark/execution/session/SessionTestUtil.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/execution/session/SessionTestUtil.java rename to async-query/src/test/java/org/opensearch/sql/spark/execution/session/SessionTestUtil.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/execution/session/TestEMRServerlessClient.java b/async-query/src/test/java/org/opensearch/sql/spark/execution/session/TestEMRServerlessClient.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/execution/session/TestEMRServerlessClient.java rename to async-query/src/test/java/org/opensearch/sql/spark/execution/session/TestEMRServerlessClient.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/execution/statement/StatementTest.java b/async-query/src/test/java/org/opensearch/sql/spark/execution/statement/StatementTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/execution/statement/StatementTest.java rename to async-query/src/test/java/org/opensearch/sql/spark/execution/statement/StatementTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/execution/xcontent/AsyncQueryJobMetadataXContentSerializerTest.java b/async-query/src/test/java/org/opensearch/sql/spark/execution/xcontent/AsyncQueryJobMetadataXContentSerializerTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/execution/xcontent/AsyncQueryJobMetadataXContentSerializerTest.java rename to async-query/src/test/java/org/opensearch/sql/spark/execution/xcontent/AsyncQueryJobMetadataXContentSerializerTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/execution/xcontent/FlintIndexStateModelXContentSerializerTest.java b/async-query/src/test/java/org/opensearch/sql/spark/execution/xcontent/FlintIndexStateModelXContentSerializerTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/execution/xcontent/FlintIndexStateModelXContentSerializerTest.java rename to async-query/src/test/java/org/opensearch/sql/spark/execution/xcontent/FlintIndexStateModelXContentSerializerTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/execution/xcontent/IndexDMLResultXContentSerializerTest.java b/async-query/src/test/java/org/opensearch/sql/spark/execution/xcontent/IndexDMLResultXContentSerializerTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/execution/xcontent/IndexDMLResultXContentSerializerTest.java rename to async-query/src/test/java/org/opensearch/sql/spark/execution/xcontent/IndexDMLResultXContentSerializerTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/execution/xcontent/SessionModelXContentSerializerTest.java b/async-query/src/test/java/org/opensearch/sql/spark/execution/xcontent/SessionModelXContentSerializerTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/execution/xcontent/SessionModelXContentSerializerTest.java rename to async-query/src/test/java/org/opensearch/sql/spark/execution/xcontent/SessionModelXContentSerializerTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/execution/xcontent/StatementModelXContentSerializerTest.java b/async-query/src/test/java/org/opensearch/sql/spark/execution/xcontent/StatementModelXContentSerializerTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/execution/xcontent/StatementModelXContentSerializerTest.java rename to async-query/src/test/java/org/opensearch/sql/spark/execution/xcontent/StatementModelXContentSerializerTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/execution/xcontent/XContentSerializerTestUtil.java b/async-query/src/test/java/org/opensearch/sql/spark/execution/xcontent/XContentSerializerTestUtil.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/execution/xcontent/XContentSerializerTestUtil.java rename to async-query/src/test/java/org/opensearch/sql/spark/execution/xcontent/XContentSerializerTestUtil.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/flint/FlintIndexMetadataServiceImplTest.java b/async-query/src/test/java/org/opensearch/sql/spark/flint/FlintIndexMetadataServiceImplTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/flint/FlintIndexMetadataServiceImplTest.java rename to async-query/src/test/java/org/opensearch/sql/spark/flint/FlintIndexMetadataServiceImplTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexStateModelServiceTest.java b/async-query/src/test/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexStateModelServiceTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexStateModelServiceTest.java rename to async-query/src/test/java/org/opensearch/sql/spark/flint/OpenSearchFlintIndexStateModelServiceTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/leasemanager/DefaultLeaseManagerTest.java b/async-query/src/test/java/org/opensearch/sql/spark/leasemanager/DefaultLeaseManagerTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/leasemanager/DefaultLeaseManagerTest.java rename to async-query/src/test/java/org/opensearch/sql/spark/leasemanager/DefaultLeaseManagerTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReaderTest.java b/async-query/src/test/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReaderTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReaderTest.java rename to async-query/src/test/java/org/opensearch/sql/spark/response/OpenSearchJobExecutionResponseReaderTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/transport/TransportCancelAsyncQueryRequestActionTest.java b/async-query/src/test/java/org/opensearch/sql/spark/transport/TransportCancelAsyncQueryRequestActionTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/transport/TransportCancelAsyncQueryRequestActionTest.java rename to async-query/src/test/java/org/opensearch/sql/spark/transport/TransportCancelAsyncQueryRequestActionTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/transport/TransportCreateAsyncQueryRequestActionTest.java b/async-query/src/test/java/org/opensearch/sql/spark/transport/TransportCreateAsyncQueryRequestActionTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/transport/TransportCreateAsyncQueryRequestActionTest.java rename to async-query/src/test/java/org/opensearch/sql/spark/transport/TransportCreateAsyncQueryRequestActionTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/transport/TransportGetAsyncQueryResultActionTest.java b/async-query/src/test/java/org/opensearch/sql/spark/transport/TransportGetAsyncQueryResultActionTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/transport/TransportGetAsyncQueryResultActionTest.java rename to async-query/src/test/java/org/opensearch/sql/spark/transport/TransportGetAsyncQueryResultActionTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModuleTest.java b/async-query/src/test/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModuleTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModuleTest.java rename to async-query/src/test/java/org/opensearch/sql/spark/transport/config/AsyncExecutorServiceModuleTest.java diff --git a/spark/src/test/java/org/opensearch/sql/spark/transport/format/AsyncQueryResultResponseFormatterTest.java b/async-query/src/test/java/org/opensearch/sql/spark/transport/format/AsyncQueryResultResponseFormatterTest.java similarity index 100% rename from spark/src/test/java/org/opensearch/sql/spark/transport/format/AsyncQueryResultResponseFormatterTest.java rename to async-query/src/test/java/org/opensearch/sql/spark/transport/format/AsyncQueryResultResponseFormatterTest.java diff --git a/async-query/src/test/java/org/opensearch/sql/spark/utils/TestUtils.java b/async-query/src/test/java/org/opensearch/sql/spark/utils/TestUtils.java new file mode 100644 index 0000000000..bf3edeb70d --- /dev/null +++ b/async-query/src/test/java/org/opensearch/sql/spark/utils/TestUtils.java @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.utils; + +import com.google.common.base.Charsets; +import com.google.common.io.Resources; +import java.io.IOException; +import java.net.URL; +import java.util.Objects; +import lombok.SneakyThrows; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.client.Client; +import org.opensearch.common.xcontent.XContentType; + +public class TestUtils { + @SneakyThrows + public static String loadMappings(String path) { + URL url = Resources.getResource(path); + return Resources.toString(url, Charsets.UTF_8); + } + + public static void createIndexWithMappings( + Client client, String indexName, String metadataFileLocation) { + CreateIndexRequest request = new CreateIndexRequest(indexName); + request.mapping(loadMappings(metadataFileLocation), XContentType.JSON); + client.admin().indices().create(request).actionGet(); + } +} diff --git a/spark/src/test/resources/flint-index-mappings/0.1.1/flint_covering_index.json b/async-query/src/test/resources/flint-index-mappings/0.1.1/flint_covering_index.json similarity index 100% rename from spark/src/test/resources/flint-index-mappings/0.1.1/flint_covering_index.json rename to async-query/src/test/resources/flint-index-mappings/0.1.1/flint_covering_index.json diff --git a/spark/src/test/resources/flint-index-mappings/0.1.1/flint_mv.json b/async-query/src/test/resources/flint-index-mappings/0.1.1/flint_mv.json similarity index 100% rename from spark/src/test/resources/flint-index-mappings/0.1.1/flint_mv.json rename to async-query/src/test/resources/flint-index-mappings/0.1.1/flint_mv.json diff --git a/spark/src/test/resources/flint-index-mappings/0.1.1/flint_skipping_index.json b/async-query/src/test/resources/flint-index-mappings/0.1.1/flint_skipping_index.json similarity index 100% rename from spark/src/test/resources/flint-index-mappings/0.1.1/flint_skipping_index.json rename to async-query/src/test/resources/flint-index-mappings/0.1.1/flint_skipping_index.json diff --git a/spark/src/test/resources/flint-index-mappings/0.1.1/flint_special_character_index.json b/async-query/src/test/resources/flint-index-mappings/0.1.1/flint_special_character_index.json similarity index 100% rename from spark/src/test/resources/flint-index-mappings/0.1.1/flint_special_character_index.json rename to async-query/src/test/resources/flint-index-mappings/0.1.1/flint_special_character_index.json diff --git a/spark/src/test/resources/flint-index-mappings/flint_covering_index.json b/async-query/src/test/resources/flint-index-mappings/flint_covering_index.json similarity index 100% rename from spark/src/test/resources/flint-index-mappings/flint_covering_index.json rename to async-query/src/test/resources/flint-index-mappings/flint_covering_index.json diff --git a/spark/src/test/resources/flint-index-mappings/flint_mv.json b/async-query/src/test/resources/flint-index-mappings/flint_mv.json similarity index 100% rename from spark/src/test/resources/flint-index-mappings/flint_mv.json rename to async-query/src/test/resources/flint-index-mappings/flint_mv.json diff --git a/spark/src/test/resources/flint-index-mappings/flint_my_glue_mydb_http_logs_covering_corrupted_index_mapping.json b/async-query/src/test/resources/flint-index-mappings/flint_my_glue_mydb_http_logs_covering_corrupted_index_mapping.json similarity index 100% rename from spark/src/test/resources/flint-index-mappings/flint_my_glue_mydb_http_logs_covering_corrupted_index_mapping.json rename to async-query/src/test/resources/flint-index-mappings/flint_my_glue_mydb_http_logs_covering_corrupted_index_mapping.json diff --git a/spark/src/test/resources/flint-index-mappings/flint_my_glue_mydb_http_logs_covering_error_index_mapping.json b/async-query/src/test/resources/flint-index-mappings/flint_my_glue_mydb_http_logs_covering_error_index_mapping.json similarity index 100% rename from spark/src/test/resources/flint-index-mappings/flint_my_glue_mydb_http_logs_covering_error_index_mapping.json rename to async-query/src/test/resources/flint-index-mappings/flint_my_glue_mydb_http_logs_covering_error_index_mapping.json diff --git a/spark/src/test/resources/flint-index-mappings/flint_my_glue_mydb_http_logs_covering_index_mapping.json b/async-query/src/test/resources/flint-index-mappings/flint_my_glue_mydb_http_logs_covering_index_mapping.json similarity index 100% rename from spark/src/test/resources/flint-index-mappings/flint_my_glue_mydb_http_logs_covering_index_mapping.json rename to async-query/src/test/resources/flint-index-mappings/flint_my_glue_mydb_http_logs_covering_index_mapping.json diff --git a/spark/src/test/resources/flint-index-mappings/flint_my_glue_mydb_http_logs_skipping_index_mapping.json b/async-query/src/test/resources/flint-index-mappings/flint_my_glue_mydb_http_logs_skipping_index_mapping.json similarity index 100% rename from spark/src/test/resources/flint-index-mappings/flint_my_glue_mydb_http_logs_skipping_index_mapping.json rename to async-query/src/test/resources/flint-index-mappings/flint_my_glue_mydb_http_logs_skipping_index_mapping.json diff --git a/spark/src/test/resources/flint-index-mappings/flint_my_glue_mydb_mv_mapping.json b/async-query/src/test/resources/flint-index-mappings/flint_my_glue_mydb_mv_mapping.json similarity index 100% rename from spark/src/test/resources/flint-index-mappings/flint_my_glue_mydb_mv_mapping.json rename to async-query/src/test/resources/flint-index-mappings/flint_my_glue_mydb_mv_mapping.json diff --git a/spark/src/test/resources/flint-index-mappings/flint_mys3_default_http_logs_cv1_index.json b/async-query/src/test/resources/flint-index-mappings/flint_mys3_default_http_logs_cv1_index.json similarity index 100% rename from spark/src/test/resources/flint-index-mappings/flint_mys3_default_http_logs_cv1_index.json rename to async-query/src/test/resources/flint-index-mappings/flint_mys3_default_http_logs_cv1_index.json diff --git a/spark/src/test/resources/flint-index-mappings/flint_mys3_default_http_logs_skipping_index.json b/async-query/src/test/resources/flint-index-mappings/flint_mys3_default_http_logs_skipping_index.json similarity index 100% rename from spark/src/test/resources/flint-index-mappings/flint_mys3_default_http_logs_skipping_index.json rename to async-query/src/test/resources/flint-index-mappings/flint_mys3_default_http_logs_skipping_index.json diff --git a/spark/src/test/resources/flint-index-mappings/flint_skipping_index.json b/async-query/src/test/resources/flint-index-mappings/flint_skipping_index.json similarity index 100% rename from spark/src/test/resources/flint-index-mappings/flint_skipping_index.json rename to async-query/src/test/resources/flint-index-mappings/flint_skipping_index.json diff --git a/spark/src/test/resources/flint-index-mappings/flint_special_character_index.json b/async-query/src/test/resources/flint-index-mappings/flint_special_character_index.json similarity index 100% rename from spark/src/test/resources/flint-index-mappings/flint_special_character_index.json rename to async-query/src/test/resources/flint-index-mappings/flint_special_character_index.json diff --git a/spark/src/test/resources/flint-index-mappings/npe_mapping.json b/async-query/src/test/resources/flint-index-mappings/npe_mapping.json similarity index 100% rename from spark/src/test/resources/flint-index-mappings/npe_mapping.json rename to async-query/src/test/resources/flint-index-mappings/npe_mapping.json diff --git a/async-query/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/async-query/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 0000000000..ca6ee9cea8 --- /dev/null +++ b/async-query/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline \ No newline at end of file diff --git a/spark/src/test/resources/query_execution_result_mapping.json b/async-query/src/test/resources/query_execution_result_mapping.json similarity index 100% rename from spark/src/test/resources/query_execution_result_mapping.json rename to async-query/src/test/resources/query_execution_result_mapping.json diff --git a/plugin/build.gradle b/plugin/build.gradle index 49cb52721d..ed6fe53247 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -149,6 +149,7 @@ dependencies { api project(':prometheus') api project(':datasources') api project(':spark') + api project(':async-query') testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.14.9' testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' diff --git a/spark/build.gradle b/spark/build.gradle index c221c4e36c..d9d5c96413 100644 --- a/spark/build.gradle +++ b/spark/build.gradle @@ -7,45 +7,15 @@ plugins { id 'java-library' id "io.freefair.lombok" id 'jacoco' - id 'antlr' } repositories { mavenCentral() } -tasks.register('downloadG4Files', Exec) { - description = 'Download remote .g4 files from GitHub' - - executable 'curl' - - args '-o', 'src/main/antlr/FlintSparkSqlExtensions.g4', 'https://raw.githubusercontent.com/opensearch-project/opensearch-spark/main/flint-spark-integration/src/main/antlr4/FlintSparkSqlExtensions.g4' - args '-o', 'src/main/antlr/SparkSqlBase.g4', 'https://raw.githubusercontent.com/opensearch-project/opensearch-spark/main/flint-spark-integration/src/main/antlr4/SparkSqlBase.g4' - args '-o', 'src/main/antlr/SqlBaseParser.g4', 'https://raw.githubusercontent.com/apache/spark/master/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4' - args '-o', 'src/main/antlr/SqlBaseLexer.g4', 'https://raw.githubusercontent.com/apache/spark/master/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4' -} - -generateGrammarSource { - arguments += ['-visitor', '-package', 'org.opensearch.sql.spark.antlr.parser'] - source = sourceSets.main.antlr - outputDirectory = file("build/generated-src/antlr/main/org/opensearch/sql/spark/antlr/parser") -} -configurations { - compile { - extendsFrom = extendsFrom.findAll { it != configurations.antlr } - } -} - -// Make sure the downloadG4File task runs before the generateGrammarSource task -generateGrammarSource.dependsOn downloadG4Files - dependencies { - antlr "org.antlr:antlr4:4.7.1" - api project(':core') - implementation project(':protocol') implementation project(':datasources') - implementation project(':legacy') implementation group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" implementation group: 'org.json', name: 'json', version: '20231013' @@ -59,20 +29,12 @@ dependencies { testImplementation group: 'org.mockito', name: 'mockito-core', version: '5.7.0' testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: '5.7.0' - testCompileOnly('junit:junit:4.13.1') { - exclude group: 'org.hamcrest', module: 'hamcrest-core' - } - testRuntimeOnly("org.junit.vintage:junit-vintage-engine") { - exclude group: 'org.hamcrest', module: 'hamcrest-core' - } testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine") { exclude group: 'org.hamcrest', module: 'hamcrest-core' } testRuntimeOnly("org.junit.platform:junit-platform-launcher") { because 'allows tests to run from IDEs that bundle older version of launcher' } - testImplementation("org.opensearch.test:framework:${opensearch_version}") - testImplementation project(':opensearch') } test { @@ -84,54 +46,28 @@ test { exceptionFormat "full" } } -task junit4(type: Test) { - useJUnitPlatform { - includeEngines("junit-vintage") - } - systemProperty 'tests.security.manager', 'false' - testLogging { - events "failed" - exceptionFormat "full" - } -} jacocoTestReport { - dependsOn test, junit4 - executionData test, junit4 + dependsOn test + executionData test reports { html.required = true xml.required = true } afterEvaluate { classDirectories.setFrom(files(classDirectories.files.collect { - fileTree(dir: it, exclude: ['**/antlr/parser/**']) })) } } jacocoTestCoverageVerification { - dependsOn test, junit4 - executionData test, junit4 + dependsOn test + executionData test violationRules { rule { element = 'CLASS' excludes = [ 'org.opensearch.sql.spark.data.constants.*', - 'org.opensearch.sql.spark.rest.*', - 'org.opensearch.sql.spark.transport.model.*', - 'org.opensearch.sql.spark.asyncquery.model.*', - 'org.opensearch.sql.spark.asyncquery.exceptions.*', - 'org.opensearch.sql.spark.dispatcher.model.*', - 'org.opensearch.sql.spark.flint.FlintIndexType', - // ignore because XContext IOException - 'org.opensearch.sql.spark.execution.statestore.StateStore', - 'org.opensearch.sql.spark.execution.session.SessionModel', - 'org.opensearch.sql.spark.execution.statement.StatementModel', - 'org.opensearch.sql.spark.flint.FlintIndexStateModel', - // TODO: add tests for purging flint indices - 'org.opensearch.sql.spark.cluster.ClusterManagerEventListener*', - 'org.opensearch.sql.spark.cluster.FlintIndexRetention', - 'org.opensearch.sql.spark.cluster.IndexCleanup' ] limit { counter = 'LINE' @@ -145,7 +81,6 @@ jacocoTestCoverageVerification { } afterEvaluate { classDirectories.setFrom(files(classDirectories.files.collect { - fileTree(dir: it, exclude: ['**/antlr/parser/**']) })) } } diff --git a/spark/src/main/java/org/opensearch/sql/spark/client/EmrClientImpl.java b/spark/src/main/java/org/opensearch/sql/spark/client/EmrClientImpl.java index 87f35bbc1e..3ef911c8d8 100644 --- a/spark/src/main/java/org/opensearch/sql/spark/client/EmrClientImpl.java +++ b/spark/src/main/java/org/opensearch/sql/spark/client/EmrClientImpl.java @@ -6,7 +6,6 @@ package org.opensearch.sql.spark.client; import static org.opensearch.sql.datasource.model.DataSourceMetadata.DEFAULT_RESULT_INDEX; -import static org.opensearch.sql.spark.data.constants.SparkConstants.SPARK_SQL_APPLICATION_JAR; import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduce; import com.amazonaws.services.elasticmapreduce.model.ActionOnFailure; @@ -26,6 +25,10 @@ import org.opensearch.sql.spark.response.SparkResponse; public class EmrClientImpl implements SparkClient { + // EMR-S will download JAR to local maven + public static final String SPARK_SQL_APPLICATION_JAR = + "file:///home/hadoop/.ivy2/jars/org.opensearch_opensearch-spark-sql-application_2.12-0.3.0-SNAPSHOT.jar"; + private final AmazonElasticMapReduce emr; private final String emrCluster; private final FlintHelper flint; diff --git a/spark/src/main/java/org/opensearch/sql/spark/helper/FlintHelper.java b/spark/src/main/java/org/opensearch/sql/spark/helper/FlintHelper.java index 10d880187f..206ff4aed4 100644 --- a/spark/src/main/java/org/opensearch/sql/spark/helper/FlintHelper.java +++ b/spark/src/main/java/org/opensearch/sql/spark/helper/FlintHelper.java @@ -5,16 +5,18 @@ package org.opensearch.sql.spark.helper; -import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_DEFAULT_AUTH; -import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_DEFAULT_HOST; -import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_DEFAULT_PORT; -import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_DEFAULT_REGION; -import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_DEFAULT_SCHEME; -import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_INTEGRATION_JAR; - import lombok.Getter; public class FlintHelper { + // TODO should be replaced with mvn jar. + public static final String FLINT_INTEGRATION_JAR = + "s3://spark-datasource/flint-spark-integration-assembly-0.3.0-SNAPSHOT.jar"; + public static final String FLINT_DEFAULT_HOST = "localhost"; + public static final String FLINT_DEFAULT_PORT = "9200"; + public static final String FLINT_DEFAULT_SCHEME = "http"; + public static final String FLINT_DEFAULT_AUTH = "noauth"; + public static final String FLINT_DEFAULT_REGION = "us-west-2"; + @Getter private final String flintIntegrationJar; @Getter private final String flintHost; @Getter private final String flintPort; diff --git a/spark/src/main/java/org/opensearch/sql/spark/storage/SparkStorageFactory.java b/spark/src/main/java/org/opensearch/sql/spark/storage/SparkStorageFactory.java index 467bacbaea..4495eb0fac 100644 --- a/spark/src/main/java/org/opensearch/sql/spark/storage/SparkStorageFactory.java +++ b/spark/src/main/java/org/opensearch/sql/spark/storage/SparkStorageFactory.java @@ -5,9 +5,6 @@ package org.opensearch.sql.spark.storage; -import static org.opensearch.sql.spark.data.constants.SparkConstants.EMR; -import static org.opensearch.sql.spark.data.constants.SparkConstants.STEP_ID_FIELD; - import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.services.elasticmapreduce.AmazonElasticMapReduce; @@ -36,6 +33,8 @@ public class SparkStorageFactory implements DataSourceFactory { private final Client client; private final Settings settings; + public static final String EMR = "emr"; + public static final String STEP_ID_FIELD = "stepId.keyword"; // Spark datasource configuration properties public static final String CONNECTOR_TYPE = "spark.connector"; public static final String SPARK_SQL_APPLICATION = "spark.sql.application"; @@ -44,7 +43,6 @@ public class SparkStorageFactory implements DataSourceFactory { public static final String EMR_CLUSTER = "emr.cluster"; public static final String EMR_AUTH_TYPE = "emr.auth.type"; public static final String EMR_REGION = "emr.auth.region"; - public static final String EMR_ROLE_ARN = "emr.auth.role_arn"; public static final String EMR_ACCESS_KEY = "emr.auth.access_key"; public static final String EMR_SECRET_KEY = "emr.auth.secret_key"; diff --git a/spark/src/test/java/org/opensearch/sql/spark/data/type/SparkDataTypeTest.java b/spark/src/test/java/org/opensearch/sql/spark/data/type/SparkDataTypeTest.java new file mode 100644 index 0000000000..ff6cee2a5e --- /dev/null +++ b/spark/src/test/java/org/opensearch/sql/spark/data/type/SparkDataTypeTest.java @@ -0,0 +1,19 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.spark.data.type; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class SparkDataTypeTest { + @Test + public void testTypeName() { + SparkDataType sparkDataType = new SparkDataType("TYPE_NAME"); + + assertEquals("TYPE_NAME", sparkDataType.typeName()); + } +} diff --git a/spark/src/test/java/org/opensearch/sql/spark/utils/TestUtils.java b/spark/src/test/java/org/opensearch/sql/spark/utils/TestUtils.java index 4cab6afa9c..4336b13aa9 100644 --- a/spark/src/test/java/org/opensearch/sql/spark/utils/TestUtils.java +++ b/spark/src/test/java/org/opensearch/sql/spark/utils/TestUtils.java @@ -5,41 +5,13 @@ package org.opensearch.sql.spark.utils; -import com.google.common.base.Charsets; -import com.google.common.io.Resources; import java.io.IOException; -import java.net.URL; import java.util.Objects; -import lombok.SneakyThrows; -import org.opensearch.action.admin.indices.create.CreateIndexRequest; -import org.opensearch.client.Client; -import org.opensearch.common.xcontent.XContentType; public class TestUtils { - - /** - * Get Json document from the files in resources folder. - * - * @param filename filename. - * @return String. - * @throws IOException IOException. - */ public static String getJson(String filename) throws IOException { ClassLoader classLoader = TestUtils.class.getClassLoader(); return new String( Objects.requireNonNull(classLoader.getResourceAsStream(filename)).readAllBytes()); } - - @SneakyThrows - public static String loadMappings(String path) { - URL url = Resources.getResource(path); - return Resources.toString(url, Charsets.UTF_8); - } - - public static void createIndexWithMappings( - Client client, String indexName, String metadataFileLocation) { - CreateIndexRequest request = new CreateIndexRequest(indexName); - request.mapping(loadMappings(metadataFileLocation), XContentType.JSON); - client.admin().indices().create(request).actionGet(); - } }