Enable session by default (#2373) (#2375)

* enable session by default * update doc --------- (cherry picked from commit ff38081) Signed-off-by: Peng Huo <[email protected]> Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
opensearch-project · Oct 26, 2023 · 7d95e4c · 7d95e4c
1 parent 5ab7858
commit 7d95e4c
Show file tree

Hide file tree

Showing 4 changed files with 48 additions and 9 deletions.
diff --git a/docs/user/admin/settings.rst b/docs/user/admin/settings.rst
@@ -317,9 +317,9 @@ plugins.query.executionengine.spark.session.enabled
 Description
 -----------
 
-By default, execution engine is executed in job mode. You can enable session mode by this setting.
+By default, execution engine is executed in session mode. You can disable session mode by this setting.
 
-1. The default value is false.
+1. The default value is true.
 2. This setting is node scope.
 3. This setting can be updated dynamically.
 
@@ -328,7 +328,7 @@ You can update the setting with a new value like this.
 SQL query::
 
     sh$ curl -sS -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_query/settings \
-    ... -d '{"transient":{"plugins.query.executionengine.spark.session.enabled":"true"}}'
+    ... -d '{"transient":{"plugins.query.executionengine.spark.session.enabled":"false"}}'
     {
       "acknowledged": true,
       "persistent": {},
@@ -338,7 +338,7 @@ SQL query::
             "executionengine": {
               "spark": {
                 "session": {
-                  "enabled": "true"
+                  "enabled": "false"
                 }
               }
             }

diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java
@@ -138,7 +138,7 @@ public class OpenSearchSettings extends Settings {
   public static final Setting<?> SPARK_EXECUTION_SESSION_ENABLED_SETTING =
       Setting.boolSetting(
           Key.SPARK_EXECUTION_SESSION_ENABLED.getKeyValue(),
-          false,
+          true,
           Setting.Property.NodeScope,
           Setting.Property.Dynamic);
 

diff --git a/spark/src/main/java/org/opensearch/sql/spark/execution/statestore/StateStore.java b/spark/src/main/java/org/opensearch/sql/spark/execution/statestore/StateStore.java
@@ -60,7 +60,9 @@ public class StateStore {
   public static String SETTINGS_FILE_NAME = "query_execution_request_settings.yml";
   public static String MAPPING_FILE_NAME = "query_execution_request_mapping.yml";
   public static Function<String, String> DATASOURCE_TO_REQUEST_INDEX =
-      datasourceName -> String.format("%s_%s", SPARK_REQUEST_BUFFER_INDEX_NAME, datasourceName);
+      datasourceName ->
+          String.format(
+              "%s_%s", SPARK_REQUEST_BUFFER_INDEX_NAME, datasourceName.toLowerCase(Locale.ROOT));
 
   private static final Logger LOG = LogManager.getLogger();
 

diff --git a/.../test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java b/.../test/java/org/opensearch/sql/spark/asyncquery/AsyncQueryExecutorServiceImplSpecTest.java
@@ -37,6 +37,7 @@
 import lombok.Getter;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.opensearch.action.search.SearchRequest;
 import org.opensearch.action.search.SearchResponse;
@@ -212,9 +213,6 @@ public void withSessionCreateAsyncQueryThenGetResultThenCancel() {
     AsyncQueryExecutorService asyncQueryExecutorService =
         createAsyncQueryExecutorService(emrsClient);
 
-    // enable session
-    enableSession(true);
-
     // 1. create async query.
     CreateAsyncQueryResponse response =
         asyncQueryExecutorService.createAsyncQuery(
@@ -315,6 +313,9 @@ public void interactiveQueryNoTimeout() {
     assertEquals(0L, (long) emrsClient.getJobRequest().executionTimeout());
   }
 
+  @Ignore(
+      "flaky test, java.lang.IllegalArgumentException: Right now only AES/GCM/NoPadding is"
+          + " supported")
   @Test
   public void datasourceWithBasicAuth() {
     Map<String, String> properties = new HashMap<>();
@@ -480,6 +481,42 @@ public void submitQueryInInvalidSessionWillCreateNewSession() {
     assertNotEquals(invalidSessionId.getSessionId(), asyncQuery.getSessionId());
   }
 
+  @Test
+  public void datasourceNameIncludeUppercase() {
+    dataSourceService.createDataSource(
+        new DataSourceMetadata(
+            "TESTS3",
+            DataSourceType.S3GLUE,
+            ImmutableList.of(),
+            ImmutableMap.of(
+                "glue.auth.type",
+                "iam_role",
+                "glue.auth.role_arn",
+                "arn:aws:iam::924196221507:role/FlintOpensearchServiceRole",
+                "glue.indexstore.opensearch.uri",
+                "http://localhost:9200",
+                "glue.indexstore.opensearch.auth",
+                "noauth"),
+            null));
+
+    LocalEMRSClient emrsClient = new LocalEMRSClient();
+    AsyncQueryExecutorService asyncQueryExecutorService =
+        createAsyncQueryExecutorService(emrsClient);
+
+    // enable session
+    enableSession(true);
+
+    CreateAsyncQueryResponse response =
+        asyncQueryExecutorService.createAsyncQuery(
+            new CreateAsyncQueryRequest("select 1", "TESTS3", LangType.SQL, null));
+    String params = emrsClient.getJobRequest().getSparkSubmitParams();
+
+    assertNotNull(response.getSessionId());
+    assertTrue(
+        params.contains(
+            "--conf spark.sql.catalog.TESTS3=org.opensearch.sql.FlintDelegatingSessionCatalog"));
+  }
+
   private DataSourceServiceImpl createDataSourceService() {
     String masterKey = "a57d991d9b573f75b9bba1df";
     DataSourceMetadataStorage dataSourceMetadataStorage =