Refactor Flint Auth

Signed-off-by: Vamsi Manohar <[email protected]>

datasources/src/main/java/org/opensearch/sql/datasources/auth/AuthenticationType.java

@@ -12,6 +12,7 @@
 import java.util.Map;
 
 public enum AuthenticationType {
+  NOAUTH("noauth"),
   BASICAUTH("basicauth"),
   AWSSIGV4AUTH("awssigv4");
 

datasources/src/main/java/org/opensearch/sql/datasources/glue/GlueDataSourceFactory.java

@@ -9,6 +9,7 @@
 import org.opensearch.sql.datasource.model.DataSource;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;
 import org.opensearch.sql.datasource.model.DataSourceType;
+import org.opensearch.sql.datasources.auth.AuthenticationType;
 import org.opensearch.sql.datasources.utils.DatasourceValidationUtils;
 import org.opensearch.sql.storage.DataSourceFactory;
 
@@ -20,9 +21,14 @@ public class GlueDataSourceFactory implements DataSourceFactory {
   // Glue configuration properties
   public static final String GLUE_AUTH_TYPE = "glue.auth.type";
   public static final String GLUE_ROLE_ARN = "glue.auth.role_arn";
-  public static final String FLINT_URI = "glue.indexstore.opensearch.uri";
-  public static final String FLINT_AUTH = "glue.indexstore.opensearch.auth";
-  public static final String FLINT_REGION = "glue.indexstore.opensearch.region";
+  public static final String GLUE_INDEX_STORE_OPENSEARCH_URI = "glue.indexstore.opensearch.uri";
+  public static final String GLUE_INDEX_STORE_OPENSEARCH_AUTH = "glue.indexstore.opensearch.auth";
+  public static final String GLUE_INDEX_STORE_OPENSEARCH_AUTH_USERNAME =
+      "glue.indexstore.opensearch.auth.username";
+  public static final String GLUE_INDEX_STORE_OPENSEARCH_AUTH_PASSWORD =
+      "glue.indexstore.opensearch.auth.password";
+  public static final String GLUE_INDEX_STORE_OPENSEARCH_REGION =
+      "glue.indexstore.opensearch.region";
 
   @Override
   public DataSourceType getDataSourceType() {
@@ -46,11 +52,28 @@ public DataSource createDataSource(DataSourceMetadata metadata) {
 
   private void validateGlueDataSourceConfiguration(Map<String, String> dataSourceMetadataConfig)
       throws URISyntaxException, UnknownHostException {
+
     DatasourceValidationUtils.validateLengthAndRequiredFields(
         dataSourceMetadataConfig,
-        Set.of(GLUE_AUTH_TYPE, GLUE_ROLE_ARN, FLINT_URI, FLINT_REGION, FLINT_AUTH));
+        Set.of(
+            GLUE_AUTH_TYPE,
+            GLUE_ROLE_ARN,
+            GLUE_INDEX_STORE_OPENSEARCH_URI,
+            GLUE_INDEX_STORE_OPENSEARCH_AUTH));
+    AuthenticationType authenticationType =
+        AuthenticationType.get(dataSourceMetadataConfig.get(GLUE_INDEX_STORE_OPENSEARCH_AUTH));
+    if (AuthenticationType.BASICAUTH.equals(authenticationType)) {
+      DatasourceValidationUtils.validateLengthAndRequiredFields(
+          dataSourceMetadataConfig,
+          Set.of(
+              GLUE_INDEX_STORE_OPENSEARCH_AUTH_USERNAME,
+              GLUE_INDEX_STORE_OPENSEARCH_AUTH_PASSWORD));
+    } else if (AuthenticationType.AWSSIGV4AUTH.equals(authenticationType)) {
+      DatasourceValidationUtils.validateLengthAndRequiredFields(
+          dataSourceMetadataConfig, Set.of(GLUE_INDEX_STORE_OPENSEARCH_REGION));
+    }
     DatasourceValidationUtils.validateHost(
-        dataSourceMetadataConfig.get(FLINT_URI),
+        dataSourceMetadataConfig.get(GLUE_INDEX_STORE_OPENSEARCH_URI),
         pluginSettings.getSettingValue(Settings.Key.DATASOURCES_URI_HOSTS_DENY_LIST));
   }
 }

datasources/src/main/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorage.java

@@ -43,7 +43,6 @@
 import org.opensearch.search.SearchHit;
 import org.opensearch.search.builder.SearchSourceBuilder;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;
-import org.opensearch.sql.datasources.auth.AuthenticationType;
 import org.opensearch.sql.datasources.encryptor.Encryptor;
 import org.opensearch.sql.datasources.exceptions.DataSourceNotFoundException;
 import org.opensearch.sql.datasources.service.DataSourceMetadataStorage;
@@ -252,26 +251,13 @@ private List<DataSourceMetadata> searchInDataSourcesIndex(QueryBuilder query) {
     }
   }
 
-  @SuppressWarnings("missingswitchdefault")
+  // Encrypt and Decrypt irrespective of auth type.If properties name ends in username, password,
+  // secret_key and access_key.
   private DataSourceMetadata encryptDecryptAuthenticationData(
       DataSourceMetadata dataSourceMetadata, Boolean isEncryption) {
     Map<String, String> propertiesMap = dataSourceMetadata.getProperties();
-    Optional<AuthenticationType> authTypeOptional =
-        propertiesMap.keySet().stream()
-            .filter(s -> s.endsWith("auth.type"))
-            .findFirst()
-            .map(propertiesMap::get)
-            .map(AuthenticationType::get);
-    if (authTypeOptional.isPresent()) {
-      switch (authTypeOptional.get()) {
-        case BASICAUTH:
-          handleBasicAuthPropertiesEncryptionDecryption(propertiesMap, isEncryption);
-          break;
-        case AWSSIGV4AUTH:
-          handleSigV4PropertiesEncryptionDecryption(propertiesMap, isEncryption);
-          break;
-      }
-    }
+    handleBasicAuthPropertiesEncryptionDecryption(propertiesMap, isEncryption);
+    handleSigV4PropertiesEncryptionDecryption(propertiesMap, isEncryption);
     return dataSourceMetadata;
   }
 

datasources/src/test/java/org/opensearch/sql/datasources/glue/GlueDataSourceFactoryTest.java

@@ -40,7 +40,7 @@ void testCreateGLueDatSource() {
     properties.put("glue.auth.type", "iam_role");
     properties.put("glue.auth.role_arn", "role_arn");
     properties.put("glue.indexstore.opensearch.uri", "http://localhost:9200");
-    properties.put("glue.indexstore.opensearch.auth", "false");
+    properties.put("glue.indexstore.opensearch.auth", "noauth");
     properties.put("glue.indexstore.opensearch.region", "us-west-2");
 
     metadata.setName("my_glue");
@@ -59,6 +59,94 @@ void testCreateGLueDatSource() {
         "Glue storage engine is not supported.", unsupportedOperationException.getMessage());
   }
 
+  @Test
+  @SneakyThrows
+  void testCreateGLueDatSourceWithBasicAuthForIndexStore() {
+    when(settings.getSettingValue(Settings.Key.DATASOURCES_URI_HOSTS_DENY_LIST))
+        .thenReturn(Collections.emptyList());
+    GlueDataSourceFactory glueDatasourceFactory = new GlueDataSourceFactory(settings);
+
+    DataSourceMetadata metadata = new DataSourceMetadata();
+    HashMap<String, String> properties = new HashMap<>();
+    properties.put("glue.auth.type", "iam_role");
+    properties.put("glue.auth.role_arn", "role_arn");
+    properties.put("glue.indexstore.opensearch.uri", "http://localhost:9200");
+    properties.put("glue.indexstore.opensearch.auth", "basicauth");
+    properties.put("glue.indexstore.opensearch.auth.username", "username");
+    properties.put("glue.indexstore.opensearch.auth.password", "password");
+    properties.put("glue.indexstore.opensearch.region", "us-west-2");
+
+    metadata.setName("my_glue");
+    metadata.setConnector(DataSourceType.S3GLUE);
+    metadata.setProperties(properties);
+    DataSource dataSource = glueDatasourceFactory.createDataSource(metadata);
+    Assertions.assertEquals(DataSourceType.S3GLUE, dataSource.getConnectorType());
+    UnsupportedOperationException unsupportedOperationException =
+        Assertions.assertThrows(
+            UnsupportedOperationException.class,
+            () ->
+                dataSource
+                    .getStorageEngine()
+                    .getTable(new DataSourceSchemaName("my_glue", "default"), "alb_logs"));
+    Assertions.assertEquals(
+        "Glue storage engine is not supported.", unsupportedOperationException.getMessage());
+  }
+
+  @Test
+  @SneakyThrows
+  void testCreateGLueDatSourceWithAwsSigV4AuthForIndexStore() {
+    when(settings.getSettingValue(Settings.Key.DATASOURCES_URI_HOSTS_DENY_LIST))
+        .thenReturn(Collections.emptyList());
+    GlueDataSourceFactory glueDatasourceFactory = new GlueDataSourceFactory(settings);
+
+    DataSourceMetadata metadata = new DataSourceMetadata();
+    HashMap<String, String> properties = new HashMap<>();
+    properties.put("glue.auth.type", "iam_role");
+    properties.put("glue.auth.role_arn", "role_arn");
+    properties.put("glue.indexstore.opensearch.uri", "http://localhost:9200");
+    properties.put("glue.indexstore.opensearch.auth", "awssigv4");
+    properties.put("glue.indexstore.opensearch.region", "us-west-2");
+
+    metadata.setName("my_glue");
+    metadata.setConnector(DataSourceType.S3GLUE);
+    metadata.setProperties(properties);
+    DataSource dataSource = glueDatasourceFactory.createDataSource(metadata);
+    Assertions.assertEquals(DataSourceType.S3GLUE, dataSource.getConnectorType());
+    UnsupportedOperationException unsupportedOperationException =
+        Assertions.assertThrows(
+            UnsupportedOperationException.class,
+            () ->
+                dataSource
+                    .getStorageEngine()
+                    .getTable(new DataSourceSchemaName("my_glue", "default"), "alb_logs"));
+    Assertions.assertEquals(
+        "Glue storage engine is not supported.", unsupportedOperationException.getMessage());
+  }
+
+  @Test
+  @SneakyThrows
+  void testCreateGLueDatSourceWithBasicAuthForIndexStoreAndMissingFields() {
+    GlueDataSourceFactory glueDatasourceFactory = new GlueDataSourceFactory(settings);
+
+    DataSourceMetadata metadata = new DataSourceMetadata();
+    HashMap<String, String> properties = new HashMap<>();
+    properties.put("glue.auth.type", "iam_role");
+    properties.put("glue.auth.role_arn", "role_arn");
+    properties.put("glue.indexstore.opensearch.uri", "http://localhost:9200");
+    properties.put("glue.indexstore.opensearch.auth", "basicauth");
+
+    metadata.setName("my_glue");
+    metadata.setConnector(DataSourceType.S3GLUE);
+    metadata.setProperties(properties);
+    IllegalArgumentException illegalArgumentException =
+        Assertions.assertThrows(
+            IllegalArgumentException.class, () -> glueDatasourceFactory.createDataSource(metadata));
+    Assertions.assertEquals(
+        "Missing [glue.indexstore.opensearch.auth.password,"
+            + " glue.indexstore.opensearch.auth.username] fields in the connector properties.",
+        illegalArgumentException.getMessage());
+  }
+
   @Test
   @SneakyThrows
   void testCreateGLueDatSourceWithInvalidFlintHost() {
@@ -71,7 +159,7 @@ void testCreateGLueDatSourceWithInvalidFlintHost() {
     properties.put("glue.auth.type", "iam_role");
     properties.put("glue.auth.role_arn", "role_arn");
     properties.put("glue.indexstore.opensearch.uri", "http://localhost:9200");
-    properties.put("glue.indexstore.opensearch.auth", "false");
+    properties.put("glue.indexstore.opensearch.auth", "noauth");
     properties.put("glue.indexstore.opensearch.region", "us-west-2");
 
     metadata.setName("my_glue");
@@ -100,7 +188,7 @@ void testCreateGLueDatSourceWithInvalidFlintHostSyntax() {
     properties.put(
         "glue.indexstore.opensearch.uri",
         "http://dummyprometheus.com:9090? paramt::localhost:9200");
-    properties.put("glue.indexstore.opensearch.auth", "false");
+    properties.put("glue.indexstore.opensearch.auth", "noauth");
     properties.put("glue.indexstore.opensearch.region", "us-west-2");
 
     metadata.setName("my_glue");

docs/user/ppl/admin/connectors/s3glue_connector.rst

@@ -39,9 +39,10 @@ Glue Connector Properties.
     * This parameters provides the Opensearch domain host information for glue connector. This opensearch instance is used for writing index data back and also
     * ``glue.indexstore.opensearch.uri`` [Required]
     * ``glue.indexstore.opensearch.auth`` [Required]
-        * Default value for auth is ``false``.
-    * ``glue.indexstore.opensearch.region`` [Required]
-        * Default value for auth is ``us-west-2``.
+        * Accepted values include ["noauth", "basicauth", "awssigv4"]
+        * Basic Auth required ``glue.indexstore.opensearch.auth.username`` and ``glue.indexstore.opensearch.auth.password``
+        * AWSSigV4 Auth requires ``glue.indexstore.opensearch.auth.region``  and ``glue.auth.role_arn``
+    * ``glue.indexstore.opensearch.region`` [Required for awssigv4 auth]
 
 Sample Glue dataSource configuration
 ========================================
@@ -55,11 +56,23 @@ Glue datasource configuration::
                 "glue.auth.type": "iam_role",
                 "glue.auth.role_arn": "role_arn",
                 "glue.indexstore.opensearch.uri": "http://localhost:9200",
-                "glue.indexstore.opensearch.auth" :"false",
-                "glue.indexstore.opensearch.region": "us-west-2"
+                "glue.indexstore.opensearch.auth" :"basicauth",
+                "glue.indexstore.opensearch.auth.username" :"username"
+                "glue.indexstore.opensearch.auth.password" :"password"
         }
     }]
 
+    [{
+        "name" : "my_glue",
+        "connector": "s3glue",
+        "properties" : {
+                "glue.auth.type": "iam_role",
+                "glue.auth.role_arn": "role_arn",
+                "glue.indexstore.opensearch.uri": "http://adsasdf.amazonopensearch.com:9200",
+                "glue.indexstore.opensearch.auth" :"awssigv4",
+                "glue.indexstore.opensearch.auth.region" :"awssigv4",
+        }
+    }]
 
 Sample s3Glue datasource queries
 ================================

prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactory.java

@@ -43,7 +43,6 @@ public class PrometheusStorageFactory implements DataSourceFactory {
   public static final String REGION = "prometheus.auth.region";
   public static final String ACCESS_KEY = "prometheus.auth.access_key";
   public static final String SECRET_KEY = "prometheus.auth.secret_key";
-  private static final Integer MAX_LENGTH_FOR_CONFIG_PROPERTY = 1000;
 
   private final Settings settings;
 

spark/src/main/java/org/opensearch/sql/spark/asyncquery/model/S3GlueSparkSubmitParameters.java

@@ -17,10 +17,8 @@
 import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_DEFAULT_AUTH;
 import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_DEFAULT_HOST;
 import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_DEFAULT_PORT;
-import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_DEFAULT_REGION;
 import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_DEFAULT_SCHEME;
 import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_INDEX_STORE_AUTH_KEY;
-import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_INDEX_STORE_AWSREGION_KEY;
 import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_INDEX_STORE_HOST_KEY;
 import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_INDEX_STORE_PORT_KEY;
 import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_INDEX_STORE_SCHEME_KEY;
@@ -69,7 +67,6 @@ public S3GlueSparkSubmitParameters() {
     this.config.put(FLINT_INDEX_STORE_PORT_KEY, FLINT_DEFAULT_PORT);
     this.config.put(FLINT_INDEX_STORE_SCHEME_KEY, FLINT_DEFAULT_SCHEME);
     this.config.put(FLINT_INDEX_STORE_AUTH_KEY, FLINT_DEFAULT_AUTH);
-    this.config.put(FLINT_INDEX_STORE_AWSREGION_KEY, FLINT_DEFAULT_REGION);
     this.config.put(FLINT_CREDENTIALS_PROVIDER_KEY, EMR_ASSUME_ROLE_CREDENTIALS_PROVIDER);
     this.config.put(SPARK_SQL_EXTENSIONS_KEY, FLINT_SQL_EXTENSION);
     this.config.put(HIVE_METASTORE_CLASS_KEY, GLUE_HIVE_CATALOG_FACTORY_CLASS);

spark/src/main/java/org/opensearch/sql/spark/data/constants/SparkConstants.java

@@ -24,7 +24,7 @@ public class SparkConstants {
   public static final String FLINT_DEFAULT_HOST = "localhost";
   public static final String FLINT_DEFAULT_PORT = "9200";
   public static final String FLINT_DEFAULT_SCHEME = "http";
-  public static final String FLINT_DEFAULT_AUTH = "-1";
+  public static final String FLINT_DEFAULT_AUTH = "noauth";
   public static final String FLINT_DEFAULT_REGION = "us-west-2";
   public static final String DEFAULT_CLASS_NAME = "org.opensearch.sql.FlintJob";
   public static final String S3_AWS_CREDENTIALS_PROVIDER_KEY =
@@ -46,6 +46,10 @@ public class SparkConstants {
   public static final String FLINT_INDEX_STORE_PORT_KEY = "spark.datasource.flint.port";
   public static final String FLINT_INDEX_STORE_SCHEME_KEY = "spark.datasource.flint.scheme";
   public static final String FLINT_INDEX_STORE_AUTH_KEY = "spark.datasource.flint.auth";
+  public static final String FLINT_INDEX_STORE_AUTH_USERNAME =
+      "spark.datasource.flint.auth.username";
+  public static final String FLINT_INDEX_STORE_AUTH_PASSWORD =
+      "spark.datasource.flint.auth.password";
   public static final String FLINT_INDEX_STORE_AWSREGION_KEY = "spark.datasource.flint.region";
   public static final String FLINT_CREDENTIALS_PROVIDER_KEY =
       "spark.datasource.flint.customAWSCredentialsProvider";