From 9f3a47bacd178c8f291f30a11ed8eb2b7e4d1eb6 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Wed, 27 Sep 2023 15:24:41 -0700 Subject: [PATCH] update doc --- docs/index.md | 12 +++++++++++- .../org/opensearch/flint/core/FlintOptions.java | 12 ++++++++---- .../spark/sql/flint/config/FlintSparkConf.scala | 4 ++-- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/docs/index.md b/docs/index.md index 7b8d7aef1..593c96f68 100644 --- a/docs/index.md +++ b/docs/index.md @@ -271,7 +271,9 @@ In the index mapping, the `_meta` and `properties`field stores meta and schema i - `spark.datasource.flint.host`: default is localhost. - `spark.datasource.flint.port`: default is 9200. - `spark.datasource.flint.scheme`: default is http. valid values [http, https] -- `spark.datasource.flint.auth`: default is false. valid values [false, sigv4] +- `spark.datasource.flint.auth`: default is false. valid values [false, sigv4, basic] +- `spark.datasource.flint.auth.username`: basic auth username. +- `spark.datasource.flint.auth.password`: basic auth password. - `spark.datasource.flint.region`: default is us-west-2. only been used when auth=sigv4 - `spark.datasource.flint.customAWSCredentialsProvider`: default is empty. - `spark.datasource.flint.write.id_name`: no default value. @@ -455,3 +457,11 @@ Flint use [DefaultAWSCredentialsProviderChain](https://docs.aws.amazon.com/AWSJa --conf spark.emr-serverless.driverEnv.ASSUME_ROLE_CREDENTIALS_ROLE_ARN=arn:aws:iam::AccountB:role/CrossAccountRoleB \ --conf spark.executorEnv.ASSUME_ROLE_CREDENTIALS_ROLE_ARN=arn:aws:iam::AccountBB:role/CrossAccountRoleB ``` + +### Basic Auth +Add Basic Auth configuration when submit Spark Job. Replace username and password with correct one. +``` +--conf spark.datasource.flint.auth=basic +--conf spark.datasource.flint.auth.username=username +--conf spark.datasource.flint.auth.password=password +``` diff --git a/flint-core/src/main/scala/org/opensearch/flint/core/FlintOptions.java b/flint-core/src/main/scala/org/opensearch/flint/core/FlintOptions.java index 1b6d11f7a..1870be572 100644 --- a/flint-core/src/main/scala/org/opensearch/flint/core/FlintOptions.java +++ b/flint-core/src/main/scala/org/opensearch/flint/core/FlintOptions.java @@ -33,9 +33,9 @@ public class FlintOptions implements Serializable { public static final String BASIC_AUTH = "basic"; - public static final String USERNAME = "username"; + public static final String USERNAME = "auth.username"; - public static final String PASSWORD = "password"; + public static final String PASSWORD = "auth.password"; public static final String CUSTOM_AWS_CREDENTIALS_PROVIDER = "customAWSCredentialsProvider"; @@ -94,7 +94,11 @@ public String getCustomAwsCredentialsProvider() { return options.getOrDefault(CUSTOM_AWS_CREDENTIALS_PROVIDER, ""); } - public String getUsername() {return options.getOrDefault(USERNAME, "flint");} + public String getUsername() { + return options.getOrDefault(USERNAME, "flint"); + } - public String getPassword() {return options.getOrDefault(PASSWORD, "flint");} + public String getPassword() { + return options.getOrDefault(PASSWORD, "flint"); + } } diff --git a/flint-spark-integration/src/main/scala/org/apache/spark/sql/flint/config/FlintSparkConf.scala b/flint-spark-integration/src/main/scala/org/apache/spark/sql/flint/config/FlintSparkConf.scala index 7f7539734..a7eaea857 100644 --- a/flint-spark-integration/src/main/scala/org/apache/spark/sql/flint/config/FlintSparkConf.scala +++ b/flint-spark-integration/src/main/scala/org/apache/spark/sql/flint/config/FlintSparkConf.scala @@ -54,12 +54,12 @@ object FlintSparkConf { .doc("authentication type. supported value: NONE_AUTH(false), SIGV4_AUTH(sigv4), BASIC_AUTH") .createWithDefault(FlintOptions.NONE_AUTH) - val USERNAME = FlintConfig("spark.datasource.flint.username") + val USERNAME = FlintConfig("spark.datasource.flint.auth.username") .datasourceOption() .doc("basic auth username") .createWithDefault("flint") - val PASSWORD = FlintConfig("spark.datasource.flint.password") + val PASSWORD = FlintConfig("spark.datasource.flint.auth.password") .datasourceOption() .doc("basic auth password") .createWithDefault("flint")