Merge pull request #82 from civitaspo/develop

0.4.0

.github/FUNDING.yml

@@ -0,0 +1 @@
+custom: https://www.paypal.me/civitaspo

CHANGELOG.md

@@ -1,3 +1,9 @@
+0.4.0 (2019-10-20)
+==================
+* [Enhancement] Update dependencies: aws-sdk 1.11.587  -> 1.11.653,  digdag 0.9.37  -> 0.9.39
+* [Fix] Fix README: athena.query> preview option is false by default
+* [New feature] Support auth_method: web_identity_token
+
 0.3.2 (2019-08-06)
 ==================
 

README.md

@@ -15,7 +15,7 @@ _export:
     repositories:
       - https://jitpack.io
     dependencies:
-      - pro.civitaspo:digdag-operator-athena:0.3.2
+      - pro.civitaspo:digdag-operator-athena:0.4.0
   athena:
     auth_method: profile
 
@@ -26,10 +26,9 @@ _export:
   echo>: ${athena.last_query}
 
 +stap3:
-  athena.ctas>:
-  select_query: template.sql
+  athena.ctas>: template.sql
   table: hoge
-  output: s3://mybucket/prefix/
+  location: s3://mybucket/prefix/
 ```
 
 See [examples](./example/example.dig) for more cases.
@@ -51,7 +50,10 @@ Define the below options on properties (which is indicated by `-c`, `--config`).
 - **athena.allow_auth_method_instance**: Indicates whether users can use **auth_method** `"instance"` (boolean, default: `false`)
 - **athena.allow_auth_method_profile**: Indicates whether users can use **auth_method** `"profile"` (boolean, default: `false`)
 - **athena.allow_auth_method_properties**: Indicates whether users can use **auth_method** `"properties"` (boolean, default: `false`)
+- **athena.allow_auth_method_web_identity_token**: Indicates whether users can use **auth_method** `"web_identity_token"` (boolean, default: `false`)
 - **athena.assume_role_timeout_duration**: Maximum duration which server administer allows when users assume **role_arn**. (`DurationParam`, default: `1h`)
+- **athena.default_web_identity_token_file**: Path to a web identity token file. (string, optional)
+- **athena.default_web_identity_role_arn**:  AWS Role when using a web identity token. (string, optional)
 
 ### Secrets
 
@@ -78,6 +80,9 @@ Define the below options on properties (which is indicated by `-c`, `--config`).
   - `"properties"`: uses aws.accessKeyId and aws.secretKey Java system properties.
   - `"anonymous"`: uses anonymous access. This auth method can access only public files.
   - `"session"`: uses temporary-generated access_key_id, secret_access_key and session_token.
+  - `"web_identity_token"`: uses web identity token.
+    - **web_identity_token_file**: path to a web identity token file. (string, default: given by **athena.default_web_identity_token_file**)
+    - **web_identity_role_arn**:  aws role arn when using a web identity token. (string, default: given by **athena.default_web_identity_role_arn**)
 - **use_http_proxy**: Indicate whether using when accessing AWS via http proxy. (boolean, default: `false`)
 - **region**: The AWS region to use for Athena service. (string, optional)
 - **endpoint**: The Amazon Athena endpoint address to use. (string, optional)
@@ -171,7 +176,7 @@ Nothing
 - **database**: The name of the database. (string, optional)
 - **workgroup**: The name of the workgroup in which the query is being started. (string, optional)
 - **timeout**: Specify timeout period. (`DurationParam`, default: `"10m"`)
-- **preview**: Call `athena.preview>` operator after run `athena.query>`. (boolean, default: `true`)
+- **preview**: Call `athena.preview>` operator after run `athena.query>`. (boolean, default: `false`)
 
 ### Output Parameters
 
@@ -218,7 +223,6 @@ Nothing
 - **database**: The database name for query execution context. (string, optional)
 - **table**: The table name for the new table (string, default: `digdag_athena_ctas_${session_uuid.replaceAll("-", "")}_${random}`)
 - **workgroup**: The name of the workgroup in which the query is being started. (string, optional)
-- **output**: [**Deprecated**] Use **location** option instead.
 - **location**: Output location for data created by CTAS (string, default: `"s3://aws-athena-query-results-${AWS_ACCOUNT_ID}-<AWS_REGION>/Unsaved/${YEAR}/${MONTH}/${DAY}/${athena_query_id}/"`)
 - **format**: The data format for the CTAS query results, such as `"orc"`, `"parquet"`, `"avro"`, `"json"`, or `"textfile"`. (string, default: `"parquet"`)
 - **compression**: The compression type to use for `"orc"` or `"parquet"`. (string, default: `"snappy"`)

build.gradle

@@ -5,10 +5,10 @@ plugins {
 }
 
 group = 'pro.civitaspo'
-version = '0.3.2'
+version = '0.4.0'
 
-def digdagVersion = '0.9.37'
-def awsSdkVersion = "1.11.587"
+def digdagVersion = '0.9.39'
+def awsSdkVersion = "1.11.653"
 def scalaSemanticVersion = "2.13.0"
 def depScalaVersion = "2.13"
 

example/example.dig

@@ -4,7 +4,7 @@ _export:
       - file://${repos}
       # - https://jitpack.io
     dependencies:
-      - pro.civitaspo:digdag-operator-athena:0.3.2
+      - pro.civitaspo:digdag-operator-athena:0.4.0
   athena:
     auth_method: profile
     value: 5

src/main/scala/pro/civitaspo/digdag/plugin/athena/AbstractAthenaOperator.scala

@@ -39,12 +39,17 @@ abstract class AbstractAthenaOperator(operatorName: String,
             isAllowedAuthMethodInstance = systemConfig.get("athena.allow_auth_method_instance", classOf[Boolean], false),
             isAllowedAuthMethodProfile = systemConfig.get("athena.allow_auth_method_profile", classOf[Boolean], false),
             isAllowedAuthMethodProperties = systemConfig.get("athena.allow_auth_method_properties", classOf[Boolean], false),
+            isAllowedAuthMethodWebIdentityToken = systemConfig.get("athena.allow_auth_method_web_identity_token", classOf[Boolean], false),
             assumeRoleTimeoutDuration = systemConfig.get("athena.assume_role_timeout_duration", classOf[DurationParam], DurationParam.parse("1h")),
             accessKeyId = secrets.getSecretOptional("access_key_id"),
             secretAccessKey = secrets.getSecretOptional("secret_access_key"),
             sessionToken = secrets.getSecretOptional("session_token"),
             roleArn = secrets.getSecretOptional("role_arn"),
             roleSessionName = secrets.getSecretOptional("role_session_name").or(s"digdag-athena-$sessionUuid"),
+            defaultWebIdentityTokenFile = systemConfig.getOptional("athena.default_web_identity_token_file", classOf[String]),
+            webIdentityTokenFile = params.getOptional("web_identity_token_file", classOf[String]),
+            defaultWebIdentityRoleArn = systemConfig.getOptional("athena.default_web_identity_role_arn", classOf[String]),
+            webIdentityRoleArn = params.getOptional("web_identity_role_arn", classOf[String]),
             httpProxy = secrets.getSecrets("http_proxy"),
             authMethod = params.get("auth_method", classOf[String], "basic"),
             profileName = params.get("profile_name", classOf[String], "default"),

src/main/scala/pro/civitaspo/digdag/plugin/athena/aws/Aws.scala

@@ -2,7 +2,7 @@ package pro.civitaspo.digdag.plugin.athena.aws
 
 
 import com.amazonaws.{ClientConfiguration, Protocol}
-import com.amazonaws.auth.{AnonymousAWSCredentials, AWSCredentials, AWSCredentialsProvider, AWSStaticCredentialsProvider, BasicAWSCredentials, BasicSessionCredentials, EC2ContainerCredentialsProviderWrapper, EnvironmentVariableCredentialsProvider, SystemPropertiesCredentialsProvider}
+import com.amazonaws.auth.{AnonymousAWSCredentials, AWSCredentials, AWSCredentialsProvider, AWSStaticCredentialsProvider, BasicAWSCredentials, BasicSessionCredentials, EC2ContainerCredentialsProviderWrapper, EnvironmentVariableCredentialsProvider, SystemPropertiesCredentialsProvider, WebIdentityTokenCredentialsProvider}
 import com.amazonaws.auth.profile.{ProfileCredentialsProvider, ProfilesConfigFile}
 import com.amazonaws.client.builder.AwsClientBuilder
 import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration
@@ -78,14 +78,15 @@ case class Aws(conf: AwsConf)
     private def standardCredentialsProvider: AWSCredentialsProvider =
     {
         conf.authMethod match {
-            case "basic"      => basicAuthMethodAWSCredentialsProvider
-            case "env"        => envAuthMethodAWSCredentialsProvider
-            case "instance"   => instanceAuthMethodAWSCredentialsProvider
-            case "profile"    => profileAuthMethodAWSCredentialsProvider
-            case "properties" => propertiesAuthMethodAWSCredentialsProvider
-            case "anonymous"  => anonymousAuthMethodAWSCredentialsProvider
-            case "session"    => sessionAuthMethodAWSCredentialsProvider
-            case _            =>
+            case "basic"              => basicAuthMethodAWSCredentialsProvider
+            case "env"                => envAuthMethodAWSCredentialsProvider
+            case "instance"           => instanceAuthMethodAWSCredentialsProvider
+            case "profile"            => profileAuthMethodAWSCredentialsProvider
+            case "properties"         => propertiesAuthMethodAWSCredentialsProvider
+            case "anonymous"          => anonymousAuthMethodAWSCredentialsProvider
+            case "session"            => sessionAuthMethodAWSCredentialsProvider
+            case "web_identity_token" => webIdentityTokenAuthMethodAWSCredentialsProvider
+            case _                    =>
                 throw new ConfigException(
                     s"""auth_method: "${conf.authMethod}" is not supported. available `auth_method`s are "basic", "env", "instance", "profile", "properties", "anonymous", or "session"."""
                     )
@@ -151,6 +152,18 @@ case class Aws(conf: AwsConf)
         new AWSStaticCredentialsProvider(credentials)
     }
 
+    private def webIdentityTokenAuthMethodAWSCredentialsProvider: AWSCredentialsProvider =
+    {
+        if (!conf.isAllowedAuthMethodWebIdentityToken) throw new ConfigException(s"""auth_method: "${conf.authMethod}" is not allowed.""")
+        if (!conf.webIdentityTokenFile.or(conf.defaultWebIdentityTokenFile).isPresent) throw new ConfigException(s"""`web_identity_token_file` or `athena.allow_auth_method_web_identity_token` (system) must be set when `auth_method` is "${conf.authMethod}".""")
+        if (!conf.webIdentityRoleArn.or(conf.defaultWebIdentityRoleArn).isPresent) throw new ConfigException(s"""`web_identity_role_arn` or `athena.allow_auth_method_web_identity_role_arn` (system) must be set when `auth_method` is "${conf.authMethod}".""")
+        WebIdentityTokenCredentialsProvider.builder()
+            .webIdentityTokenFile(conf.webIdentityTokenFile.or(conf.defaultWebIdentityTokenFile).get())
+            .roleArn(conf.webIdentityRoleArn.or(conf.defaultWebIdentityRoleArn).get())
+            .roleSessionName(conf.roleSessionName)
+            .build()
+    }
+
     private def clientConfiguration: ClientConfiguration =
     {
         if (!conf.useHttpProxy) return new ClientConfiguration()

src/main/scala/pro/civitaspo/digdag/plugin/athena/aws/AwsConf.scala

@@ -10,6 +10,7 @@ case class AwsConf(
     isAllowedAuthMethodInstance: Boolean,
     isAllowedAuthMethodProfile: Boolean,
     isAllowedAuthMethodProperties: Boolean,
+    isAllowedAuthMethodWebIdentityToken: Boolean,
     assumeRoleTimeoutDuration: DurationParam,
     accessKeyId: Optional[String],
     secretAccessKey: Optional[String],
@@ -20,6 +21,10 @@ case class AwsConf(
     authMethod: String,
     profileName: String,
     profileFile: Optional[String],
+    defaultWebIdentityTokenFile: Optional[String],
+    webIdentityTokenFile: Optional[String],
+    defaultWebIdentityRoleArn: Optional[String],
+    webIdentityRoleArn: Optional[String],
     useHttpProxy: Boolean,
     region: Optional[String],
     endpoint: Optional[String]