Merge pull request #127 from PierreBeucher/identity-cache-config

Identity cache config

.github/workflows/build-test.yml

@@ -56,6 +56,14 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: Free Disk Space
+        uses: jlumbroso/free-disk-space@main
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tool-cache: false
+          large-packages: false
+
       - uses: nixbuild/nix-quick-install-action@v27
 
       - name: Restore and cache Nix store

docs/schema/config-schema.json

@@ -73,6 +73,17 @@
             "null"
           ]
         },
+        "identity_cache": {
+          "description": "AWS SDK identity cache configuration",
+          "anyOf": [
+            {
+              "$ref": "#/definitions/IdentityCache"
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
         "profile": {
           "description": "AWS Profile name. Must exist locally in AWS config.\n\nIt's advised not to use this directly as profile name configuration is higly dependent on local configuration. Prefer using AWS_PROFILE environment variable where needed.",
           "type": [
@@ -643,6 +654,21 @@
         }
       }
     },
+    "IdentityCache": {
+      "description": "AWS SDK identity cache configuration",
+      "type": "object",
+      "properties": {
+        "load_timeout": {
+          "description": "Timeout to load identity (in seconds, default: 5s). Useful when asking for MFA authentication which may take more than 5 seconds for user to input.",
+          "type": [
+            "integer",
+            "null"
+          ],
+          "format": "uint64",
+          "minimum": 0.0
+        }
+      }
+    },
     "NovopsConfig": {
       "description": "Global Novops configuration defining behavior for modules",
       "type": "object",

docs/src/config/aws.md

@@ -5,6 +5,8 @@
 - [Systems Manager (SSM) Parameter Store](#systems-manager-ssm-parameter-store)
 - [Secrets Manager](#secrets-manager)
 - [S3 file](#s3-file)
+- [Advanced examples](#advanced-examples)
+  - [Using `credential_process` with TOTP or other user prompt](#using-credential_process-with-totp-or-other-user-prompt)
 
 ## Authentication & Configuration
 
@@ -20,9 +22,26 @@ You can also use `config` root element override certains configs (such as AWS en
 
 ```yaml
 config:
+
+  # Example global AWS config
+  # Every field is optional
   aws:
-    endpoint: "http://localhost:4566/" # Use LocalStack endpoint
-    region: eu-central-1 # Set AWS region name
+
+    # Use a custom endpoint
+    endpoint: "http://localhost:4566/" 
+
+    # Set AWS region name
+    region: eu-central-1 
+
+    # Set identity cache load timeout.
+    #
+    # By default identity load timeout is 5 seconds
+    # but some custom config may require more than 5 seconds to load identity, 
+    # eg. when prompting user for TOTP.
+    #
+    # See Advanced examples below for usage
+    identity_cache:
+      load_timeout: 120 # timeout in seconds
 ```
 
 ## STS Assume Role
@@ -119,3 +138,25 @@ aws_s3_object:
   key: path/to/object
   region: eu-central-1
 ```
+## Advanced examples
+
+### Using `credential_process` with TOTP or other user prompt
+
+In some scenario you might want to use `credential_process` in your config, such as [`aws-vault`], which may ask for TOTP or other user prompts.
+
+For example, using `~/.aws/config` such as:
+
+```toml
+[profile crafteo]
+credential_process = aws-vault export --format=json crafteo
+mfa_serial = arn:aws:iam::0123456789:mfa/my-mfa
+```
+
+Credential processor prompts user for TOTP but by default AWS SDK timeout after a few seconds - not enough time to enter data. You can configure identity cache load timeout to give enough time to user. In `.novops.yml`, set config such as:
+
+```yaml
+config:
+  aws:
+    identity_cache:
+      load_timeout: 120 # Give user 2 min to enter TOTP
+```

docs/src/contributing/development.md

@@ -85,12 +85,18 @@ Requirements:
 - GCP account
 
 Integration tests run with real services, preferrably in containers or using dedicated Cloud account:
-- AWS: [LocalStack](https://localstack.cloud) server
-- Hashicorp Vault: [Vault Docker image](https://hub.docker.com/_/vault)
+- AWS: [LocalStack](https://localstack.cloud) container (AWS emulated in a container)
+- Hashicorp Vault: [Vault container](https://hub.docker.com/_/vault)
 - Google Cloud: GCP account
 - Azure: Azure account
 
-Setup is done via Pulumi (see `tests/setup/pulumi` and Task `test-setup`). 
+Integration test setup is fully automated but **may create real Cloud resources**. Run:
+
+```sh
+task test-setup
+```
+
+See `tests/setup/pulumi`. 
 
 **Remember to `task teardown` after running integration tests.** Cost should be negligible if you teardown infrastructure right after running tests. Cost should still be negligible even if you forget to teardown as only free or cheap resources are deployed, but better to do it anyway. 
 

flake.nix

@@ -93,6 +93,7 @@
           sops
           age
           awscli2-patched
+          aws-vault
 
           pulumi
           pulumiPackages.pulumi-language-nodejs

src/modules/aws/client.rs

@@ -1,10 +1,12 @@
+use std::time::Duration;
+
 use crate::core::NovopsContext;
 use super::config::AwsClientConfig;
 use aws_config::{BehaviorVersion, Region};
 use aws_sdk_secretsmanager::operation::get_secret_value::GetSecretValueOutput;
 use aws_sdk_sts::{operation::assume_role::AssumeRoleOutput, types::builders::CredentialsBuilder};
 use aws_sdk_ssm::{operation::get_parameter::GetParameterOutput, types::builders::ParameterBuilder};
-use aws_sdk_s3::{operation::get_object::GetObjectOutput, primitives::ByteStream};
+use aws_sdk_s3::{config::IdentityCache, operation::get_object::GetObjectOutput, primitives::ByteStream};
 use anyhow::Context;
 use aws_smithy_types::DateTime;
 use log::debug;
@@ -162,7 +164,15 @@ pub fn build_mutable_client_config_from_context(ctx: &NovopsContext) -> AwsClien
  * Create an SdkConfig using optional overrides 
  */
 pub async fn get_sdk_config(client_conf: &AwsClientConfig) -> Result<aws_config::SdkConfig, anyhow::Error> {
-
+    let config_loader = build_config_loader(client_conf)?;
+    Ok(config_loader.load().await)
+}
+
+/**
+ * Private function only used by get_sdk_config
+ * This function wraps a logic for unit testing
+ */
+fn build_config_loader(client_conf: &AwsClientConfig) -> Result<aws_config::ConfigLoader, anyhow::Error> {
     let mut shared_config = aws_config::defaults(BehaviorVersion::v2024_03_28());
 
     if let Some(endpoint) = &client_conf.endpoint {
@@ -177,8 +187,17 @@ pub async fn get_sdk_config(client_conf: &AwsClientConfig) -> Result<aws_config:
         shared_config = shared_config.region(Region::new(region.clone()));
     }
 
-    Ok(shared_config.load().await)
+    if let Some(identity_cache) = &client_conf.identity_cache {
+        let mut id_cache_builder = IdentityCache::lazy();
+
+        if let Some(timeout) = identity_cache.load_timeout {
+            id_cache_builder = id_cache_builder.load_timeout(Duration::from_secs(timeout));
+        }
 
+        shared_config = shared_config.identity_cache(id_cache_builder.build());
+    }
+
+    Ok(shared_config)
 }
 
 pub async fn get_iam_client(novops_aws: &AwsClientConfig) -> Result<aws_sdk_iam::Client, anyhow::Error>{
@@ -191,7 +210,7 @@ pub async fn get_iam_client(novops_aws: &AwsClientConfig) -> Result<aws_sdk_iam:
 pub async fn get_sts_client(novops_aws: &AwsClientConfig) -> Result<aws_sdk_sts::Client, anyhow::Error>{
     let conf = get_sdk_config(novops_aws).await?;
 
-    debug!("Creating AWS STS client with config {:?}", conf);
+    debug!("Creating AWS STS client with config {:?}", &conf);
     Ok(aws_sdk_sts::Client::new(&conf))
 }
 
@@ -222,4 +241,4 @@ pub async fn get_s3_client(novops_aws: &AwsClientConfig, region: &Option<String>
     };
 
     Ok(aws_sdk_s3::Client::from_conf(s3_conf.build()))
-}
+}

src/modules/aws/config.rs

@@ -10,14 +10,15 @@ pub struct AwsInput {
 
 
 /**
- * Generic AWS Client config xrapped around builder pattern
+ * Generic AWS Client config wrapped around builder pattern
  * for easy loading from Novops config and per-module override
  */
 #[derive(Default)]
 pub struct AwsClientConfig {
     pub profile: Option<String>,
     pub endpoint: Option<String>,
     pub region: Option<String>,
+    pub identity_cache: Option<IdentityCache>,
 }
 
 impl From<&AwsConfig> for AwsClientConfig {
@@ -26,6 +27,7 @@ impl From<&AwsConfig> for AwsClientConfig {
             profile: cf.profile.clone(),
             endpoint: cf.endpoint.clone(),
             region: cf.region.clone(),
+            identity_cache: cf.identity_cache.clone()
         }
     }
 }
@@ -58,6 +60,17 @@ pub struct AwsConfig {
     pub profile: Option<String>,
 
     /// AWS region to use. Default to currently configured region. 
-    pub region: Option<String>
+    pub region: Option<String>,
+
+    /// AWS SDK identity cache configuration
+    pub identity_cache: Option<IdentityCache>
+}
+
+/// AWS SDK identity cache configuration
+#[derive(Debug, Deserialize, Clone, PartialEq, JsonSchema, Default)]
+pub struct IdentityCache {
+    /// Timeout to load identity (in seconds, default: 5s).
+    /// Useful when asking for MFA authentication which may take more than 5 seconds for user to input.
+    pub load_timeout: Option<u64>
 }
 

src/modules/hashivault/client.rs

@@ -9,7 +9,7 @@ use std::env::VarError;
 use vaultrs::{kv2, kv1, aws, auth, api::aws::requests::GenerateCredentialsRequest};
 use log::debug;
 use home;
-use crate::modules::hashivault::config::{HashiVaultAuth};
+use crate::modules::hashivault::config::HashiVaultAuth;
 
 
 const KUBERNETES_SA_JWT_PATH: &str = "/var/run/secrets/kubernetes.io/serviceaccount/token";

tests/.novops.aws_assumerole_id_cache_load_timeout.yml

@@ -0,0 +1,18 @@
+# Test config using config.aws.identity_cache.load_timeout
+# used by unit test to check identity cache config is used as expected
+environments:
+  timeout:
+    aws:
+      assume_role: 
+        role_arn: arn:aws:iam::111122223333:role/NovopsTestAssumeRole
+        source_profile: novops-aws-test-identity-cache-load-timeout
+
+
+config:
+  aws:
+    endpoint: "http://localhost:4566/" # LocalStack
+    # Set timeout to 10
+    # Source profile novops-aws-test-identity-cache-load-timeout will wait for 7 seconds
+    # to check timeout is used as expected
+    identity_cache:
+      load_timeout: 10

tests/.novops.aws_assumerole_id_cache_load_timeout_short.yml

@@ -0,0 +1,12 @@
+environments:
+  timeout:
+    aws:
+      assume_role: 
+        role_arn: arn:aws:iam::111122223333:role/NovopsTestAssumeRole
+        source_profile: novops-aws-test-identity-cache-load-timeout
+
+config:
+  aws:
+    endpoint: "http://localhost:4566/" # LocalStack
+    identity_cache:
+      load_timeout: 1 # Very short, should cause timeout with novops-aws-test-identity-cache-load-timeout profile

tests/setup/aws/config

@@ -4,4 +4,11 @@ output = json
 
 [profile novops-aws-test]
 region = eu-west-3
-output = json
+output = json
+
+# Profile used to test identity cache load timeout config
+# Cause a few seconds delay to check timeout config works
+[profile novops-aws-test-identity-cache-load-timeout]
+region = eu-west-3
+output = json
+credential_process = sh -c "sleep 7 && aws sts get-session-token --output json | jq '{Version: 1, AccessKeyId: .Credentials.AccessKeyId, SecretAccessKey: .Credentials.SecretAccessKey, SessionToken: .Credentials.SessionToken, Expiration: .Credentials.Expiration}' -r"

tests/test_aws.rs

@@ -58,6 +58,43 @@ async fn test_assume_role_duration() -> Result<(), anyhow::Error> {
     Ok(())
 }
 
+#[tokio::test]
+async fn test_assume_role_identity_cache_load_timeout() -> Result<(), anyhow::Error> {
+
+    test_setup().await?;
+
+    // Takes a few seconds to run but should not timeout
+    // No error is sufficient to validate test
+    let outputs = load_env_for("aws_assumerole_id_cache_load_timeout", "timeout").await?;
+
+    let access_key = outputs.variables.get("AWS_ACCESS_KEY_ID").unwrap().clone().value;
+
+    assert!(!access_key.is_empty());
+
+    Ok(())
+}
+
+
+#[tokio::test]
+async fn test_assume_role_identity_cache_load_timeout_error() -> Result<(), anyhow::Error> {
+
+    test_setup().await?;
+
+    // Loading AWS credentials should cause a timeout
+    let outputs = load_env_for("aws_assumerole_id_cache_load_timeout_short", "timeout").await;
+
+    assert!(outputs.is_err(), "Expected timeout to occur as per identity cache load timeout config.");
+
+    let error = outputs.err().unwrap();
+    let error_str = format!("{:?}", error); // a bit hard way to print all error messages in a single string
+
+    info!("test_assume_role_identity_cache_load_timeout_error: Got expected error: {}", &error_str);
+
+    assert!(error_str.contains("identity resolver timed out after"), "Error message did not contain the expected 'identity resolver timed out after' string.");
+
+    Ok(())
+}
+
 #[tokio::test]
 async fn test_ssm_param() -> Result<(), anyhow::Error> {