diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 1c1c904..e5cd682 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -56,6 +56,14 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Free Disk Space + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: false + large-packages: false + - uses: nixbuild/nix-quick-install-action@v27 - name: Restore and cache Nix store diff --git a/docs/schema/config-schema.json b/docs/schema/config-schema.json index 7ca2d31..7bf8de7 100644 --- a/docs/schema/config-schema.json +++ b/docs/schema/config-schema.json @@ -73,6 +73,17 @@ "null" ] }, + "identity_cache": { + "description": "AWS SDK identity cache configuration", + "anyOf": [ + { + "$ref": "#/definitions/IdentityCache" + }, + { + "type": "null" + } + ] + }, "profile": { "description": "AWS Profile name. Must exist locally in AWS config.\n\nIt's advised not to use this directly as profile name configuration is higly dependent on local configuration. Prefer using AWS_PROFILE environment variable where needed.", "type": [ @@ -643,6 +654,21 @@ } } }, + "IdentityCache": { + "description": "AWS SDK identity cache configuration", + "type": "object", + "properties": { + "load_timeout": { + "description": "Timeout to load identity (in seconds, default: 5s). Useful when asking for MFA authentication which may take more than 5 seconds for user to input.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + } + } + }, "NovopsConfig": { "description": "Global Novops configuration defining behavior for modules", "type": "object", diff --git a/docs/src/config/aws.md b/docs/src/config/aws.md index 8f0c2b6..aeb9311 100644 --- a/docs/src/config/aws.md +++ b/docs/src/config/aws.md @@ -5,6 +5,8 @@ - [Systems Manager (SSM) Parameter Store](#systems-manager-ssm-parameter-store) - [Secrets Manager](#secrets-manager) - [S3 file](#s3-file) +- [Advanced examples](#advanced-examples) + - [Using `credential_process` with TOTP or other user prompt](#using-credential_process-with-totp-or-other-user-prompt) ## Authentication & Configuration @@ -20,9 +22,26 @@ You can also use `config` root element override certains configs (such as AWS en ```yaml config: + + # Example global AWS config + # Every field is optional aws: - endpoint: "http://localhost:4566/" # Use LocalStack endpoint - region: eu-central-1 # Set AWS region name + + # Use a custom endpoint + endpoint: "http://localhost:4566/" + + # Set AWS region name + region: eu-central-1 + + # Set identity cache load timeout. + # + # By default identity load timeout is 5 seconds + # but some custom config may require more than 5 seconds to load identity, + # eg. when prompting user for TOTP. + # + # See Advanced examples below for usage + identity_cache: + load_timeout: 120 # timeout in seconds ``` ## STS Assume Role @@ -119,3 +138,25 @@ aws_s3_object: key: path/to/object region: eu-central-1 ``` +## Advanced examples + +### Using `credential_process` with TOTP or other user prompt + +In some scenario you might want to use `credential_process` in your config, such as [`aws-vault`], which may ask for TOTP or other user prompts. + +For example, using `~/.aws/config` such as: + +```toml +[profile crafteo] +credential_process = aws-vault export --format=json crafteo +mfa_serial = arn:aws:iam::0123456789:mfa/my-mfa +``` + +Credential processor prompts user for TOTP but by default AWS SDK timeout after a few seconds - not enough time to enter data. You can configure identity cache load timeout to give enough time to user. In `.novops.yml`, set config such as: + +```yaml +config: + aws: + identity_cache: + load_timeout: 120 # Give user 2 min to enter TOTP +``` \ No newline at end of file diff --git a/docs/src/contributing/development.md b/docs/src/contributing/development.md index ef7e8e5..9a7297e 100644 --- a/docs/src/contributing/development.md +++ b/docs/src/contributing/development.md @@ -85,12 +85,18 @@ Requirements: - GCP account Integration tests run with real services, preferrably in containers or using dedicated Cloud account: -- AWS: [LocalStack](https://localstack.cloud) server -- Hashicorp Vault: [Vault Docker image](https://hub.docker.com/_/vault) +- AWS: [LocalStack](https://localstack.cloud) container (AWS emulated in a container) +- Hashicorp Vault: [Vault container](https://hub.docker.com/_/vault) - Google Cloud: GCP account - Azure: Azure account -Setup is done via Pulumi (see `tests/setup/pulumi` and Task `test-setup`). +Integration test setup is fully automated but **may create real Cloud resources**. Run: + +```sh +task test-setup +``` + +See `tests/setup/pulumi`. **Remember to `task teardown` after running integration tests.** Cost should be negligible if you teardown infrastructure right after running tests. Cost should still be negligible even if you forget to teardown as only free or cheap resources are deployed, but better to do it anyway. diff --git a/flake.nix b/flake.nix index 7880510..7857466 100644 --- a/flake.nix +++ b/flake.nix @@ -93,6 +93,7 @@ sops age awscli2-patched + aws-vault pulumi pulumiPackages.pulumi-language-nodejs diff --git a/src/modules/aws/client.rs b/src/modules/aws/client.rs index 462a61c..770857a 100644 --- a/src/modules/aws/client.rs +++ b/src/modules/aws/client.rs @@ -1,10 +1,12 @@ +use std::time::Duration; + use crate::core::NovopsContext; use super::config::AwsClientConfig; use aws_config::{BehaviorVersion, Region}; use aws_sdk_secretsmanager::operation::get_secret_value::GetSecretValueOutput; use aws_sdk_sts::{operation::assume_role::AssumeRoleOutput, types::builders::CredentialsBuilder}; use aws_sdk_ssm::{operation::get_parameter::GetParameterOutput, types::builders::ParameterBuilder}; -use aws_sdk_s3::{operation::get_object::GetObjectOutput, primitives::ByteStream}; +use aws_sdk_s3::{config::IdentityCache, operation::get_object::GetObjectOutput, primitives::ByteStream}; use anyhow::Context; use aws_smithy_types::DateTime; use log::debug; @@ -162,7 +164,15 @@ pub fn build_mutable_client_config_from_context(ctx: &NovopsContext) -> AwsClien * Create an SdkConfig using optional overrides */ pub async fn get_sdk_config(client_conf: &AwsClientConfig) -> Result { - + let config_loader = build_config_loader(client_conf)?; + Ok(config_loader.load().await) +} + +/** + * Private function only used by get_sdk_config + * This function wraps a logic for unit testing + */ +fn build_config_loader(client_conf: &AwsClientConfig) -> Result { let mut shared_config = aws_config::defaults(BehaviorVersion::v2024_03_28()); if let Some(endpoint) = &client_conf.endpoint { @@ -177,8 +187,17 @@ pub async fn get_sdk_config(client_conf: &AwsClientConfig) -> Result Result{ @@ -191,7 +210,7 @@ pub async fn get_iam_client(novops_aws: &AwsClientConfig) -> Result Result{ let conf = get_sdk_config(novops_aws).await?; - debug!("Creating AWS STS client with config {:?}", conf); + debug!("Creating AWS STS client with config {:?}", &conf); Ok(aws_sdk_sts::Client::new(&conf)) } @@ -222,4 +241,4 @@ pub async fn get_s3_client(novops_aws: &AwsClientConfig, region: &Option }; Ok(aws_sdk_s3::Client::from_conf(s3_conf.build())) -} +} \ No newline at end of file diff --git a/src/modules/aws/config.rs b/src/modules/aws/config.rs index 5c3a3fe..fcc8369 100644 --- a/src/modules/aws/config.rs +++ b/src/modules/aws/config.rs @@ -10,7 +10,7 @@ pub struct AwsInput { /** - * Generic AWS Client config xrapped around builder pattern + * Generic AWS Client config wrapped around builder pattern * for easy loading from Novops config and per-module override */ #[derive(Default)] @@ -18,6 +18,7 @@ pub struct AwsClientConfig { pub profile: Option, pub endpoint: Option, pub region: Option, + pub identity_cache: Option, } impl From<&AwsConfig> for AwsClientConfig { @@ -26,6 +27,7 @@ impl From<&AwsConfig> for AwsClientConfig { profile: cf.profile.clone(), endpoint: cf.endpoint.clone(), region: cf.region.clone(), + identity_cache: cf.identity_cache.clone() } } } @@ -58,6 +60,17 @@ pub struct AwsConfig { pub profile: Option, /// AWS region to use. Default to currently configured region. - pub region: Option + pub region: Option, + + /// AWS SDK identity cache configuration + pub identity_cache: Option +} + +/// AWS SDK identity cache configuration +#[derive(Debug, Deserialize, Clone, PartialEq, JsonSchema, Default)] +pub struct IdentityCache { + /// Timeout to load identity (in seconds, default: 5s). + /// Useful when asking for MFA authentication which may take more than 5 seconds for user to input. + pub load_timeout: Option } diff --git a/src/modules/hashivault/client.rs b/src/modules/hashivault/client.rs index cc4654e..f1eaa82 100644 --- a/src/modules/hashivault/client.rs +++ b/src/modules/hashivault/client.rs @@ -9,7 +9,7 @@ use std::env::VarError; use vaultrs::{kv2, kv1, aws, auth, api::aws::requests::GenerateCredentialsRequest}; use log::debug; use home; -use crate::modules::hashivault::config::{HashiVaultAuth}; +use crate::modules::hashivault::config::HashiVaultAuth; const KUBERNETES_SA_JWT_PATH: &str = "/var/run/secrets/kubernetes.io/serviceaccount/token"; diff --git a/tests/.novops.aws_assumerole_id_cache_load_timeout.yml b/tests/.novops.aws_assumerole_id_cache_load_timeout.yml new file mode 100644 index 0000000..d22f59f --- /dev/null +++ b/tests/.novops.aws_assumerole_id_cache_load_timeout.yml @@ -0,0 +1,18 @@ +# Test config using config.aws.identity_cache.load_timeout +# used by unit test to check identity cache config is used as expected +environments: + timeout: + aws: + assume_role: + role_arn: arn:aws:iam::111122223333:role/NovopsTestAssumeRole + source_profile: novops-aws-test-identity-cache-load-timeout + + +config: + aws: + endpoint: "http://localhost:4566/" # LocalStack + # Set timeout to 10 + # Source profile novops-aws-test-identity-cache-load-timeout will wait for 7 seconds + # to check timeout is used as expected + identity_cache: + load_timeout: 10 diff --git a/tests/.novops.aws_assumerole_id_cache_load_timeout_short.yml b/tests/.novops.aws_assumerole_id_cache_load_timeout_short.yml new file mode 100644 index 0000000..7270492 --- /dev/null +++ b/tests/.novops.aws_assumerole_id_cache_load_timeout_short.yml @@ -0,0 +1,12 @@ +environments: + timeout: + aws: + assume_role: + role_arn: arn:aws:iam::111122223333:role/NovopsTestAssumeRole + source_profile: novops-aws-test-identity-cache-load-timeout + +config: + aws: + endpoint: "http://localhost:4566/" # LocalStack + identity_cache: + load_timeout: 1 # Very short, should cause timeout with novops-aws-test-identity-cache-load-timeout profile diff --git a/tests/setup/aws/config b/tests/setup/aws/config index 5500797..a98fb1d 100644 --- a/tests/setup/aws/config +++ b/tests/setup/aws/config @@ -4,4 +4,11 @@ output = json [profile novops-aws-test] region = eu-west-3 -output = json \ No newline at end of file +output = json + +# Profile used to test identity cache load timeout config +# Cause a few seconds delay to check timeout config works +[profile novops-aws-test-identity-cache-load-timeout] +region = eu-west-3 +output = json +credential_process = sh -c "sleep 7 && aws sts get-session-token --output json | jq '{Version: 1, AccessKeyId: .Credentials.AccessKeyId, SecretAccessKey: .Credentials.SecretAccessKey, SessionToken: .Credentials.SessionToken, Expiration: .Credentials.Expiration}' -r" \ No newline at end of file diff --git a/tests/test_aws.rs b/tests/test_aws.rs index 2adaa2a..99d7e21 100644 --- a/tests/test_aws.rs +++ b/tests/test_aws.rs @@ -58,6 +58,43 @@ async fn test_assume_role_duration() -> Result<(), anyhow::Error> { Ok(()) } +#[tokio::test] +async fn test_assume_role_identity_cache_load_timeout() -> Result<(), anyhow::Error> { + + test_setup().await?; + + // Takes a few seconds to run but should not timeout + // No error is sufficient to validate test + let outputs = load_env_for("aws_assumerole_id_cache_load_timeout", "timeout").await?; + + let access_key = outputs.variables.get("AWS_ACCESS_KEY_ID").unwrap().clone().value; + + assert!(!access_key.is_empty()); + + Ok(()) +} + + +#[tokio::test] +async fn test_assume_role_identity_cache_load_timeout_error() -> Result<(), anyhow::Error> { + + test_setup().await?; + + // Loading AWS credentials should cause a timeout + let outputs = load_env_for("aws_assumerole_id_cache_load_timeout_short", "timeout").await; + + assert!(outputs.is_err(), "Expected timeout to occur as per identity cache load timeout config."); + + let error = outputs.err().unwrap(); + let error_str = format!("{:?}", error); // a bit hard way to print all error messages in a single string + + info!("test_assume_role_identity_cache_load_timeout_error: Got expected error: {}", &error_str); + + assert!(error_str.contains("identity resolver timed out after"), "Error message did not contain the expected 'identity resolver timed out after' string."); + + Ok(()) +} + #[tokio::test] async fn test_ssm_param() -> Result<(), anyhow::Error> {