From f00a130000adb002c25eee90633356a42285bab9 Mon Sep 17 00:00:00 2001 From: Rohith Jayawardene Date: Tue, 19 Nov 2024 08:50:19 +0000 Subject: [PATCH 01/19] chore: ensure additional payer accounts are included in the data collection template --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 188ac1e..753db64 100644 --- a/main.tf +++ b/main.tf @@ -391,7 +391,7 @@ resource "aws_cloudformation_stack" "cudos_data_collection" { "IncludeRightsizingModule" = var.enable_rightsizing_module ? "yes" : "no", "IncludeTAModule" = var.enable_tao_module ? "yes" : "no", "IncludeTransitGatewayModule" = var.enable_transit_gateway_module ? "yes" : "no", - "ManagementAccountID" = local.management_account_id, + "ManagementAccountID" = join(",", local.payer_account_ids), } depends_on = [ From eded7ee711032bf3d1ce832b5de0bf643019f554 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Nov 2024 09:19:01 +0000 Subject: [PATCH 02/19] chore(deps): bump dashboards::aws-cudos-framework-deployment (#89) Bumps [dashboards::aws-cudos-framework-deployment](https://github.com/aws-samples/aws-cudos-framework-deployment) from 4.0.2 to 4.0.5. - [Release notes](https://github.com/aws-samples/aws-cudos-framework-deployment/releases) - [Changelog](https://github.com/aws-samples/aws-cudos-framework-deployment/blob/main/bump-release.py) - [Commits](https://github.com/aws-samples/aws-cudos-framework-deployment/compare/4.0.2...4.0.5) --- updated-dependencies: - dependency-name: dashboards::github::aws-samples/aws-cudos-framework-deployment::4.0.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 753db64..a51e824 100644 --- a/main.tf +++ b/main.tf @@ -310,7 +310,7 @@ resource "aws_cloudformation_stack" "cora_data_export_collector" { ## Provision the cloud intelligence dashboards module "dashboards" { - source = "github.com/aws-samples/aws-cudos-framework-deployment//terraform-modules/cid-dashboards?ref=4.0.2" + source = "github.com/aws-samples/aws-cudos-framework-deployment//terraform-modules/cid-dashboards?ref=4.0.5" stack_name = var.stack_name_cloud_intelligence template_bucket = module.dashboard_bucket.s3_bucket_id From b37d54e4d3ebc2bc9377cf54e52a0df9314d830f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Nov 2024 09:19:09 +0000 Subject: [PATCH 03/19] chore(deps): bump source::aws-cudos-framework-deployment (#88) Bumps [source::aws-cudos-framework-deployment](https://github.com/aws-samples/aws-cudos-framework-deployment) from 4.0.2 to 4.0.5. - [Release notes](https://github.com/aws-samples/aws-cudos-framework-deployment/releases) - [Changelog](https://github.com/aws-samples/aws-cudos-framework-deployment/blob/main/bump-release.py) - [Commits](https://github.com/aws-samples/aws-cudos-framework-deployment/compare/4.0.2...4.0.5) --- updated-dependencies: - dependency-name: source::github::aws-samples/aws-cudos-framework-deployment::4.0.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index a51e824..d1710c4 100644 --- a/main.tf +++ b/main.tf @@ -242,7 +242,7 @@ module "collector" { # tfsec:ignore:aws-s3-enable-bucket-logging # tfsec:ignore:aws-iam-no-policy-wildcards module "source" { - source = "github.com/aws-samples/aws-cudos-framework-deployment//terraform-modules/cur-setup-source?ref=4.0.2" + source = "github.com/aws-samples/aws-cudos-framework-deployment//terraform-modules/cur-setup-source?ref=4.0.5" # The destination bucket to repliaction the CUR data to destination_bucket_arn = module.collector.cur_bucket_arn From 12875958d391f93827b0a117f3e461aa9db47c2b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Nov 2024 09:19:18 +0000 Subject: [PATCH 04/19] chore(deps): bump collector::aws-cudos-framework-deployment (#87) Bumps [collector::aws-cudos-framework-deployment](https://github.com/aws-samples/aws-cudos-framework-deployment) from 4.0.2 to 4.0.5. - [Release notes](https://github.com/aws-samples/aws-cudos-framework-deployment/releases) - [Changelog](https://github.com/aws-samples/aws-cudos-framework-deployment/blob/main/bump-release.py) - [Commits](https://github.com/aws-samples/aws-cudos-framework-deployment/compare/4.0.2...4.0.5) --- updated-dependencies: - dependency-name: collector::github::aws-samples/aws-cudos-framework-deployment::4.0.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index d1710c4..8486f60 100644 --- a/main.tf +++ b/main.tf @@ -224,7 +224,7 @@ module "dashboard_bucket" { ## First we configure the collector to accept the CUR (Cost and Usage Report) from the source account # tfsec:ignore:aws-s3-enable-bucket-logging module "collector" { - source = "github.com/aws-samples/aws-cudos-framework-deployment//terraform-modules/cur-setup-destination?ref=4.0.2" + source = "github.com/aws-samples/aws-cudos-framework-deployment//terraform-modules/cur-setup-destination?ref=4.0.5" # Source account whom will be replicating the CUR data to the collector account source_account_ids = local.payer_account_ids From 04adf29f30aa10ebaaa5ba984d45fcf027a52b3d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Nov 2024 11:11:15 +0000 Subject: [PATCH 05/19] chore(deps): bump hashicorp/aws from 5.76.0 to 5.77.0 (#91) Bumps [hashicorp/aws](https://github.com/hashicorp/terraform-provider-aws) from 5.76.0 to 5.77.0. - [Release notes](https://github.com/hashicorp/terraform-provider-aws/releases) - [Changelog](https://github.com/hashicorp/terraform-provider-aws/blob/main/CHANGELOG.md) - [Commits](https://github.com/hashicorp/terraform-provider-aws/compare/v5.76.0...v5.77.0) --- updated-dependencies: - dependency-name: hashicorp/aws dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .terraform.lock.hcl | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index 7ed4d8a..e2e5997 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -2,24 +2,24 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/aws" { - version = "5.76.0" + version = "5.77.0" constraints = ">= 3.0.0, ~> 5.0, >= 5.27.0" hashes = [ - "h1:bYc0hbgVRXYCiapr/EgjdP8ohcwFjninfknZvqHQZPQ=", - "zh:05b2a0d25fc07576f6698d4840d0d2ae2599484c49f1b911ea1154584557bc13", - "zh:1b22dd1d9c482739e133adb996a9c8b285ca7d978d0fe04deaa5588eba5d254c", - "zh:216088c8800e7b8d7eff7b1a822317bc6faec64f27946ffd22bb3494ac4175cb", - "zh:43e994112b1484bf49945c4885aa2fee32486c9a5d64b9146bbd6f309f24e332", - "zh:46a28ba800f176eef500f998217bccc331605ef05f11abb1728f727a81f3a8b0", - "zh:4fad2743174a600da76a0cceeec2fef8399a18d880ba8929d811cd5cea1b5dee", - "zh:5c42a2c1438cd7533456026f52b562715664490711fdea809f44610a7565c145", - "zh:792d4fd4be434682e4540d2579505c7f11f39d0efe1d12ee2761ed0d46c8cd51", - "zh:7bb5f9f87c9da6d62d6f89504f01a9d6d2f19dcaa0efc46ea51ebdc4bb6fd536", - "zh:81cdbd97f81b1110fce793944d5668a4389904979eb7d178d3142a6b0e175e5e", + "h1:7yv9NDANq8B0hKcxySR053tYoG8rKHC2EobEEXjUdDg=", + "zh:0bb61ed8a86a231e466ceffd010cb446418483853aa7e35ecb628cf578fa3905", + "zh:15d37511e55db46a50e703195858b816b7bbfd7bd6d193abf45aec1cb31cfc29", + "zh:1cdaec2ca4408e90aee6ea550ff4ff01a46033854c26d71309541975aa6317bd", + "zh:1dd2d1af44004b35a1597e82f9aa9d6396a77808371aa4dfd2045a2a144b7329", + "zh:329bf790ef57b29b95eee847090bffb74751b2b5e5a4c23e07367cc0bf9cce10", + "zh:40949e13342a0a738036e66420b7a546bda91ef68038981badbe454545076f16", + "zh:5674eb93c8edd308abac408ae45ee90e59e171d45011f00f5036ff4d43a1de52", + "zh:747624ce0e938dd773bca295df226d39d425d3805e6afe50248159d0f2ec6d3a", + "zh:761795909c5cba10f138d276384fb034031eb1e8c5cdfe3b93794c8a78d909ce", "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:ab4b881eb0f3812b702aaecf921c5c16bbcc33d61d668be4d72d6da9c57ded85", - "zh:c1d9d1166fd948845614deef81f3197568d0d3c2a03b8b97fff308ebc59043f9", - "zh:cda7530f2c01434e483d3faf62fc0685295e7f844176aa38df1ba65fa6a4407a", - "zh:fdad558b1c41aa68123d0da82cc0d65bc86d09eaa1ab1d3a167ec3bce0fc0c66", + "zh:9b95901dae3f2c7eea870d57940117ef5391676689efc565351bb087816674e4", + "zh:9bb86e159828dedc1302844d29ee6d79d6fee732c830a36838c359b9319ab304", + "zh:9e72dfbd7c28da259d51af92c21e580efd0045103cba2bb01cd1a8acb4185883", + "zh:a226b88521022598d1be8361b4f2976834d305ff58c8ea9b9a12c82f9a23f2c2", + "zh:faabcdfa36365359dca214da534cfb2fd5738edb40786c2afd09702f42ad1651", ] } From eaea1d3ef698f33b4e26c73a36edc0b62f91d036 Mon Sep 17 00:00:00 2001 From: Rohith Jayawardene Date: Fri, 1 Nov 2024 15:20:00 +0000 Subject: [PATCH 06/19] feat: separating out the codebase into individual modules --- README.md | 55 +---- data.tf | 19 -- appvia_banner.jpg => docs/banner.jpg | Bin examples/basic/.terraform.lock.hcl | 66 +++--- examples/basic/main.tf | 61 +++-- locals.tf | 31 --- modules/destination/.terraform.lock.hcl | 25 ++ modules/destination/README.md | 60 +++++ modules/destination/data.tf | 3 + modules/destination/locals.tf | 19 ++ main.tf => modules/destination/main.tf | 216 ++---------------- modules/destination/outputs.tf | 11 + .../destination/quicksights.tf | 6 - .../destination/terraform.tf | 5 +- .../destination/variables.tf | 28 +-- modules/source/.terraform.lock.hcl | 25 ++ modules/source/README.md | 39 ++++ .../cudos/data-exports-aggregation.yaml | 0 .../cudos/deploy-data-collection.yaml | 0 .../cudos/deploy-data-read-permissions.yaml | 0 modules/source/data.tf | 9 + modules/source/locals.tf | 14 ++ modules/source/main.tf | 152 ++++++++++++ modules/source/outputs.tf | 17 ++ modules/source/terraform.tf | 13 ++ modules/source/variables.tf | 111 +++++++++ outputs.tf | 50 ---- 27 files changed, 605 insertions(+), 430 deletions(-) delete mode 100644 data.tf rename appvia_banner.jpg => docs/banner.jpg (100%) delete mode 100644 locals.tf create mode 100644 modules/destination/.terraform.lock.hcl create mode 100644 modules/destination/README.md create mode 100644 modules/destination/data.tf create mode 100644 modules/destination/locals.tf rename main.tf => modules/destination/main.tf (51%) create mode 100644 modules/destination/outputs.tf rename quicksights.tf => modules/destination/quicksights.tf (92%) rename terraform.tf => modules/destination/terraform.tf (60%) rename variables.tf => modules/destination/variables.tf (93%) create mode 100644 modules/source/.terraform.lock.hcl create mode 100644 modules/source/README.md rename {assets => modules/source/assets}/cloudformation/cudos/data-exports-aggregation.yaml (100%) rename {assets => modules/source/assets}/cloudformation/cudos/deploy-data-collection.yaml (100%) rename {assets => modules/source/assets}/cloudformation/cudos/deploy-data-read-permissions.yaml (100%) create mode 100644 modules/source/data.tf create mode 100644 modules/source/locals.tf create mode 100644 modules/source/main.tf create mode 100644 modules/source/outputs.tf create mode 100644 modules/source/terraform.tf create mode 100644 modules/source/variables.tf delete mode 100644 outputs.tf diff --git a/README.md b/README.md index 925e3a2..65bac84 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ -Appvia Banner

Terraform Registry Latest Release Slack Community Contributors + +Appvia Banner

Terraform Registry Latest Release Slack Community Contributors ## Providers -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | ~> 5.0 | -| [aws.cost\_analysis](#provider\_aws.cost\_analysis) | ~> 5.0 | -| [aws.management](#provider\_aws.management) | ~> 5.0 | +No providers. ## Inputs -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [dashboards\_bucket\_name](#input\_dashboards\_bucket\_name) | The name of the bucket to store the dashboards configurations | `string` | n/a | yes | -| [tags](#input\_tags) | Tags to apply to all resources | `map(string)` | n/a | yes | -| [additional\_payer\_accounts](#input\_additional\_payer\_accounts) | List of additional payer accounts to be included in the collectors module | `list(string)` | `[]` | no | -| [enable\_backup\_module](#input\_enable\_backup\_module) | Indicates if the Backup module should be enabled | `bool` | `true` | no | -| [enable\_budgets\_module](#input\_enable\_budgets\_module) | Indicates if the Budget module should be enabled | `bool` | `true` | no | -| [enable\_compute\_optimizer\_dashboard](#input\_enable\_compute\_optimizer\_dashboard) | Indicates if the Compute Optimizer dashboard should be enabled | `bool` | `true` | no | -| [enable\_compute\_optimizer\_module](#input\_enable\_compute\_optimizer\_module) | Indicates if the Compute Optimizer module should be enabled | `bool` | `true` | no | -| [enable\_cora\_data\_exports](#input\_enable\_cora\_data\_exports) | Indicates if the CORA Data Exports module should be enabled | `bool` | `false` | no | -| [enable\_cost\_anomaly\_module](#input\_enable\_cost\_anomaly\_module) | Indicates if the Cost Anomaly module should be enabled | `bool` | `true` | no | -| [enable\_cost\_intelligence\_dashboard](#input\_enable\_cost\_intelligence\_dashboard) | Indicates if the Cost Intelligence dashboard should be enabled | `bool` | `true` | no | -| [enable\_cudos\_dashboard](#input\_enable\_cudos\_dashboard) | Indicates if the CUDOS dashboard should be enabled | `bool` | `false` | no | -| [enable\_cudos\_v5\_dashboard](#input\_enable\_cudos\_v5\_dashboard) | Indicates if the CUDOS V5 framework should be enabled | `bool` | `true` | no | -| [enable\_ecs\_chargeback\_module](#input\_enable\_ecs\_chargeback\_module) | Indicates if the ECS Chargeback module should be enabled | `bool` | `false` | no | -| [enable\_health\_events](#input\_enable\_health\_events) | Indicates if the Health Events module should be enabled | `bool` | `true` | no | -| [enable\_inventory\_module](#input\_enable\_inventory\_module) | Indicates if the Inventory module should be enabled | `bool` | `true` | no | -| [enable\_kpi\_dashboard](#input\_enable\_kpi\_dashboard) | Indicates if the KPI dashboard should be enabled | `bool` | `true` | no | -| [enable\_license\_manager\_module](#input\_enable\_license\_manager\_module) | Indicates if the License Manager module should be enabled | `bool` | `false` | no | -| [enable\_org\_data\_module](#input\_enable\_org\_data\_module) | Indicates if the Organization Data module should be enabled | `bool` | `true` | no | -| [enable\_prerequisites\_quicksight](#input\_enable\_prerequisites\_quicksight) | Indicates if the prerequisites for QuickSight should be enabled | `bool` | `true` | no | -| [enable\_prerequisites\_quicksight\_permissions](#input\_enable\_prerequisites\_quicksight\_permissions) | Indicates if the prerequisites for QuickSight permissions should be enabled | `bool` | `true` | no | -| [enable\_quicksight\_subscription](#input\_enable\_quicksight\_subscription) | Enable QuickSight subscription | `bool` | `false` | no | -| [enable\_rds\_utilization\_module](#input\_enable\_rds\_utilization\_module) | Indicates if the RDS Utilization module should be enabled | `bool` | `true` | no | -| [enable\_rightsizing\_module](#input\_enable\_rightsizing\_module) | Indicates if the Rightsizing module should be enabled | `bool` | `true` | no | -| [enable\_scad](#input\_enable\_scad) | Indicates if the SCAD module should be enabled, only available when Cora enabled | `bool` | `false` | no | -| [enable\_sso](#input\_enable\_sso) | Enable integration with identity center for QuickSight | `bool` | `true` | no | -| [enable\_tao\_dashboard](#input\_enable\_tao\_dashboard) | Indicates if the TAO dashboard should be enabled | `bool` | `false` | no | -| [enable\_tao\_module](#input\_enable\_tao\_module) | Indicates if the TAO module should be enabled | `bool` | `true` | no | -| [enable\_transit\_gateway\_module](#input\_enable\_transit\_gateway\_module) | Indicates if the Transit Gateway module should be enabled | `bool` | `true` | no | -| [quicksight\_groups](#input\_quicksight\_groups) | Map of groups with user membership to be added to QuickSight | 

map(object({
    description = optional(string)
    namespace   = optional(string)
    members     = optional(list(string), [])
  }))
| `{}` | no | -| [quicksight\_subscription\_account\_name](#input\_quicksight\_subscription\_account\_name) | The account name for the QuickSight quicksight\_subscription edition | `string` | `null` | no | -| [quicksight\_subscription\_authentication\_method](#input\_quicksight\_subscription\_authentication\_method) | The identity for the QuickSight quicksight\_subscription edition | `string` | `"IAM_AND_QUICKSIGHT"` | no | -| [quicksight\_subscription\_edition](#input\_quicksight\_subscription\_edition) | The edition for the QuickSight quicksight\_subscription | `string` | `"ENTERPRISE"` | no | -| [quicksight\_subscription\_email](#input\_quicksight\_subscription\_email) | The email address for the QuickSight quicksight\_subscription edition | `string` | `null` | no | -| [quicksight\_users](#input\_quicksight\_users) | Map of user accounts to be registered in QuickSight | 
map(object({
    identity_type = optional(string, "IAM")
    namespace     = optional(string, "default")
    role          = optional(string, "READER")
  }))
| `{}` | no | -| [quicksights\_username](#input\_quicksights\_username) | The username for the QuickSight user | `string` | `"admin"` | no | -| [saml\_metadata](#input\_saml\_metadata) | The configuration for the SAML identity provider | `string` | `null` | no | -| [stack\_name\_cloud\_intelligence](#input\_stack\_name\_cloud\_intelligence) | The name of the CloudFormation stack to create the dashboards | `string` | `"CI-Cloud-Intelligence-Dashboards"` | no | -| [stack\_name\_collectors](#input\_stack\_name\_collectors) | The name of the CloudFormation stack to create the collectors | `string` | `"CidDataCollectionStack"` | no | -| [stack\_name\_cora\_data\_exports\_destination](#input\_stack\_name\_cora\_data\_exports\_destination) | The name of the CloudFormation stack to create the CORA Data Exports | `string` | `"CidCoraCoraDataExportsDestinationStack"` | no | -| [stack\_name\_cora\_data\_exports\_source](#input\_stack\_name\_cora\_data\_exports\_source) | The name of the CloudFormation stack to create the CORA Data Exports | `string` | `"CidCoraCoraDataExportsSourceStack"` | no | -| [stack\_name\_read\_permissions](#input\_stack\_name\_read\_permissions) | The name of the CloudFormation stack to create the collectors | `string` | `"CidDataCollectionReadPermissionsStack"` | no | -| [stacks\_bucket\_name](#input\_stacks\_bucket\_name) | The name of the bucket to store the CloudFormation templates | `string` | `"cid-cloudformation-templates"` | no | +No inputs. ## Outputs @@ -152,4 +106,3 @@ To enable the Cora Data Exports, please see https://catalog.workshops.aws/awscid | [destination\_bucket\_website\_url](#output\_destination\_bucket\_website\_url) | The URL for the destination bucket | | [source\_account\_id](#output\_source\_account\_id) | The account ID of the source account i.e. the management account | - diff --git a/data.tf b/data.tf deleted file mode 100644 index 5ef96c2..0000000 --- a/data.tf +++ /dev/null @@ -1,19 +0,0 @@ - -## Find the current identity for the cost analysis session -data "aws_caller_identity" "cost_analysis" { - provider = aws.cost_analysis -} - -## Find the account id for the management account -data "aws_caller_identity" "management" { - provider = aws.management -} - -## Find the current organization -data "aws_organizations_organization" "current" { - provider = aws.management -} -## Find the current region -data "aws_region" "cost_analysis" { - provider = aws.cost_analysis -} diff --git a/appvia_banner.jpg b/docs/banner.jpg similarity index 100% rename from appvia_banner.jpg rename to docs/banner.jpg diff --git a/examples/basic/.terraform.lock.hcl b/examples/basic/.terraform.lock.hcl index 05881aa..81e5bac 100644 --- a/examples/basic/.terraform.lock.hcl +++ b/examples/basic/.terraform.lock.hcl @@ -2,47 +2,47 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/aws" { - version = "5.45.0" - constraints = ">= 5.0.0" + version = "5.74.0" + constraints = ">= 3.0.0, >= 5.0.0, ~> 5.0, >= 5.27.0" hashes = [ - "h1:8m3+C1VNevzU/8FsABoKp2rTOx3Ue7674INfhfk0TZY=", - "zh:1379bcf45aef3d486ee18b4f767bfecd40a0056510d26107f388be3d7994c368", - "zh:1615a6f5495acfb3a0cb72324587261dd4d72711a3cc51aff13167b14531501e", - "zh:18b69a0f33f8b1862fbd3f200756b7e83e087b73687085f2cf9c7da4c318e3e6", - "zh:2c5e7aecd197bc3d3b19290bad8cf4c390c2c6a77bb165da4e11f53f2dfe2e54", - "zh:3794da9bef97596e3bc60e12cdd915bda5ec2ed62cd1cd93723d58b4981905fe", - "zh:40a5e45ed91801f83db76dffd467dcf425ea2ca8642327cf01119601cb86021c", - "zh:4abfc3f53d0256a7d5d1fa5e931e4601b02db3d1da28f452341d3823d0518f1a", - "zh:4eb0e98078f79aeb06b5ff6115286dc2135d12a80287885698d04036425494a2", - "zh:75470efbadea4a8d783642497acaeec5077fc4a7f3df3340defeaa1c7de29bf7", - "zh:8861a0b4891d5fa2fa7142f236ae613cea966c45b5472e3915a4ac3abcbaf487", - "zh:8bf6f21cd9390b742ca0b4393fde92616ca9e6553fb75003a0999006ad233d35", + "h1:0Iq3x8RSdWedvATBO1RZbCQqRCHPNsdhkYVrRs9crEE=", + "zh:1e2d65add4d63af5b396ae33d55c48303eca6c86bd1be0f6fae13267a9b47bc4", + "zh:20ddec3dac3d06a188f12e58b6428854949b1295e937c5d4dca4866dc1c937af", + "zh:35b72de4e6a3e3d69efc07184fb413406262fe447b2d82d57eaf8c787a068a06", + "zh:44eada24a50cd869aadc4b29f9e791fdf262d7f426921e9ac2893bbb86013176", + "zh:455e666e3a9a2312b3b9f434b87a404b6515d64a8853751e20566a6548f9df9e", + "zh:58b3ae74abfca7b9b61f42f0c8b10d97f9b01aff18bd1d4ab091129c9d203707", + "zh:840a8a32d5923f9e7422f9c80d165c3f89bb6ea370b8283095081e39050a8ea8", + "zh:87cb6dbbdbc1b73bdde4b8b5d6d780914a3e8f1df0385da4ea7323dc1a68468f", + "zh:8b8953e39b0e6e6156c5570d1ca653450bfa0d9b280e2475f01ee5c51a6554db", "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:ad73008a044e75d337acda910fb54d8b81a366873c8a413fec1291034899a814", - "zh:bf261713b0b8bebfe8c199291365b87d9043849f28a2dc764bafdde73ae43693", - "zh:da3bafa1fd830be418dfcc730e85085fe67c0d415c066716f2ac350a2306f40a", + "zh:9bd750262e2fb0187a8420a561e55b0a1da738f690f53f5c7df170cb1f380459", + "zh:9d2474c1432dfa5e1db197e2dd6cd61a6a15452e0bc7acd09ca86b3cdb228871", + "zh:b763ecaf471c7737a5c6e4cf257b5318e922a6610fd83b36ed8eb68582a8642e", + "zh:c1344cd8fe03ff7433a19b14b14a1898c2ca5ba22a468fb8e1687f0a7f564d52", + "zh:dc0e0abf3be7402d0d022ced82816884356115ed27646df9c7222609e96840e6", ] } provider "registry.terraform.io/hashicorp/awscc" { - version = "0.74.0" + version = "1.19.0" constraints = ">= 0.11.0" hashes = [ - "h1:Ul1832evjT1Gu2DOH9sw4WxBbPOf7NHLJ3UfG/SAnqU=", - "zh:033b1bf5b8b0fa412aa7e09610d03f603b83b91ba78dfc26548c450b16bdaa6a", - "zh:1069cf96afe3e3b14f4934706523afeb07938f53a1e78567d2dcb4b336886ff2", - "zh:25561adfb2ea86c21db300ec16f2b34f80babe0b8290efb28074c56e64f889e5", - "zh:5136f6b60f8f64ea902ddd8694818d94d4508aa20b67284ee73053446bd3717d", - "zh:5b6d010847de9ecc6f3d05dd2c3771b25a44fb53cc81133acefe810b868514a7", - "zh:64426f51d1d96637c354e58a3629c3ba1286be2dbe8e8cce09d2f0aa3f08e064", - "zh:6addfc79b9fd7012741f0b966ad30729cea9966bf653cb7bf61592b8881d388e", - "zh:71b742c71699b25d1b51d4179a23564f295829568711b2137eb1c16df1867a75", - "zh:8638a9eef6e2dd9d4351236620d5217124f65eb319365cefd275ab2ed743bf07", - "zh:93ac5394c70aac348a4cf52e2d12187f89aca3c42769b0be3b6371fa3db6266c", - "zh:95a8c81b3c453db533fa476f0d03075a1032b76ea8d342637ee92d3b04254c95", - "zh:bc00447a578f533f690214e2be641af60ccd6f0a4f9b9b46e1abeac841205c88", - "zh:c63406c121f66b77ebd202dd673f324cb4dd6b5d8d11f68ef6dcc23d6ee8730d", - "zh:f3537bd1fa5e5fdee3a9a39016c0681d3f88b2105835aae980e9aada876efdd4", + "h1:RuObcNRltFowO6sYBca/yo+wQnqUyUv26mLW9nqM0Mw=", + "zh:080eac85cc5a7f626d2f45241741d0d9d6437b8402875c3a4d929b51eebbd00e", + "zh:1088a52b79cb97dc2457a35acf137a5730d44a3cf96e3bed5550f4ab0504c914", + "zh:44c1a24b7c7dd0a123ca4d8af72e4aeba8661d394c5a816445e16470552710f1", + "zh:4e4a88359cec750646cf121d6ffb83f1c8dc18b3f5a1e495e14d41b7dacf2dba", + "zh:4f3954ef869cde3e302ec34c2f29809cef18ea9e1228d2a236496937dd7267b8", + "zh:6b1f0960b004cdd24ee8de02c6dbea558a1ff395d8c5713747d98ce1751f11b2", + "zh:6c36ed71471ad94f4f0d2bb4f8934a3fab002caf4a717179551b5ad54f5759ad", + "zh:b3bf6ad70a2fe10e26d64113e085dcd27da2262b97283572d2882c3f087e2bae", + "zh:b6d68048bebd2598c4051fa7f5661d1a12b0d65c233c3597df1d61dc857f9aa7", + "zh:c77c2178dd1cd35dfe591ad5630c8eed8b1e60d71a6199c88e40f5175e657cb9", + "zh:d09eba5889a97553a8047cb44ceb352d9780658a363ffc4bd693b665627209e5", + "zh:ddad8b1f75708adbe9b50701d3ab5bbc305f80dd2e59193488ae89d9be3e57f9", + "zh:eb683c5091a171d93741e0b82ad487fd962af21df456f22a304dec0a2b39b5d2", "zh:f809ab383cca0a5f83072981c64208cbd7fa67e986a86ee02dd2c82333221e32", + "zh:fd808721ef4d60a0b03f1ae548eb2f40d6395acbe5b317089017cc1990e22ac7", ] } diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 4a6dcaf..8974268 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -4,25 +4,50 @@ # to build your own root module that invokes this module ##################################################################################### -module "cudos_framework" { - source = "../.." - - dashboards_bucket_name = var.dashboard_bucket_name - enable_compute_optimizer_dashboard = true - enable_cost_intelligence_dashboard = true - enable_cudos_dashboard = true - enable_cudos_v5_dashboard = true - enable_kpi_dashboard = true - enable_sso = true - enable_tao_dashboard = false - saml_metadata = file("${path.module}/assets/saml-metadata.xml") - quicksights_username = var.quicksights_username - tags = var.tags +locals { + ## Name of the bucket where the cloudformation scripts are stored + cloudformation_bucket_name = "cid-cloudformation-templates" + ## The external URL for the cloudformation bucket + cloudformation_bucket_url = format("https://%s.s3.%s.amazonaws.com", local.cloudformation_bucket_name, "eu-west-2") +} + +module "destination" { + source = "../../modules/destination" + + cloudformation_bucket_url = local.cloudformation_bucket_url + dashboards_bucket_name = var.dashboard_bucket_name + enable_sso = true + management_account_id = module.source.management_account_id + quicksights_username = var.quicksights_username + saml_metadata = file("${path.module}/assets/saml-metadata.xml") + tags = var.tags + + providers = { + aws = aws.cost_analysis + aws.us_east_1 = aws.cost_analysis_us_east_1 + } +} + +module "source" { + source = "../../modules/source" + + ## The account id for the destination below + destination_account_id = "1234343434" + destination_bucket_name = module.destination.destination_bucket_name + enable_backup_module = true + enable_budgets_module = true + enable_cora_data_exports = true + enable_ecs_chargeback_module = true + enable_health_events_module = true + enable_inventory_module = true + enable_rds_utilization_module = true + enable_scad = true + stacks_bucket_name = local.cloudformation_bucket_name + tags = var.tags providers = { - aws.management = aws.management - aws.management_us_east_1 = aws.management_us_east_1 - aws.cost_analysis = aws.cost_analysis - aws.cost_analysis_us_east_1 = aws.cost_analysis_us_east_1 + aws = aws.management + aws.us_east_1 = aws.management_us_east_1 } } + diff --git a/locals.tf b/locals.tf deleted file mode 100644 index 27e2413..0000000 --- a/locals.tf +++ /dev/null @@ -1,31 +0,0 @@ - -locals { - ## The region where the stack is being deployed - region = data.aws_region.cost_analysis.name - - ## Is the account id for the cost analysis account - cost_analysis_account_id = data.aws_caller_identity.cost_analysis.account_id - ## Is the management account id - management_account_id = data.aws_caller_identity.management.account_id - ## Is the organization root id - organization_root_id = data.aws_organizations_organization.current.roots[0].id - ## The s3 bucket name for the cloudformation scripts - stacks_base_url = format("https://%s.s3.%s.amazonaws.com", var.stacks_bucket_name, local.region) - ## Is the user mappings for the quicksight groups - user_group_mappings = merge([ - for n, g in var.quicksight_groups : { - for u in g.members : - join("-", [n, u]) => { - user = u - group = n - } - } if var.enable_sso - ]...) - - ## Is the payer account id used in the collection configuration - payer_account_ids = distinct(sort(concat([local.management_account_id], var.additional_payer_accounts))) - - ## Is the list of accounts permitted to retrieve the cloudformation templates - cloudformation_accounts_ids = distinct(concat([local.management_account_id, local.cost_analysis_account_id], var.additional_payer_accounts)) -} - diff --git a/modules/destination/.terraform.lock.hcl b/modules/destination/.terraform.lock.hcl new file mode 100644 index 0000000..78d0674 --- /dev/null +++ b/modules/destination/.terraform.lock.hcl @@ -0,0 +1,25 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.74.0" + constraints = ">= 3.0.0, ~> 5.0, >= 5.27.0" + hashes = [ + "h1:0Iq3x8RSdWedvATBO1RZbCQqRCHPNsdhkYVrRs9crEE=", + "zh:1e2d65add4d63af5b396ae33d55c48303eca6c86bd1be0f6fae13267a9b47bc4", + "zh:20ddec3dac3d06a188f12e58b6428854949b1295e937c5d4dca4866dc1c937af", + "zh:35b72de4e6a3e3d69efc07184fb413406262fe447b2d82d57eaf8c787a068a06", + "zh:44eada24a50cd869aadc4b29f9e791fdf262d7f426921e9ac2893bbb86013176", + "zh:455e666e3a9a2312b3b9f434b87a404b6515d64a8853751e20566a6548f9df9e", + "zh:58b3ae74abfca7b9b61f42f0c8b10d97f9b01aff18bd1d4ab091129c9d203707", + "zh:840a8a32d5923f9e7422f9c80d165c3f89bb6ea370b8283095081e39050a8ea8", + "zh:87cb6dbbdbc1b73bdde4b8b5d6d780914a3e8f1df0385da4ea7323dc1a68468f", + "zh:8b8953e39b0e6e6156c5570d1ca653450bfa0d9b280e2475f01ee5c51a6554db", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9bd750262e2fb0187a8420a561e55b0a1da738f690f53f5c7df170cb1f380459", + "zh:9d2474c1432dfa5e1db197e2dd6cd61a6a15452e0bc7acd09ca86b3cdb228871", + "zh:b763ecaf471c7737a5c6e4cf257b5318e922a6610fd83b36ed8eb68582a8642e", + "zh:c1344cd8fe03ff7433a19b14b14a1898c2ca5ba22a468fb8e1687f0a7f564d52", + "zh:dc0e0abf3be7402d0d022ced82816884356115ed27646df9c7222609e96840e6", + ] +} diff --git a/modules/destination/README.md b/modules/destination/README.md new file mode 100644 index 0000000..553ecf9 --- /dev/null +++ b/modules/destination/README.md @@ -0,0 +1,60 @@ + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | ~> 5.0 | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [cloudformation\_bucket\_url](#input\_cloudformation\_bucket\_url) | The name of the bucket to store the CloudFormation templates | `string` | n/a | yes | +| [dashboards\_bucket\_name](#input\_dashboards\_bucket\_name) | The name of the bucket to store the dashboards configurations | `string` | n/a | yes | +| [management\_account\_id](#input\_management\_account\_id) | The AWS account ID for the management account | `string` | n/a | yes | +| [tags](#input\_tags) | Tags to apply to all resources | `map(string)` | n/a | yes | +| [additional\_payer\_accounts](#input\_additional\_payer\_accounts) | List of additional payer accounts to be included in the collectors module | `list(string)` | `[]` | no | +| [enable\_backup\_module](#input\_enable\_backup\_module) | Indicates if the Backup module should be enabled | `bool` | `true` | no | +| [enable\_budgets\_module](#input\_enable\_budgets\_module) | Indicates if the Budget module should be enabled | `bool` | `true` | no | +| [enable\_compute\_optimizer\_dashboard](#input\_enable\_compute\_optimizer\_dashboard) | Indicates if the Compute Optimizer dashboard should be enabled | `bool` | `true` | no | +| [enable\_compute\_optimizer\_module](#input\_enable\_compute\_optimizer\_module) | Indicates if the Compute Optimizer module should be enabled | `bool` | `true` | no | +| [enable\_cora\_data\_exports](#input\_enable\_cora\_data\_exports) | Indicates if the CORA Data Exports module should be enabled | `bool` | `false` | no | +| [enable\_cost\_anomaly\_module](#input\_enable\_cost\_anomaly\_module) | Indicates if the Cost Anomaly module should be enabled | `bool` | `true` | no | +| [enable\_cost\_intelligence\_dashboard](#input\_enable\_cost\_intelligence\_dashboard) | Indicates if the Cost Intelligence dashboard should be enabled | `bool` | `true` | no | +| [enable\_cudos\_dashboard](#input\_enable\_cudos\_dashboard) | Indicates if the CUDOS dashboard should be enabled | `bool` | `false` | no | +| [enable\_cudos\_v5\_dashboard](#input\_enable\_cudos\_v5\_dashboard) | Indicates if the CUDOS V5 framework should be enabled | `bool` | `true` | no | +| [enable\_ecs\_chargeback\_module](#input\_enable\_ecs\_chargeback\_module) | Indicates if the ECS Chargeback module should be enabled | `bool` | `false` | no | +| [enable\_health\_events](#input\_enable\_health\_events) | Indicates if the Health Events module should be enabled | `bool` | `true` | no | +| [enable\_inventory\_module](#input\_enable\_inventory\_module) | Indicates if the Inventory module should be enabled | `bool` | `true` | no | +| [enable\_kpi\_dashboard](#input\_enable\_kpi\_dashboard) | Indicates if the KPI dashboard should be enabled | `bool` | `true` | no | +| [enable\_license\_manager\_module](#input\_enable\_license\_manager\_module) | Indicates if the License Manager module should be enabled | `bool` | `false` | no | +| [enable\_org\_data\_module](#input\_enable\_org\_data\_module) | Indicates if the Organization Data module should be enabled | `bool` | `true` | no | +| [enable\_prerequisites\_quicksight](#input\_enable\_prerequisites\_quicksight) | Indicates if the prerequisites for QuickSight should be enabled | `bool` | `true` | no | +| [enable\_prerequisites\_quicksight\_permissions](#input\_enable\_prerequisites\_quicksight\_permissions) | Indicates if the prerequisites for QuickSight permissions should be enabled | `bool` | `true` | no | +| [enable\_quicksight\_subscription](#input\_enable\_quicksight\_subscription) | Enable QuickSight subscription | `bool` | `false` | no | +| [enable\_rds\_utilization\_module](#input\_enable\_rds\_utilization\_module) | Indicates if the RDS Utilization module should be enabled | `bool` | `true` | no | +| [enable\_rightsizing\_module](#input\_enable\_rightsizing\_module) | Indicates if the Rightsizing module should be enabled | `bool` | `true` | no | +| [enable\_scad](#input\_enable\_scad) | Indicates if the SCAD module should be enabled, only available when Cora enabled | `bool` | `false` | no | +| [enable\_sso](#input\_enable\_sso) | Enable integration with identity center for QuickSight | `bool` | `true` | no | +| [enable\_tao\_dashboard](#input\_enable\_tao\_dashboard) | Indicates if the TAO dashboard should be enabled | `bool` | `false` | no | +| [enable\_tao\_module](#input\_enable\_tao\_module) | Indicates if the TAO module should be enabled | `bool` | `true` | no | +| [enable\_transit\_gateway\_module](#input\_enable\_transit\_gateway\_module) | Indicates if the Transit Gateway module should be enabled | `bool` | `true` | no | +| [quicksight\_groups](#input\_quicksight\_groups) | Map of groups with user membership to be added to QuickSight | 
map(object({
    description = optional(string)
    namespace   = optional(string)
    members     = optional(list(string), [])
  }))
| `{}` | no | +| [quicksight\_subscription\_account\_name](#input\_quicksight\_subscription\_account\_name) | The account name for the QuickSight quicksight\_subscription edition | `string` | `null` | no | +| [quicksight\_subscription\_authentication\_method](#input\_quicksight\_subscription\_authentication\_method) | The identity for the QuickSight quicksight\_subscription edition | `string` | `"IAM_AND_QUICKSIGHT"` | no | +| [quicksight\_subscription\_edition](#input\_quicksight\_subscription\_edition) | The edition for the QuickSight quicksight\_subscription | `string` | `"ENTERPRISE"` | no | +| [quicksight\_subscription\_email](#input\_quicksight\_subscription\_email) | The email address for the QuickSight quicksight\_subscription edition | `string` | `null` | no | +| [quicksight\_users](#input\_quicksight\_users) | Map of user accounts to be registered in QuickSight | 
map(object({
    identity_type = optional(string, "IAM")
    namespace     = optional(string, "default")
    role          = optional(string, "READER")
  }))
| `{}` | no | +| [quicksights\_username](#input\_quicksights\_username) | The username for the QuickSight user | `string` | `"admin"` | no | +| [saml\_metadata](#input\_saml\_metadata) | The configuration for the SAML identity provider | `string` | `null` | no | +| [stack\_name\_cloud\_intelligence](#input\_stack\_name\_cloud\_intelligence) | The name of the CloudFormation stack to create the dashboards | `string` | `"CI-Cloud-Intelligence-Dashboards"` | no | +| [stack\_name\_collectors](#input\_stack\_name\_collectors) | The name of the CloudFormation stack to create the collectors | `string` | `"CidDataCollectionStack"` | no | +| [stack\_name\_cora\_data\_exports\_destination](#input\_stack\_name\_cora\_data\_exports\_destination) | The name of the CloudFormation stack to create the CORA Data Exports | `string` | `"CidCoraCoraDataExportsDestinationStack"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [destination\_bucket\_arn](#output\_destination\_bucket\_arn) | The name of the bucket where to replicate the data from the CUR | +| [destination\_bucket\_name](#output\_destination\_bucket\_name) | The name of the bucket where to replicate the data from the CUR | + \ No newline at end of file diff --git a/modules/destination/data.tf b/modules/destination/data.tf new file mode 100644 index 0000000..aa834e1 --- /dev/null +++ b/modules/destination/data.tf @@ -0,0 +1,3 @@ + +## Find the current identity for the cost analysis session +data "aws_caller_identity" "current" {} diff --git a/modules/destination/locals.tf b/modules/destination/locals.tf new file mode 100644 index 0000000..ac892bb --- /dev/null +++ b/modules/destination/locals.tf @@ -0,0 +1,19 @@ + +locals { + ## Is the account id for the cost analysis account + account_id = data.aws_caller_identity.current.account_id + ## Is the payer account id used in the collection configuration + payer_account_ids = distinct(sort(concat([var.management_account_id], var.additional_payer_accounts))) + + ## Is the user mappings for the quicksight groups + user_group_mappings = merge([ + for n, g in var.quicksight_groups : { + for u in g.members : + join("-", [n, u]) => { + user = u + group = n + } + } if var.enable_sso + ]...) +} + diff --git a/main.tf b/modules/destination/main.tf similarity index 51% rename from main.tf rename to modules/destination/main.tf index 8486f60..fc5e05c 100644 --- a/main.tf +++ b/modules/destination/main.tf @@ -7,8 +7,6 @@ resource "aws_quicksight_account_subscription" "subscription" { authentication_method = var.quicksight_subscription_authentication_method edition = var.quicksight_subscription_edition notification_email = var.quicksight_subscription_email - - provider = aws.cost_analysis } ## Provision a SAML identity provider in the data collection account - this will be @@ -19,8 +17,6 @@ resource "aws_iam_saml_provider" "saml" { name = "aws-cudos-sso" saml_metadata_document = var.saml_metadata tags = var.tags - - provider = aws.cost_analysis } ## Provision a trust policy for the above SAML identity provider @@ -65,53 +61,15 @@ resource "aws_iam_role" "cudos_sso" { name = "aws-cudos-sso" assume_role_policy = data.aws_iam_policy_document.cudos_sso[0].json tags = var.tags - - inline_policy { - name = "quicksight-permissions" - policy = data.aws_iam_policy_document.cudos_sso_permissions[0].json - } - - provider = aws.cost_analysis } -## Craft and IAM policy that allows the account to access the bucket -data "aws_iam_policy_document" "stack_bucket_policy" { - statement { - effect = "Allow" - actions = [ - "s3:DeleteObject", - "s3:GetObject", - "s3:ListBucket", - "s3:PutObject", - ] - principals { - type = "AWS" - identifiers = [local.management_account_id] - } - resources = [ - format("arn:aws:s3:::%s", var.stacks_bucket_name), - format("arn:aws:s3:::%s/*", var.stacks_bucket_name), - ] - } +## Attach an inline policy to the IAM role +resource "aws_iam_role_policy" "cudos_sso" { + count = var.enable_sso ? 1 : 0 - statement { - effect = "Allow" - actions = [ - "s3:GetObject", - "s3:ListBucket", - ] - principals { - type = "AWS" - identifiers = [ - local.cost_analysis_account_id, - local.management_account_id, - ] - } - resources = [ - format("arn:aws:s3:::%s", var.stacks_bucket_name), - format("arn:aws:s3:::%s/*", var.stacks_bucket_name), - ] - } + name = "quicksight-permissions" + policy = data.aws_iam_policy_document.cudos_sso_permissions[0].json + role = aws_iam_role.cudos_sso[0].name } ## Craft and IAM policy that allows the account to access the bucket @@ -126,7 +84,7 @@ data "aws_iam_policy_document" "dashboards_bucket_policy" { ] principals { type = "AWS" - identifiers = local.cloudformation_accounts_ids + identifiers = [local.account_id] } resources = [ format("arn:aws:s3:::%s", var.dashboards_bucket_name), @@ -136,54 +94,6 @@ data "aws_iam_policy_document" "dashboards_bucket_policy" { } -## Provision a bucket used to contain the cloudformation templates -# tfsec:ignore:aws-s3-enable-bucket-logging -module "cloudformation_bucket" { - source = "terraform-aws-modules/s3-bucket/aws" - version = "4.1.2" - - attach_policy = true - block_public_acls = true - block_public_policy = true - bucket = var.stacks_bucket_name - expected_bucket_owner = local.management_account_id - force_destroy = true - ignore_public_acls = true - object_ownership = "BucketOwnerPreferred" - policy = data.aws_iam_policy_document.stack_bucket_policy.json - restrict_public_buckets = true - tags = var.tags - - server_side_encryption_configuration = { - rule = { - apply_server_side_encryption_by_default = { - sse_algorithm = "AES256" - } - } - } - - versioning = { - enabled = true - } - - providers = { - aws = aws.management - } -} - -## Upload the cloudformation templates to the bucket -resource "aws_s3_object" "cloudformation_templates" { - for_each = fileset("${path.module}/assets/cloudformation/", "**/*.yaml") - - bucket = module.cloudformation_bucket.s3_bucket_id - etag = filemd5("${path.module}/assets/cloudformation/${each.value}") - key = each.value - server_side_encryption = "AES256" - source = "${path.module}/assets/cloudformation/${each.value}" - - provider = aws.management -} - ## Provision a bucket used to contain the cudos dashboards - note this ## bucket must be public due to the consuming tterraform module # @@ -196,7 +106,7 @@ module "dashboard_bucket" { block_public_acls = true block_public_policy = true bucket = var.dashboards_bucket_name - expected_bucket_owner = local.cost_analysis_account_id + expected_bucket_owner = local.account_id force_destroy = true ignore_public_acls = true object_ownership = "BucketOwnerPreferred" @@ -215,16 +125,12 @@ module "dashboard_bucket" { versioning = { enabled = true } - - providers = { - aws = aws.cost_analysis - } } ## First we configure the collector to accept the CUR (Cost and Usage Report) from the source account # tfsec:ignore:aws-s3-enable-bucket-logging module "collector" { - source = "github.com/aws-samples/aws-cudos-framework-deployment//terraform-modules/cur-setup-destination?ref=4.0.5" + source = "github.com/aws-samples/aws-cudos-framework-deployment//terraform-modules/cur-setup-destination?ref=4.0.2" # Source account whom will be replicating the CUR data to the collector account source_account_ids = local.payer_account_ids @@ -232,55 +138,10 @@ module "collector" { create_cur = false providers = { - aws = aws.cost_analysis - aws.useast1 = aws.cost_analysis_us_east_1 - } -} - -## Setup the replication from the management account to the collector account -## to receive the CUR data -# tfsec:ignore:aws-s3-enable-bucket-logging -# tfsec:ignore:aws-iam-no-policy-wildcards -module "source" { - source = "github.com/aws-samples/aws-cudos-framework-deployment//terraform-modules/cur-setup-source?ref=4.0.5" - - # The destination bucket to repliaction the CUR data to - destination_bucket_arn = module.collector.cur_bucket_arn - - providers = { - aws = aws.management - aws.useast1 = aws.management_us_east_1 + aws.useast1 = aws.us_east_1 } } -## Provision the stack contain the cora data exports in the management account -## Deployment of same stacko the management account -resource "aws_cloudformation_stack" "core_data_export_management" { - count = var.enable_cora_data_exports ? 1 : 0 - - capabilities = ["CAPABILITY_NAMED_IAM", "CAPABILITY_AUTO_EXPAND"] - name = var.stack_name_cora_data_exports_source - on_failure = "ROLLBACK" - tags = var.tags - template_url = format("%s/cudos/%s", local.stacks_base_url, "data-exports-aggregation.yaml") - - parameters = { - "DestinationAccountId" = local.cost_analysis_account_id, - "EnableSCAD" = var.enable_scad ? "yes" : "no", - "ManageCOH" = "yes", - "ManageCUR2" = "no", - "SourceAccountIds" = local.management_account_id, - } - - lifecycle { - ignore_changes = [ - capabilities, - ] - } - - provider = aws.management -} - ## Provision the Cora data exports in the collector account resource "aws_cloudformation_stack" "cora_data_export_collector" { count = var.enable_cora_data_exports ? 1 : 0 @@ -289,14 +150,14 @@ resource "aws_cloudformation_stack" "cora_data_export_collector" { name = var.stack_name_cora_data_exports_destination on_failure = "ROLLBACK" tags = var.tags - template_url = format("%s/cudos/%s", local.stacks_base_url, "data-exports-aggregation.yaml") + template_url = format("%s/cudos/%s", var.cloudformation_bucket_url, "data-exports-aggregation.yaml") parameters = { - "DestinationAccountId" = local.cost_analysis_account_id, + "DestinationAccountId" = local.account_id, "EnableSCAD" = var.enable_scad ? "yes" : "no", "ManageCOH" = "yes", "ManageCUR2" = "no", - "SourceAccountIds" = local.management_account_id, + "SourceAccountIds" = local.account_id, } lifecycle { @@ -304,13 +165,11 @@ resource "aws_cloudformation_stack" "cora_data_export_collector" { capabilities, ] } - - provider = aws.cost_analysis } ## Provision the cloud intelligence dashboards module "dashboards" { - source = "github.com/aws-samples/aws-cudos-framework-deployment//terraform-modules/cid-dashboards?ref=4.0.5" + source = "github.com/aws-samples/aws-cudos-framework-deployment//terraform-modules/cid-dashboards?ref=4.0.2" stack_name = var.stack_name_cloud_intelligence template_bucket = module.dashboard_bucket.s3_bucket_id @@ -329,53 +188,15 @@ module "dashboards" { depends_on = [ module.collector, - module.source, aws_quicksight_account_subscription.subscription, ] - - providers = { - aws = aws.cost_analysis - } -} - -## We need to provision the read permissions stack in the management account -resource "aws_cloudformation_stack" "cudos_read_permissions" { - name = var.stack_name_read_permissions - capabilities = ["CAPABILITY_NAMED_IAM", "CAPABILITY_AUTO_EXPAND"] - template_url = format("%s/cudos/%s", local.stacks_base_url, "deploy-data-read-permissions.yaml") - - parameters = { - "AllowModuleReadInMgmt" = "yes", - "DataCollectionAccountID" = local.cost_analysis_account_id, - "IncludeBackupModule" = var.enable_backup_module ? "yes" : "no", - "IncludeBudgetsModule" = var.enable_budgets_module ? "yes" : "no", - "IncludeComputeOptimizerModule" = var.enable_compute_optimizer_module ? "yes" : "no", - "IncludeCostAnomalyModule" = var.enable_cost_anomaly_module ? "yes" : "no", - "IncludeECSChargebackModule" = var.enable_ecs_chargeback_module ? "yes" : "no", - "IncludeHealthEventsModule" = var.enable_health_events ? "yes" : "no" - "IncludeInventoryCollectorModule" = var.enable_inventory_module ? "yes" : "no", - "IncludeRDSUtilizationModule" = var.enable_rds_utilization_module ? "yes" : "no", - "IncludeRightsizingModule" = var.enable_rightsizing_module ? "yes" : "no", - "IncludeTAModule" = var.enable_tao_module ? "yes" : "no", - "IncludeTransitGatewayModule" = var.enable_transit_gateway_module ? "yes" : "no", - "OrganizationalUnitIds" = local.organization_root_id, - } - - depends_on = [ - aws_s3_object.cloudformation_templates, - module.collector, - module.dashboards, - module.source, - ] - - provider = aws.management } ## We need to provision the data collection stack in the colletor account resource "aws_cloudformation_stack" "cudos_data_collection" { name = var.stack_name_collectors capabilities = ["CAPABILITY_NAMED_IAM", "CAPABILITY_AUTO_EXPAND"] - template_url = format("%s/cudos/%s", local.stacks_base_url, "deploy-data-collection.yaml") + template_url = format("%s/cudos/%s", var.cloudformation_bucket_url, "deploy-data-collection.yaml") parameters = { "IncludeBackupModule" = var.enable_backup_module ? "yes" : "no", @@ -391,16 +212,11 @@ resource "aws_cloudformation_stack" "cudos_data_collection" { "IncludeRightsizingModule" = var.enable_rightsizing_module ? "yes" : "no", "IncludeTAModule" = var.enable_tao_module ? "yes" : "no", "IncludeTransitGatewayModule" = var.enable_transit_gateway_module ? "yes" : "no", - "ManagementAccountID" = join(",", local.payer_account_ids), + "ManagementAccountID" = local.management_account_id, } depends_on = [ - aws_cloudformation_stack.cudos_read_permissions, - aws_s3_object.cloudformation_templates, module.collector, module.dashboards, - module.source, ] - - provider = aws.cost_analysis } diff --git a/modules/destination/outputs.tf b/modules/destination/outputs.tf new file mode 100644 index 0000000..157a2db --- /dev/null +++ b/modules/destination/outputs.tf @@ -0,0 +1,11 @@ + +output "destination_bucket_arn" { + description = "The name of the bucket where to replicate the data from the CUR" + value = module.collector.cur_bucket_arn +} + +output "destination_bucket_name" { + description = "The name of the bucket where to replicate the data from the CUR" + value = module.collector.cur_bucket_name +} + diff --git a/quicksights.tf b/modules/destination/quicksights.tf similarity index 92% rename from quicksights.tf rename to modules/destination/quicksights.tf index 4db3077..7b9ef22 100644 --- a/quicksights.tf +++ b/modules/destination/quicksights.tf @@ -6,8 +6,6 @@ resource "aws_quicksight_group" "groups" { description = each.value.description group_name = each.key namespace = each.value.namespace - - provider = aws.cost_analysis } ## Provision any users within QuickSight @@ -25,8 +23,6 @@ resource "aws_quicksight_user" "users" { lifecycle { ignore_changes = [user_name] } - - provider = aws.cost_analysis } ## Provision any group memberships within QuickSight @@ -37,6 +33,4 @@ resource "aws_quicksight_group_membership" "members" { member_name = format("%s/%s", aws_iam_role.cudos_sso[0].name, each.value.user) depends_on = [aws_quicksight_user.users] - - provider = aws.cost_analysis } diff --git a/terraform.tf b/modules/destination/terraform.tf similarity index 60% rename from terraform.tf rename to modules/destination/terraform.tf index 4586017..ac73a40 100644 --- a/terraform.tf +++ b/modules/destination/terraform.tf @@ -6,10 +6,7 @@ terraform { source = "hashicorp/aws" version = "~> 5.0" configuration_aliases = [ - aws.cost_analysis, - aws.cost_analysis_us_east_1, - aws.management, - aws.management_us_east_1, + aws.us_east_1, ] } } diff --git a/variables.tf b/modules/destination/variables.tf similarity index 93% rename from variables.tf rename to modules/destination/variables.tf index 1d558fe..43b192e 100644 --- a/variables.tf +++ b/modules/destination/variables.tf @@ -4,6 +4,16 @@ variable "tags" { type = map(string) } +variable "management_account_id" { + description = "The AWS account ID for the management account" + type = string +} + +variable "cloudformation_bucket_url" { + description = "The name of the bucket to store the CloudFormation templates" + type = string +} + variable "additional_payer_accounts" { description = "List of additional payer accounts to be included in the collectors module" type = list(string) @@ -40,36 +50,18 @@ variable "quicksight_subscription_account_name" { default = null } -variable "stacks_bucket_name" { - description = "The name of the bucket to store the CloudFormation templates" - type = string - default = "cid-cloudformation-templates" -} - variable "stack_name_cloud_intelligence" { description = "The name of the CloudFormation stack to create the dashboards" type = string default = "CI-Cloud-Intelligence-Dashboards" } -variable "stack_name_read_permissions" { - description = "The name of the CloudFormation stack to create the collectors" - type = string - default = "CidDataCollectionReadPermissionsStack" -} - variable "stack_name_collectors" { description = "The name of the CloudFormation stack to create the collectors" type = string default = "CidDataCollectionStack" } -variable "stack_name_cora_data_exports_source" { - description = "The name of the CloudFormation stack to create the CORA Data Exports" - type = string - default = "CidCoraCoraDataExportsSourceStack" -} - variable "stack_name_cora_data_exports_destination" { description = "The name of the CloudFormation stack to create the CORA Data Exports" type = string diff --git a/modules/source/.terraform.lock.hcl b/modules/source/.terraform.lock.hcl new file mode 100644 index 0000000..78d0674 --- /dev/null +++ b/modules/source/.terraform.lock.hcl @@ -0,0 +1,25 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.74.0" + constraints = ">= 3.0.0, ~> 5.0, >= 5.27.0" + hashes = [ + "h1:0Iq3x8RSdWedvATBO1RZbCQqRCHPNsdhkYVrRs9crEE=", + "zh:1e2d65add4d63af5b396ae33d55c48303eca6c86bd1be0f6fae13267a9b47bc4", + "zh:20ddec3dac3d06a188f12e58b6428854949b1295e937c5d4dca4866dc1c937af", + "zh:35b72de4e6a3e3d69efc07184fb413406262fe447b2d82d57eaf8c787a068a06", + "zh:44eada24a50cd869aadc4b29f9e791fdf262d7f426921e9ac2893bbb86013176", + "zh:455e666e3a9a2312b3b9f434b87a404b6515d64a8853751e20566a6548f9df9e", + "zh:58b3ae74abfca7b9b61f42f0c8b10d97f9b01aff18bd1d4ab091129c9d203707", + "zh:840a8a32d5923f9e7422f9c80d165c3f89bb6ea370b8283095081e39050a8ea8", + "zh:87cb6dbbdbc1b73bdde4b8b5d6d780914a3e8f1df0385da4ea7323dc1a68468f", + "zh:8b8953e39b0e6e6156c5570d1ca653450bfa0d9b280e2475f01ee5c51a6554db", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9bd750262e2fb0187a8420a561e55b0a1da738f690f53f5c7df170cb1f380459", + "zh:9d2474c1432dfa5e1db197e2dd6cd61a6a15452e0bc7acd09ca86b3cdb228871", + "zh:b763ecaf471c7737a5c6e4cf257b5318e922a6610fd83b36ed8eb68582a8642e", + "zh:c1344cd8fe03ff7433a19b14b14a1898c2ca5ba22a468fb8e1687f0a7f564d52", + "zh:dc0e0abf3be7402d0d022ced82816884356115ed27646df9c7222609e96840e6", + ] +} diff --git a/modules/source/README.md b/modules/source/README.md new file mode 100644 index 0000000..b1fbe2c --- /dev/null +++ b/modules/source/README.md @@ -0,0 +1,39 @@ + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | ~> 5.0 | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [destination\_account\_id](#input\_destination\_account\_id) | The AWS account ID for the destination account | `string` | n/a | yes | +| [destination\_bucket\_name](#input\_destination\_bucket\_name) | The name of the bucket where to replicate the data from the CUR | `string` | n/a | yes | +| [tags](#input\_tags) | Tags to apply to all resources | `map(string)` | n/a | yes | +| [enable\_backup\_module](#input\_enable\_backup\_module) | Indicates if the Backup module should be enabled | `bool` | `true` | no | +| [enable\_budgets\_module](#input\_enable\_budgets\_module) | Indicates if the Budget module should be enabled | `bool` | `true` | no | +| [enable\_compute\_optimizer\_module](#input\_enable\_compute\_optimizer\_module) | Indicates if the Compute Optimizer module should be enabled | `bool` | `true` | no | +| [enable\_cora\_data\_exports](#input\_enable\_cora\_data\_exports) | Indicates if the CORA Data Exports module should be enabled | `bool` | `false` | no | +| [enable\_cost\_anomaly\_module](#input\_enable\_cost\_anomaly\_module) | Indicates if the Cost Anomaly module should be enabled | `bool` | `true` | no | +| [enable\_ecs\_chargeback\_module](#input\_enable\_ecs\_chargeback\_module) | Indicates if the ECS Chargeback module should be enabled | `bool` | `false` | no | +| [enable\_health\_events\_module](#input\_enable\_health\_events\_module) | Indicates if the Health Events module should be enabled | `bool` | `true` | no | +| [enable\_inventory\_module](#input\_enable\_inventory\_module) | Indicates if the Inventory module should be enabled | `bool` | `true` | no | +| [enable\_rds\_utilization\_module](#input\_enable\_rds\_utilization\_module) | Indicates if the RDS Utilization module should be enabled | `bool` | `true` | no | +| [enable\_rightsizing\_module](#input\_enable\_rightsizing\_module) | Indicates if the Rightsizing module should be enabled | `bool` | `true` | no | +| [enable\_scad](#input\_enable\_scad) | Indicates if the SCAD module should be enabled, only available when Cora enabled | `bool` | `false` | no | +| [enable\_tao\_module](#input\_enable\_tao\_module) | Indicates if the TAO module should be enabled | `bool` | `true` | no | +| [enable\_transit\_gateway\_module](#input\_enable\_transit\_gateway\_module) | Indicates if the Transit Gateway module should be enabled | `bool` | `true` | no | +| [stack\_name\_cora\_data\_exports\_source](#input\_stack\_name\_cora\_data\_exports\_source) | The name of the CloudFormation stack to create the CORA Data Exports | `string` | `"CidCoraCoraDataExportsSourceStack"` | no | +| [stack\_name\_read\_permissions](#input\_stack\_name\_read\_permissions) | The name of the CloudFormation stack to create the collectors | `string` | `"CidDataCollectionReadPermissionsStack"` | no | +| [stacks\_bucket\_name](#input\_stacks\_bucket\_name) | The name of the bucket to store the CloudFormation templates | `string` | `"cid-cloudformation-templates"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [cloudformation\_bucket\_name](#output\_cloudformation\_bucket\_name) | The name of the bucket to store the CloudFormation templates | +| [cloudformation\_bucket\_url](#output\_cloudformation\_bucket\_url) | The URL of the bucket to store the CloudFormation templates | +| [management\_account\_id](#output\_management\_account\_id) | The AWS account ID for the management account | + \ No newline at end of file diff --git a/assets/cloudformation/cudos/data-exports-aggregation.yaml b/modules/source/assets/cloudformation/cudos/data-exports-aggregation.yaml similarity index 100% rename from assets/cloudformation/cudos/data-exports-aggregation.yaml rename to modules/source/assets/cloudformation/cudos/data-exports-aggregation.yaml diff --git a/assets/cloudformation/cudos/deploy-data-collection.yaml b/modules/source/assets/cloudformation/cudos/deploy-data-collection.yaml similarity index 100% rename from assets/cloudformation/cudos/deploy-data-collection.yaml rename to modules/source/assets/cloudformation/cudos/deploy-data-collection.yaml diff --git a/assets/cloudformation/cudos/deploy-data-read-permissions.yaml b/modules/source/assets/cloudformation/cudos/deploy-data-read-permissions.yaml similarity index 100% rename from assets/cloudformation/cudos/deploy-data-read-permissions.yaml rename to modules/source/assets/cloudformation/cudos/deploy-data-read-permissions.yaml diff --git a/modules/source/data.tf b/modules/source/data.tf new file mode 100644 index 0000000..40ca1e9 --- /dev/null +++ b/modules/source/data.tf @@ -0,0 +1,9 @@ + +## Find the account id for the management account +data "aws_caller_identity" "current" {} + +## Find the current region +data "aws_region" "current" {} + +## Find the current organization +data "aws_organizations_organization" "current" {} diff --git a/modules/source/locals.tf b/modules/source/locals.tf new file mode 100644 index 0000000..cb8b415 --- /dev/null +++ b/modules/source/locals.tf @@ -0,0 +1,14 @@ + +locals { + ## The region where the stack is being deployed + region = data.aws_region.current.name + ## Is the management account id + management_account_id = data.aws_caller_identity.current.account_id + ## Is the organization root id + organization_root_id = data.aws_organizations_organization.current.roots[0].id + ## The s3 bucket name for the cloudformation scripts + stacks_base_url = format("https://%s.s3.%s.amazonaws.com", var.stacks_bucket_name, local.region) + ## The account id where the dashboard is being deployed + destination_account_id = var.destination_account_id +} + diff --git a/modules/source/main.tf b/modules/source/main.tf new file mode 100644 index 0000000..79af722 --- /dev/null +++ b/modules/source/main.tf @@ -0,0 +1,152 @@ + +## Craft and IAM policy that allows the account to access the bucket +data "aws_iam_policy_document" "stack_bucket_policy" { + statement { + effect = "Allow" + actions = [ + "s3:DeleteObject", + "s3:GetObject", + "s3:ListBucket", + "s3:PutObject", + ] + principals { + type = "AWS" + identifiers = [local.management_account_id] + } + resources = [ + format("arn:aws:s3:::%s", var.stacks_bucket_name), + format("arn:aws:s3:::%s/*", var.stacks_bucket_name), + ] + } + + statement { + effect = "Allow" + actions = [ + "s3:GetObject", + "s3:ListBucket", + ] + principals { + type = "AWS" + identifiers = [ + local.destination_account_id, + local.management_account_id, + ] + } + resources = [ + format("arn:aws:s3:::%s", var.stacks_bucket_name), + format("arn:aws:s3:::%s/*", var.stacks_bucket_name), + ] + } +} + +## Provision a bucket used to contain the cloudformation templates +# tfsec:ignore:aws-s3-enable-bucket-logging +module "cloudformation_bucket" { + source = "terraform-aws-modules/s3-bucket/aws" + version = "4.1.2" + + attach_policy = true + block_public_acls = true + block_public_policy = true + bucket = var.stacks_bucket_name + expected_bucket_owner = local.management_account_id + force_destroy = true + ignore_public_acls = true + object_ownership = "BucketOwnerPreferred" + policy = data.aws_iam_policy_document.stack_bucket_policy.json + restrict_public_buckets = true + tags = var.tags + + server_side_encryption_configuration = { + rule = { + apply_server_side_encryption_by_default = { + sse_algorithm = "AES256" + } + } + } + + versioning = { + enabled = true + } +} + +## Upload the cloudformation templates to the bucket +resource "aws_s3_object" "cloudformation_templates" { + for_each = fileset("${path.module}/assets/cloudformation/", "**/*.yaml") + + bucket = module.cloudformation_bucket.s3_bucket_id + etag = filemd5("${path.module}/assets/cloudformation/${each.value}") + key = each.value + server_side_encryption = "AES256" + source = "${path.module}/assets/cloudformation/${each.value}" +} + +## Setup the replication from the management account to the collector account +## to receive the CUR data +# tfsec:ignore:aws-s3-enable-bucket-logging +# tfsec:ignore:aws-iam-no-policy-wildcards +module "source" { + source = "github.com/aws-samples/aws-cudos-framework-deployment//terraform-modules/cur-setup-source?ref=0.3.13" + + # The destination bucket to repliaction the CUR data to + destination_bucket_arn = format("arn:aws:s3:%s: %s:bucket/%s", local.region, local.destination_account_id, var.destination_bucket_name) + + providers = { + aws.useast1 = aws.us_east_1 + } +} + +## Provision the stack contain the cora data exports in the management account +## Deployment of same stack the management account +resource "aws_cloudformation_stack" "core_data_export_management" { + count = var.enable_cora_data_exports ? 1 : 0 + + capabilities = ["CAPABILITY_NAMED_IAM", "CAPABILITY_AUTO_EXPAND"] + name = var.stack_name_cora_data_exports_source + on_failure = "ROLLBACK" + tags = var.tags + template_url = format("%s/cudos/%s", local.stacks_base_url, "data-exports-aggregation.yaml") + + parameters = { + "DestinationAccountId" = local.destination_account_id, + "EnableSCAD" = var.enable_scad ? "yes" : "no", + "ManageCOH" = "yes", + "ManageCUR2" = "no", + "SourceAccountIds" = local.management_account_id, + } + + lifecycle { + ignore_changes = [ + capabilities, + ] + } +} + +## We need to provision the read permissions stack in the management account +resource "aws_cloudformation_stack" "cudos_read_permissions" { + name = var.stack_name_read_permissions + capabilities = ["CAPABILITY_NAMED_IAM", "CAPABILITY_AUTO_EXPAND"] + template_url = format("%s/cudos/%s", local.stacks_base_url, "deploy-data-read-permissions.yaml") + + parameters = { + "AllowModuleReadInMgmt" = "yes", + "DataCollectionAccountID" = local.destination_account_id, + "IncludeBackupModule" = var.enable_backup_module ? "yes" : "no", + "IncludeBudgetsModule" = var.enable_budgets_module ? "yes" : "no", + "IncludeComputeOptimizerModule" = var.enable_compute_optimizer_module ? "yes" : "no", + "IncludeCostAnomalyModule" = var.enable_cost_anomaly_module ? "yes" : "no", + "IncludeECSChargebackModule" = var.enable_ecs_chargeback_module ? "yes" : "no", + "IncludeHealthEventsModule" = var.enable_health_events_module ? "yes" : "no" + "IncludeInventoryCollectorModule" = var.enable_inventory_module ? "yes" : "no", + "IncludeRDSUtilizationModule" = var.enable_rds_utilization_module ? "yes" : "no", + "IncludeRightsizingModule" = var.enable_rightsizing_module ? "yes" : "no", + "IncludeTAModule" = var.enable_tao_module ? "yes" : "no", + "IncludeTransitGatewayModule" = var.enable_transit_gateway_module ? "yes" : "no", + "OrganizationalUnitIds" = local.organization_root_id, + } + + depends_on = [ + aws_s3_object.cloudformation_templates, + module.source, + ] +} diff --git a/modules/source/outputs.tf b/modules/source/outputs.tf new file mode 100644 index 0000000..e69f5ec --- /dev/null +++ b/modules/source/outputs.tf @@ -0,0 +1,17 @@ + +output "management_account_id" { + description = "The AWS account ID for the management account" + value = local.management_account_id +} + +output "cloudformation_bucket_name" { + description = "The name of the bucket to store the CloudFormation templates" + value = var.stacks_bucket_name +} + +output "cloudformation_bucket_url" { + description = "The URL of the bucket to store the CloudFormation templates" + value = format("https://%s.s3.%s.amazonaws.com", var.stacks_bucket_name, local.region) +} + + diff --git a/modules/source/terraform.tf b/modules/source/terraform.tf new file mode 100644 index 0000000..ac73a40 --- /dev/null +++ b/modules/source/terraform.tf @@ -0,0 +1,13 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + configuration_aliases = [ + aws.us_east_1, + ] + } + } +} diff --git a/modules/source/variables.tf b/modules/source/variables.tf new file mode 100644 index 0000000..6fc456a --- /dev/null +++ b/modules/source/variables.tf @@ -0,0 +1,111 @@ + +variable "tags" { + description = "Tags to apply to all resources" + type = map(string) +} + +variable "destination_account_id" { + description = "The AWS account ID for the destination account" + type = string +} + +variable "destination_bucket_name" { + description = "The name of the bucket where to replicate the data from the CUR" + type = string +} + +variable "stacks_bucket_name" { + description = "The name of the bucket to store the CloudFormation templates" + type = string + default = "cid-cloudformation-templates" +} + +variable "stack_name_read_permissions" { + description = "The name of the CloudFormation stack to create the collectors" + type = string + default = "CidDataCollectionReadPermissionsStack" +} + +variable "stack_name_cora_data_exports_source" { + description = "The name of the CloudFormation stack to create the CORA Data Exports" + type = string + default = "CidCoraCoraDataExportsSourceStack" +} + +variable "enable_cost_anomaly_module" { + description = "Indicates if the Cost Anomaly module should be enabled" + type = bool + default = true +} + +variable "enable_scad" { + description = "Indicates if the SCAD module should be enabled, only available when Cora enabled" + type = bool + default = false +} + +variable "enable_health_events_module" { + description = "Indicates if the Health Events module should be enabled" + type = bool + default = true +} + +variable "enable_backup_module" { + description = "Indicates if the Backup module should be enabled" + type = bool + default = true +} + +variable "enable_budgets_module" { + description = "Indicates if the Budget module should be enabled" + type = bool + default = true +} + +variable "enable_ecs_chargeback_module" { + description = "Indicates if the ECS Chargeback module should be enabled" + type = bool + default = false +} + +variable "enable_compute_optimizer_module" { + description = "Indicates if the Compute Optimizer module should be enabled" + type = bool + default = true +} + +variable "enable_tao_module" { + description = "Indicates if the TAO module should be enabled" + type = bool + default = true +} + +variable "enable_transit_gateway_module" { + description = "Indicates if the Transit Gateway module should be enabled" + type = bool + default = true +} + +variable "enable_inventory_module" { + description = "Indicates if the Inventory module should be enabled" + type = bool + default = true +} + +variable "enable_rds_utilization_module" { + description = "Indicates if the RDS Utilization module should be enabled" + type = bool + default = true +} + +variable "enable_cora_data_exports" { + description = "Indicates if the CORA Data Exports module should be enabled" + type = bool + default = false +} + +variable "enable_rightsizing_module" { + description = "Indicates if the Rightsizing module should be enabled" + type = bool + default = true +} diff --git a/outputs.tf b/outputs.tf deleted file mode 100644 index 76c1851..0000000 --- a/outputs.tf +++ /dev/null @@ -1,50 +0,0 @@ - -output "destination_bucket_name" { - description = "The name of the destination bucket" - value = module.collector.cur_bucket_name -} - -output "destination_bucket_arn" { - description = "The ARN of the destination bucket" - value = module.collector.cur_bucket_arn -} - -output "destination_bucket_short_url" { - description = "The domain name of the destination bucket" - value = format("s3://%s", module.cloudformation_bucket.s3_bucket_id) -} - -output "destination_bucket_website_url" { - description = "The URL for the destination bucket" - value = format("https://%s.amazonaws.com", module.cloudformation_bucket.s3_bucket_id) -} - -output "destination_account_id" { - description = "The account ID of the destination bucket" - value = local.cost_analysis_account_id -} - -output "source_account_id" { - description = "The account ID of the source account i.e. the management account" - value = local.management_account_id -} - -output "cloudformation_bucket_name" { - description = "The name of the bucket to store the CloudFormation templates" - value = var.stacks_bucket_name -} - -output "cloudformation_bucket_arn" { - description = "The ARN of the bucket to store the CloudFormation templates" - value = format("arn:aws:s3:::%s", var.stacks_bucket_name) -} - -output "cloudformation_bucket_short_url" { - description = "The domain name of the bucket to store the CloudFormation templates" - value = format("s3://%s", var.stacks_bucket_name) -} - -output "cloudformation_bucket_website_url" { - description = "The URL for the bucket to store the CloudFormation templates" - value = format("https://%s.amazonaws.com", var.stacks_bucket_name) -} From f9b2386a0671d9b97a113901be64cabef087b832 Mon Sep 17 00:00:00 2001 From: Rohith Jayawardene Date: Thu, 14 Nov 2024 13:11:27 +0000 Subject: [PATCH 07/19] chore: adding the updated makefile --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 2a42925..286e99b 100644 --- a/Makefile +++ b/Makefile @@ -70,6 +70,10 @@ upgrade-terraform-example-providers: init: @echo "--> Running terraform init" @terraform init -backend=false + @find . -type f -name "*.tf" -not -path '*.terraform*' -exec dirname {} \; | sort -u | while read -r dir; do \ + echo "--> Running terraform init in $$dir"; \ + terraform -chdir=$$dir init -backend=false; \ + done; security: init @echo "--> Running Security checks" From 45c2883dc0a631a95499b084c5d4488a1c892b76 Mon Sep 17 00:00:00 2001 From: Rohith Jayawardene Date: Mon, 18 Nov 2024 08:27:38 +0000 Subject: [PATCH 08/19] fix: switching to using the local.account_id --- modules/destination/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/destination/main.tf b/modules/destination/main.tf index fc5e05c..8603ac5 100644 --- a/modules/destination/main.tf +++ b/modules/destination/main.tf @@ -49,7 +49,7 @@ data "aws_iam_policy_document" "cudos_sso_permissions" { statement { actions = ["quicksight:CreateReader"] effect = "Allow" - resources = ["arn:aws:quicksight::${local.cost_analysis_account_id}:user/$${aws:userid}"] + resources = ["arn:aws:quicksight::${local.account_id}:user/$${aws:userid}"] } } From c9e42e69a5382c8635d920d2fe7c28c515046fb7 Mon Sep 17 00:00:00 2001 From: Rohith Jayawardene Date: Mon, 18 Nov 2024 10:32:29 +0000 Subject: [PATCH 09/19] fix: using the bucket arn, not the name for the replication --- examples/basic/main.tf | 2 +- modules/source/README.md | 2 +- modules/source/main.tf | 2 +- modules/source/variables.tf | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 8974268..e623ec8 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -33,7 +33,7 @@ module "source" { ## The account id for the destination below destination_account_id = "1234343434" - destination_bucket_name = module.destination.destination_bucket_name + destination_bucket_name = module.destination.destination_bucket_arn enable_backup_module = true enable_budgets_module = true enable_cora_data_exports = true diff --git a/modules/source/README.md b/modules/source/README.md index b1fbe2c..ff7bad6 100644 --- a/modules/source/README.md +++ b/modules/source/README.md @@ -10,7 +10,7 @@ | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [destination\_account\_id](#input\_destination\_account\_id) | The AWS account ID for the destination account | `string` | n/a | yes | -| [destination\_bucket\_name](#input\_destination\_bucket\_name) | The name of the bucket where to replicate the data from the CUR | `string` | n/a | yes | +| [destination\_bucket\_arn](#input\_destination\_bucket\_arn) | The ARN of the bucket where to replicate the data from the CUR | `string` | n/a | yes | | [tags](#input\_tags) | Tags to apply to all resources | `map(string)` | n/a | yes | | [enable\_backup\_module](#input\_enable\_backup\_module) | Indicates if the Backup module should be enabled | `bool` | `true` | no | | [enable\_budgets\_module](#input\_enable\_budgets\_module) | Indicates if the Budget module should be enabled | `bool` | `true` | no | diff --git a/modules/source/main.tf b/modules/source/main.tf index 79af722..6fbc581 100644 --- a/modules/source/main.tf +++ b/modules/source/main.tf @@ -89,7 +89,7 @@ module "source" { source = "github.com/aws-samples/aws-cudos-framework-deployment//terraform-modules/cur-setup-source?ref=0.3.13" # The destination bucket to repliaction the CUR data to - destination_bucket_arn = format("arn:aws:s3:%s: %s:bucket/%s", local.region, local.destination_account_id, var.destination_bucket_name) + destination_bucket_arn = var.destination_bucket_arn providers = { aws.useast1 = aws.us_east_1 diff --git a/modules/source/variables.tf b/modules/source/variables.tf index 6fc456a..0c9ff8f 100644 --- a/modules/source/variables.tf +++ b/modules/source/variables.tf @@ -9,8 +9,8 @@ variable "destination_account_id" { type = string } -variable "destination_bucket_name" { - description = "The name of the bucket where to replicate the data from the CUR" +variable "destination_bucket_arn" { + description = "The ARN of the bucket where to replicate the data from the CUR" type = string } From 5b399b9fe8c39f6d92536ed558ada24863e1c6e5 Mon Sep 17 00:00:00 2001 From: Rohith Jayawardene Date: Mon, 18 Nov 2024 14:48:27 +0000 Subject: [PATCH 10/19] chore: adding the changes --- examples/basic/README.md | 1 - examples/basic/main.tf | 20 +-- examples/basic/variables.tf | 6 - modules/destination/README.md | 4 +- .../deploy-data-collection.yaml | 0 modules/destination/data.tf | 3 + modules/destination/locals.tf | 4 + modules/destination/main.tf | 123 ++++++++++++++---- modules/destination/outputs.tf | 9 ++ modules/destination/variables.tf | 4 +- modules/source/main.tf | 7 +- 11 files changed, 128 insertions(+), 53 deletions(-) rename modules/{source/assets/cloudformation/cudos => destination/assets/cloudformation}/deploy-data-collection.yaml (100%) diff --git a/examples/basic/README.md b/examples/basic/README.md index 9e67613..ea159e2 100644 --- a/examples/basic/README.md +++ b/examples/basic/README.md @@ -7,7 +7,6 @@ No providers. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [dashboard\_bucket\_name](#input\_dashboard\_bucket\_name) | The name of the bucket to store the dashboards | `string` | `"dashboard-bucket-dev"` | no | | [quicksights\_username](#input\_quicksights\_username) | The username to use for QuickSight | `string` | `"admin"` | no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | 
{
  "Environment": "Production"
}
| no | diff --git a/examples/basic/main.tf b/examples/basic/main.tf index e623ec8..756d178 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -7,20 +7,20 @@ locals { ## Name of the bucket where the cloudformation scripts are stored cloudformation_bucket_name = "cid-cloudformation-templates" - ## The external URL for the cloudformation bucket - cloudformation_bucket_url = format("https://%s.s3.%s.amazonaws.com", local.cloudformation_bucket_name, "eu-west-2") + ## Name of the bucket where the dashboards are stored + dashboard_bucket_name = "cid-dashboards" } module "destination" { source = "../../modules/destination" - cloudformation_bucket_url = local.cloudformation_bucket_url - dashboards_bucket_name = var.dashboard_bucket_name - enable_sso = true - management_account_id = module.source.management_account_id - quicksights_username = var.quicksights_username - saml_metadata = file("${path.module}/assets/saml-metadata.xml") - tags = var.tags + cloudformation_bucket_name = local.cloudformation_bucket_name + dashboards_bucket_name = local.dashboard_bucket_name + enable_sso = true + management_account_id = module.source.management_account_id + quicksights_username = var.quicksights_username + saml_metadata = file("${path.module}/assets/saml-metadata.xml") + tags = var.tags providers = { aws = aws.cost_analysis @@ -33,7 +33,7 @@ module "source" { ## The account id for the destination below destination_account_id = "1234343434" - destination_bucket_name = module.destination.destination_bucket_arn + destination_bucket_arn = module.destination.destination_bucket_arn enable_backup_module = true enable_budgets_module = true enable_cora_data_exports = true diff --git a/examples/basic/variables.tf b/examples/basic/variables.tf index 4f06a0a..6800124 100644 --- a/examples/basic/variables.tf +++ b/examples/basic/variables.tf @@ -1,10 +1,4 @@ -variable "dashboard_bucket_name" { - description = "The name of the bucket to store the dashboards" - type = string - default = "dashboard-bucket-dev" -} - variable "tags" { description = "A map of tags to add to all resources" type = map(string) diff --git a/modules/destination/README.md b/modules/destination/README.md index 553ecf9..1484914 100644 --- a/modules/destination/README.md +++ b/modules/destination/README.md @@ -9,7 +9,7 @@ | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [cloudformation\_bucket\_url](#input\_cloudformation\_bucket\_url) | The name of the bucket to store the CloudFormation templates | `string` | n/a | yes | +| [cloudformation\_bucket\_name](#input\_cloudformation\_bucket\_name) | The name of the bucket to store the CloudFormation | `string` | n/a | yes | | [dashboards\_bucket\_name](#input\_dashboards\_bucket\_name) | The name of the bucket to store the dashboards configurations | `string` | n/a | yes | | [management\_account\_id](#input\_management\_account\_id) | The AWS account ID for the management account | `string` | n/a | yes | | [tags](#input\_tags) | Tags to apply to all resources | `map(string)` | n/a | yes | @@ -55,6 +55,8 @@ | Name | Description | |------|-------------| +| [cloudformation\_bucket\_arn](#output\_cloudformation\_bucket\_arn) | The name of the bucket where to store the CloudFormation | +| [dashboard\_bucket\_arn](#output\_dashboard\_bucket\_arn) | The name of the bucket where to store the dashboards | | [destination\_bucket\_arn](#output\_destination\_bucket\_arn) | The name of the bucket where to replicate the data from the CUR | | [destination\_bucket\_name](#output\_destination\_bucket\_name) | The name of the bucket where to replicate the data from the CUR | \ No newline at end of file diff --git a/modules/source/assets/cloudformation/cudos/deploy-data-collection.yaml b/modules/destination/assets/cloudformation/deploy-data-collection.yaml similarity index 100% rename from modules/source/assets/cloudformation/cudos/deploy-data-collection.yaml rename to modules/destination/assets/cloudformation/deploy-data-collection.yaml diff --git a/modules/destination/data.tf b/modules/destination/data.tf index aa834e1..ad9ac66 100644 --- a/modules/destination/data.tf +++ b/modules/destination/data.tf @@ -1,3 +1,6 @@ ## Find the current identity for the cost analysis session data "aws_caller_identity" "current" {} + +## Find the current region +data "aws_region" "current" {} diff --git a/modules/destination/locals.tf b/modules/destination/locals.tf index ac892bb..8a29c27 100644 --- a/modules/destination/locals.tf +++ b/modules/destination/locals.tf @@ -4,6 +4,10 @@ locals { account_id = data.aws_caller_identity.current.account_id ## Is the payer account id used in the collection configuration payer_account_ids = distinct(sort(concat([var.management_account_id], var.additional_payer_accounts))) + ## The region where the stack is being deployed + region = data.aws_region.current.name + ## The URL for the s3 bucket containing cloudformation scripts + bucket_url = format("https://%s.s3.%s.amazonaws.com", var.cloudformation_bucket_name, local.region) ## Is the user mappings for the quicksight groups user_group_mappings = merge([ diff --git a/modules/destination/main.tf b/modules/destination/main.tf index 8603ac5..5148a5e 100644 --- a/modules/destination/main.tf +++ b/modules/destination/main.tf @@ -1,4 +1,97 @@ +## Craft and IAM policy that allows the account to access the bucket +data "aws_iam_policy_document" "bucket_policy" { + statement { + effect = "Allow" + actions = [ + "s3:DeleteObject", + "s3:GetObject", + "s3:ListBucket", + "s3:PutObject", + ] + principals { + type = "AWS" + identifiers = [local.account_id] + } + resources = [ + format("arn:aws:s3:::%s", var.cloudformation_bucket_name), + format("arn:aws:s3:::%s/*", var.cloudformation_bucket_name), + ] + } + + statement { + effect = "Allow" + actions = [ + "s3:GetObject", + "s3:ListBucket", + ] + principals { + type = "AWS" + identifiers = [local.account_id] + } + resources = [ + format("arn:aws:s3:::%s", var.cloudformation_bucket_name), + format("arn:aws:s3:::%s/*", var.cloudformation_bucket_name), + ] + } +} + +## Provision a bucket used to contain the cloudformation templates +# tfsec:ignore:aws-s3-enable-bucket-logging +module "cloudformation" { + source = "terraform-aws-modules/s3-bucket/aws" + version = "4.2.2" + + attach_policy = true + block_public_acls = true + block_public_policy = true + bucket = var.cloudformation_bucket_name + expected_bucket_owner = local.account_id + force_destroy = true + ignore_public_acls = true + object_ownership = "BucketOwnerPreferred" + policy = data.aws_iam_policy_document.bucket_policy.json + restrict_public_buckets = true + tags = var.tags + + server_side_encryption_configuration = { + rule = { + apply_server_side_encryption_by_default = { + sse_algorithm = "AES256" + } + } + } + + versioning = { + enabled = true + } +} + +## Create a lifecycle rule to old versions of the objects in the bucket +resource "aws_s3_bucket_lifecycle_configuration" "bucket_lifecycle" { + bucket = module.cloudformation.s3_bucket_id + + rule { + id = "DeleteOldVersions" + status = "Enabled" + + noncurrent_version_expiration { + noncurrent_days = 90 + } + } +} + +## Upload the cloudformation templates to the bucket +resource "aws_s3_object" "cloudformation_templates" { + for_each = fileset("${path.module}/assets/cloudformation/", "**/*.yaml") + + bucket = module.cloudformation.s3_bucket_id + etag = filemd5("${path.module}/assets/cloudformation/${each.value}") + key = each.value + server_side_encryption = "AES256" + source = "${path.module}/assets/cloudformation/${each.value}" +} + ## Provision enterprise quicksight if enabled resource "aws_quicksight_account_subscription" "subscription" { count = var.enable_quicksight_subscription ? 1 : 0 @@ -93,14 +186,13 @@ data "aws_iam_policy_document" "dashboards_bucket_policy" { } } - ## Provision a bucket used to contain the cudos dashboards - note this ## bucket must be public due to the consuming tterraform module # # tfsec:ignore:aws-s3-enable-bucket-logging module "dashboard_bucket" { source = "terraform-aws-modules/s3-bucket/aws" - version = "4.1.2" + version = "4.2.2" attach_policy = true block_public_acls = true @@ -142,31 +234,6 @@ module "collector" { } } -## Provision the Cora data exports in the collector account -resource "aws_cloudformation_stack" "cora_data_export_collector" { - count = var.enable_cora_data_exports ? 1 : 0 - - capabilities = ["CAPABILITY_NAMED_IAM", "CAPABILITY_AUTO_EXPAND"] - name = var.stack_name_cora_data_exports_destination - on_failure = "ROLLBACK" - tags = var.tags - template_url = format("%s/cudos/%s", var.cloudformation_bucket_url, "data-exports-aggregation.yaml") - - parameters = { - "DestinationAccountId" = local.account_id, - "EnableSCAD" = var.enable_scad ? "yes" : "no", - "ManageCOH" = "yes", - "ManageCUR2" = "no", - "SourceAccountIds" = local.account_id, - } - - lifecycle { - ignore_changes = [ - capabilities, - ] - } -} - ## Provision the cloud intelligence dashboards module "dashboards" { source = "github.com/aws-samples/aws-cudos-framework-deployment//terraform-modules/cid-dashboards?ref=4.0.2" @@ -196,7 +263,7 @@ module "dashboards" { resource "aws_cloudformation_stack" "cudos_data_collection" { name = var.stack_name_collectors capabilities = ["CAPABILITY_NAMED_IAM", "CAPABILITY_AUTO_EXPAND"] - template_url = format("%s/cudos/%s", var.cloudformation_bucket_url, "deploy-data-collection.yaml") + template_url = format("%s/cudos/%s", local.bucket_url, "deploy-data-collection.yaml") parameters = { "IncludeBackupModule" = var.enable_backup_module ? "yes" : "no", diff --git a/modules/destination/outputs.tf b/modules/destination/outputs.tf index 157a2db..355a770 100644 --- a/modules/destination/outputs.tf +++ b/modules/destination/outputs.tf @@ -1,4 +1,9 @@ +output "cloudformation_bucket_arn" { + description = "The name of the bucket where to store the CloudFormation" + value = module.cloudformation.s3_bucket_arn +} + output "destination_bucket_arn" { description = "The name of the bucket where to replicate the data from the CUR" value = module.collector.cur_bucket_arn @@ -9,3 +14,7 @@ output "destination_bucket_name" { value = module.collector.cur_bucket_name } +output "dashboard_bucket_arn" { + description = "The name of the bucket where to store the dashboards" + value = module.dashboard_bucket.s3_bucket_arn +} diff --git a/modules/destination/variables.tf b/modules/destination/variables.tf index 43b192e..011f945 100644 --- a/modules/destination/variables.tf +++ b/modules/destination/variables.tf @@ -9,8 +9,8 @@ variable "management_account_id" { type = string } -variable "cloudformation_bucket_url" { - description = "The name of the bucket to store the CloudFormation templates" +variable "cloudformation_bucket_name" { + description = "The name of the bucket to store the CloudFormation" type = string } diff --git a/modules/source/main.tf b/modules/source/main.tf index 6fbc581..1553046 100644 --- a/modules/source/main.tf +++ b/modules/source/main.tf @@ -26,11 +26,8 @@ data "aws_iam_policy_document" "stack_bucket_policy" { "s3:ListBucket", ] principals { - type = "AWS" - identifiers = [ - local.destination_account_id, - local.management_account_id, - ] + type = "AWS" + identifiers = [local.management_account_id] } resources = [ format("arn:aws:s3:::%s", var.stacks_bucket_name), From aaffce525ff46339930752e9181695a3859adb98 Mon Sep 17 00:00:00 2001 From: Rohith Jayawardene Date: Mon, 18 Nov 2024 15:19:11 +0000 Subject: [PATCH 11/19] chore: fixing up the module --- modules/destination/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/destination/main.tf b/modules/destination/main.tf index 5148a5e..cd6a21a 100644 --- a/modules/destination/main.tf +++ b/modules/destination/main.tf @@ -279,7 +279,7 @@ resource "aws_cloudformation_stack" "cudos_data_collection" { "IncludeRightsizingModule" = var.enable_rightsizing_module ? "yes" : "no", "IncludeTAModule" = var.enable_tao_module ? "yes" : "no", "IncludeTransitGatewayModule" = var.enable_transit_gateway_module ? "yes" : "no", - "ManagementAccountID" = local.management_account_id, + "ManagementAccountID" = local.account_id, } depends_on = [ From 2cc9570caa96fb12598e9490074c1a9871a29a75 Mon Sep 17 00:00:00 2001 From: Rohith Jayawardene Date: Mon, 18 Nov 2024 16:59:12 +0000 Subject: [PATCH 12/19] docs: updating the docs to reflect the changes --- README.md | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/README.md b/README.md index 65bac84..a283f30 100644 --- a/README.md +++ b/README.md @@ -93,16 +93,5 @@ No inputs. ## Outputs -| Name | Description | -|------|-------------| -| [cloudformation\_bucket\_arn](#output\_cloudformation\_bucket\_arn) | The ARN of the bucket to store the CloudFormation templates | -| [cloudformation\_bucket\_name](#output\_cloudformation\_bucket\_name) | The name of the bucket to store the CloudFormation templates | -| [cloudformation\_bucket\_short\_url](#output\_cloudformation\_bucket\_short\_url) | The domain name of the bucket to store the CloudFormation templates | -| [cloudformation\_bucket\_website\_url](#output\_cloudformation\_bucket\_website\_url) | The URL for the bucket to store the CloudFormation templates | -| [destination\_account\_id](#output\_destination\_account\_id) | The account ID of the destination bucket | -| [destination\_bucket\_arn](#output\_destination\_bucket\_arn) | The ARN of the destination bucket | -| [destination\_bucket\_name](#output\_destination\_bucket\_name) | The name of the destination bucket | -| [destination\_bucket\_short\_url](#output\_destination\_bucket\_short\_url) | The domain name of the destination bucket | -| [destination\_bucket\_website\_url](#output\_destination\_bucket\_website\_url) | The URL for the destination bucket | -| [source\_account\_id](#output\_source\_account\_id) | The account ID of the source account i.e. the management account | +No outputs. From 3bf3cf05c1b28477ba640dbd04726d60ed3331e3 Mon Sep 17 00:00:00 2001 From: Rohith Jayawardene Date: Tue, 19 Nov 2024 08:48:05 +0000 Subject: [PATCH 13/19] chore: working on the module transition --- examples/basic/main.tf | 2 +- modules/destination/README.md | 3 +-- modules/destination/locals.tf | 2 +- modules/destination/main.tf | 2 +- modules/destination/variables.tf | 7 +------ 5 files changed, 5 insertions(+), 11 deletions(-) diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 756d178..650af6b 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -17,7 +17,7 @@ module "destination" { cloudformation_bucket_name = local.cloudformation_bucket_name dashboards_bucket_name = local.dashboard_bucket_name enable_sso = true - management_account_id = module.source.management_account_id + payer_accounts = ["1234343434"] quicksights_username = var.quicksights_username saml_metadata = file("${path.module}/assets/saml-metadata.xml") tags = var.tags diff --git a/modules/destination/README.md b/modules/destination/README.md index 1484914..3621c0a 100644 --- a/modules/destination/README.md +++ b/modules/destination/README.md @@ -11,9 +11,7 @@ |------|-------------|------|---------|:--------:| | [cloudformation\_bucket\_name](#input\_cloudformation\_bucket\_name) | The name of the bucket to store the CloudFormation | `string` | n/a | yes | | [dashboards\_bucket\_name](#input\_dashboards\_bucket\_name) | The name of the bucket to store the dashboards configurations | `string` | n/a | yes | -| [management\_account\_id](#input\_management\_account\_id) | The AWS account ID for the management account | `string` | n/a | yes | | [tags](#input\_tags) | Tags to apply to all resources | `map(string)` | n/a | yes | -| [additional\_payer\_accounts](#input\_additional\_payer\_accounts) | List of additional payer accounts to be included in the collectors module | `list(string)` | `[]` | no | | [enable\_backup\_module](#input\_enable\_backup\_module) | Indicates if the Backup module should be enabled | `bool` | `true` | no | | [enable\_budgets\_module](#input\_enable\_budgets\_module) | Indicates if the Budget module should be enabled | `bool` | `true` | no | | [enable\_compute\_optimizer\_dashboard](#input\_enable\_compute\_optimizer\_dashboard) | Indicates if the Compute Optimizer dashboard should be enabled | `bool` | `true` | no | @@ -39,6 +37,7 @@ | [enable\_tao\_dashboard](#input\_enable\_tao\_dashboard) | Indicates if the TAO dashboard should be enabled | `bool` | `false` | no | | [enable\_tao\_module](#input\_enable\_tao\_module) | Indicates if the TAO module should be enabled | `bool` | `true` | no | | [enable\_transit\_gateway\_module](#input\_enable\_transit\_gateway\_module) | Indicates if the Transit Gateway module should be enabled | `bool` | `true` | no | +| [payer\_accounts](#input\_payer\_accounts) | List of additional payer accounts to be included in the collectors module | `list(string)` | `[]` | no | | [quicksight\_groups](#input\_quicksight\_groups) | Map of groups with user membership to be added to QuickSight | 
map(object({
    description = optional(string)
    namespace   = optional(string)
    members     = optional(list(string), [])
  }))
| `{}` | no | | [quicksight\_subscription\_account\_name](#input\_quicksight\_subscription\_account\_name) | The account name for the QuickSight quicksight\_subscription edition | `string` | `null` | no | | [quicksight\_subscription\_authentication\_method](#input\_quicksight\_subscription\_authentication\_method) | The identity for the QuickSight quicksight\_subscription edition | `string` | `"IAM_AND_QUICKSIGHT"` | no | diff --git a/modules/destination/locals.tf b/modules/destination/locals.tf index 8a29c27..715aaa7 100644 --- a/modules/destination/locals.tf +++ b/modules/destination/locals.tf @@ -3,7 +3,7 @@ locals { ## Is the account id for the cost analysis account account_id = data.aws_caller_identity.current.account_id ## Is the payer account id used in the collection configuration - payer_account_ids = distinct(sort(concat([var.management_account_id], var.additional_payer_accounts))) + payer_account_ids = distinct(var.payer_accounts) ## The region where the stack is being deployed region = data.aws_region.current.name ## The URL for the s3 bucket containing cloudformation scripts diff --git a/modules/destination/main.tf b/modules/destination/main.tf index cd6a21a..15e286c 100644 --- a/modules/destination/main.tf +++ b/modules/destination/main.tf @@ -279,7 +279,7 @@ resource "aws_cloudformation_stack" "cudos_data_collection" { "IncludeRightsizingModule" = var.enable_rightsizing_module ? "yes" : "no", "IncludeTAModule" = var.enable_tao_module ? "yes" : "no", "IncludeTransitGatewayModule" = var.enable_transit_gateway_module ? "yes" : "no", - "ManagementAccountID" = local.account_id, + "ManagementAccountID" = join(",", local.payer_account_ids), } depends_on = [ diff --git a/modules/destination/variables.tf b/modules/destination/variables.tf index 011f945..a49b837 100644 --- a/modules/destination/variables.tf +++ b/modules/destination/variables.tf @@ -4,17 +4,12 @@ variable "tags" { type = map(string) } -variable "management_account_id" { - description = "The AWS account ID for the management account" - type = string -} - variable "cloudformation_bucket_name" { description = "The name of the bucket to store the CloudFormation" type = string } -variable "additional_payer_accounts" { +variable "payer_accounts" { description = "List of additional payer accounts to be included in the collectors module" type = list(string) default = [] From bc3abcb853ed5c45cad234c3a22479b08e9d2844 Mon Sep 17 00:00:00 2001 From: Rohith Jayawardene Date: Wed, 27 Nov 2024 10:57:58 +0000 Subject: [PATCH 14/19] feat/v2 (#92) * feat: adding back the cora export to the data account, as this is required and providing a toggle the data read permissions * chore: removing the read permission variable and using the lack of organizational id --- .../cudos/data-exports-aggregation.yaml | 1564 ++++++++++++++++ modules/destination/README.md | 2 +- .../data-exports-aggregation.yaml | 1 + modules/destination/main.tf | 57 +- modules/destination/variables.tf | 2 +- modules/source/README.md | 1 + .../cudos/data-exports-aggregation.yaml | 1565 +---------------- modules/source/data.tf | 7 +- modules/source/locals.tf | 8 +- modules/source/main.tf | 18 +- modules/source/variables.tf | 6 + 11 files changed, 1633 insertions(+), 1598 deletions(-) create mode 100644 assets/cloudformation/cudos/data-exports-aggregation.yaml create mode 120000 modules/destination/assets/cloudformation/data-exports-aggregation.yaml mode change 100644 => 120000 modules/source/assets/cloudformation/cudos/data-exports-aggregation.yaml diff --git a/assets/cloudformation/cudos/data-exports-aggregation.yaml b/assets/cloudformation/cudos/data-exports-aggregation.yaml new file mode 100644 index 0000000..e5caa7f --- /dev/null +++ b/assets/cloudformation/cudos/data-exports-aggregation.yaml @@ -0,0 +1,1564 @@ +# +## https://raw.githubusercontent.com/aws-samples/aws-cudos-framework-deployment/refs/heads/main/cfn-templates/data-exports-aggregation.yaml +# +AWSTemplateFormatVersion: "2010-09-09" +Description: AWS Billing Data Export Aggregation v0.1.4 +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: "Common Parameters Configuration" + Parameters: + - DestinationAccountId + - ResourcePrefix + - ManageCUR2 + - ManageFOCUS + - ManageCOH + - Label: + default: "Parameters needed in Destination (Data Collection) Account only" + Parameters: + - SourceAccountIds + - Label: + default: "Technical Parameters. Please do not change." + Parameters: + - EnableSCAD + - RolePath + + ParameterLabels: + ManageCOH: + default: "Cost Optimization Recommendations" + ManageCUR2: + default: "CUR 2.0" + ManageFOCUS: + default: "FOCUS" + DestinationAccountId: + default: "Destination (Data Collection) Account Id" + ResourcePrefix: + default: "Resource Prefix" + SourceAccountIds: + default: "Source Account Ids (Comma separated list)" + EnableSCAD: + default: "Enable Split Cost Allocation Data (SCAD) in CUR 2.0" + +Parameters: + ## + # Common params + ## + DestinationAccountId: + Type: String + Description: "AWS Account Id where DataExport will be replicated to (Where you deploy CID Quicksight Dashboards or Data Collection)" + AllowedPattern: '\d{12}' + ResourcePrefix: + Type: String + Default: "cid" + Description: "Prefix used for all named resources, including S3 Bucket. Must be the same in destination and source stacks" + MaxLength: 37 # = 63 - len('-123456789012-data-exports') + AllowedPattern: "^[a-z0-9]+[a-z0-9-]{1,61}[a-z0-9]+$" + ManageCUR2: + Type: String + Description: "" + AllowedValues: ["yes", "no"] + Default: "no" + ManageFOCUS: + Type: String + Description: "NOTE: you can have only one export of this type" + AllowedValues: ["yes", "no"] + Default: "no" + ManageCOH: + Type: String + Description: "NOTE: you must have Cost Optimization Hub Enabled" + AllowedValues: ["yes", "no"] + Default: "no" + + ## + # Destination specific params + ## + SourceAccountIds: + Type: String + AllowedPattern: "^((\\d{12})\\,?)*$" + Default: "" + Description: "Ex: 12345678912,98745612312,.... If you install all in the same account (Source=Destination) please put Destination Account Id first in the list. " + + ## + # Technical params + ## + EnableSCAD: + Type: String + Description: Whether to enable Split Cost Allocation Data (Scad). Set this to 'No', if you experience performance issues due to dataset size. + AllowedValues: ["yes", "no"] + Default: "yes" + RolePath: + Type: String + Default: "/" + Description: Path for roles where PermissionBoundaries can limit location + +Conditions: + EmptySourceAccountIds: !Equals [!Ref SourceAccountIds, ""] + IsDestinationAccount: + !Equals [!Ref DestinationAccountId, !Ref "AWS::AccountId"] + IsSourceAccount: + # it is Source account if it is not a destination or if it is a destination and it is listed in Source Accounts (as the list one). + # Unfortunately, there no 'Fn::Contains' in Conditions, so we need to request user setting Dest account as the first. + Fn::Or: + - !Not [!Condition IsDestinationAccount] + - !Equals [ + !Ref "AWS::AccountId", + !Select [0, !Split [",", !Sub "${SourceAccountIds},"]], + ] + RegionSupportsDataExportsViaCFN: # CFN supports DataExports only in us-east-1 and cn-northwest-1. Other regions must use lambda. + Fn::Or: + - !Equals [!Ref "AWS::Region", "us-east-1"] + - !Equals [!Ref "AWS::Region", "cn-northwest-1"] + ManageCUR2: !Equals [!Ref ManageCUR2, "yes"] + ManageFOCUS: !Equals [!Ref ManageFOCUS, "yes"] + ManageCOH: !Equals [!Ref ManageCOH, "yes"] + EnableSCAD: !Equals [!Ref EnableSCAD, "yes"] + DeployDataExport: + Fn::Or: + - !Condition ManageCUR2 + - !Condition ManageFOCUS + - !Condition ManageCOH + DeployCOHServiceRole: !And [!Condition IsSourceAccount, !Condition ManageCOH] + DeployCUR2ViaCFN: + !And [ + !Condition IsSourceAccount, + !Condition ManageCUR2, + !Condition RegionSupportsDataExportsViaCFN, + ] + DeployFOCUSViaCFN: + !And [ + !Condition IsSourceAccount, + !Condition ManageFOCUS, + !Condition RegionSupportsDataExportsViaCFN, + ] + DeployCOHViaCFN: + !And [ + !Condition IsSourceAccount, + !Condition ManageCOH, + !Condition RegionSupportsDataExportsViaCFN, + ] + DeployCUR2ViaLambda: + !And [ + !Condition IsSourceAccount, + !Condition ManageCUR2, + !Not [!Condition RegionSupportsDataExportsViaCFN], + ] + DeployFOCUSViaLambda: + !And [ + !Condition IsSourceAccount, + !Condition ManageFOCUS, + !Not [!Condition RegionSupportsDataExportsViaCFN], + ] + DeployCOHViaLambda: + !And [ + !Condition IsSourceAccount, + !Condition ManageCOH, + !Not [!Condition RegionSupportsDataExportsViaCFN], + ] + DeployCUR2Table: !And [!Condition IsDestinationAccount, !Condition ManageCUR2] + DeployFOCUSTable: + !And [!Condition IsDestinationAccount, !Condition ManageFOCUS] + DeployCOHTable: !And [!Condition IsDestinationAccount, !Condition ManageCOH] + DeployAnyExportViaLambda: + !Or [ + !Condition DeployCUR2ViaLambda, + !Condition DeployFOCUSViaLambda, + !Condition DeployCOHViaLambda, + ] + DeployAnyTable: + !Or [ + !Condition DeployFOCUSTable, + !Condition DeployCUR2Table, + !Condition DeployCOHTable, + ] + +Mappings: + DataExports: + #Mappings for storing values for different Data Exports tables + CUR2: + DefaultQuery: >- + SELECT bill_bill_type, bill_billing_entity, bill_billing_period_end_date, bill_billing_period_start_date, bill_invoice_id, bill_invoicing_entity, bill_payer_account_id, bill_payer_account_name, cost_category, discount, discount_bundled_discount, discount_total_discount, identity_line_item_id, identity_time_interval, line_item_availability_zone, line_item_blended_cost, line_item_blended_rate, line_item_currency_code, line_item_legal_entity, line_item_line_item_description, line_item_line_item_type, line_item_net_unblended_cost, line_item_net_unblended_rate, line_item_normalization_factor, line_item_normalized_usage_amount, line_item_operation, line_item_product_code, line_item_resource_id, line_item_tax_type, line_item_unblended_cost, line_item_unblended_rate, line_item_usage_account_id, line_item_usage_account_name, line_item_usage_amount, line_item_usage_end_date, line_item_usage_start_date, line_item_usage_type, pricing_currency, pricing_lease_contract_length, pricing_offering_class, pricing_public_on_demand_cost, pricing_public_on_demand_rate, pricing_purchase_option, pricing_rate_code, pricing_rate_id, pricing_term, pricing_unit, product, product_comment, product_fee_code, product_fee_description, product_from_location, product_from_location_type, product_from_region_code, product_instance_family, product_instance_type, product_instancesku, product_location, product_location_type, product_operation, product_pricing_unit, product_product_family, product_region_code, product_servicecode, product_sku, product_to_location, product_to_location_type, product_to_region_code, product_usagetype, reservation_amortized_upfront_cost_for_usage, reservation_amortized_upfront_fee_for_billing_period, reservation_availability_zone, reservation_effective_cost, reservation_end_time, reservation_modification_status, reservation_net_amortized_upfront_cost_for_usage, reservation_net_amortized_upfront_fee_for_billing_period, reservation_net_effective_cost, reservation_net_recurring_fee_for_usage, reservation_net_unused_amortized_upfront_fee_for_billing_period, reservation_net_unused_recurring_fee, reservation_net_upfront_value, reservation_normalized_units_per_reservation, reservation_number_of_reservations, reservation_recurring_fee_for_usage, reservation_reservation_a_r_n, reservation_start_time, reservation_subscription_id, reservation_total_reserved_normalized_units, reservation_total_reserved_units, reservation_units_per_reservation, reservation_unused_amortized_upfront_fee_for_billing_period, reservation_unused_normalized_unit_quantity, reservation_unused_quantity, reservation_unused_recurring_fee, reservation_upfront_value, resource_tags, savings_plan_amortized_upfront_commitment_for_billing_period, savings_plan_end_time, savings_plan_instance_type_family, savings_plan_net_amortized_upfront_commitment_for_billing_period, savings_plan_net_recurring_commitment_for_billing_period, savings_plan_net_savings_plan_effective_cost, savings_plan_offering_type, savings_plan_payment_option, savings_plan_purchase_term, savings_plan_recurring_commitment_for_billing_period, savings_plan_region, savings_plan_savings_plan_a_r_n, savings_plan_savings_plan_effective_cost, savings_plan_savings_plan_rate, savings_plan_start_time, savings_plan_total_commitment_to_date, savings_plan_used_commitment + FROM COST_AND_USAGE_REPORT + SCADQuery: >- + SELECT bill_bill_type, bill_billing_entity, bill_billing_period_end_date, bill_billing_period_start_date, bill_invoice_id, bill_invoicing_entity, bill_payer_account_id, bill_payer_account_name, cost_category, discount, discount_bundled_discount, discount_total_discount, identity_line_item_id, identity_time_interval, line_item_availability_zone, line_item_blended_cost, line_item_blended_rate, line_item_currency_code, line_item_legal_entity, line_item_line_item_description, line_item_line_item_type, line_item_net_unblended_cost, line_item_net_unblended_rate, line_item_normalization_factor, line_item_normalized_usage_amount, line_item_operation, line_item_product_code, line_item_resource_id, line_item_tax_type, line_item_unblended_cost, line_item_unblended_rate, line_item_usage_account_id, line_item_usage_account_name, line_item_usage_amount, line_item_usage_end_date, line_item_usage_start_date, line_item_usage_type, pricing_currency, pricing_lease_contract_length, pricing_offering_class, pricing_public_on_demand_cost, pricing_public_on_demand_rate, pricing_purchase_option, pricing_rate_code, pricing_rate_id, pricing_term, pricing_unit, product, product_comment, product_fee_code, product_fee_description, product_from_location, product_from_location_type, product_from_region_code, product_instance_family, product_instance_type, product_instancesku, product_location, product_location_type, product_operation, product_pricing_unit, product_product_family, product_region_code, product_servicecode, product_sku, product_to_location, product_to_location_type, product_to_region_code, product_usagetype, reservation_amortized_upfront_cost_for_usage, reservation_amortized_upfront_fee_for_billing_period, reservation_availability_zone, reservation_effective_cost, reservation_end_time, reservation_modification_status, reservation_net_amortized_upfront_cost_for_usage, reservation_net_amortized_upfront_fee_for_billing_period, reservation_net_effective_cost, reservation_net_recurring_fee_for_usage, reservation_net_unused_amortized_upfront_fee_for_billing_period, reservation_net_unused_recurring_fee, reservation_net_upfront_value, reservation_normalized_units_per_reservation, reservation_number_of_reservations, reservation_recurring_fee_for_usage, reservation_reservation_a_r_n, reservation_start_time, reservation_subscription_id, reservation_total_reserved_normalized_units, reservation_total_reserved_units, reservation_units_per_reservation, reservation_unused_amortized_upfront_fee_for_billing_period, reservation_unused_normalized_unit_quantity, reservation_unused_quantity, reservation_unused_recurring_fee, reservation_upfront_value, resource_tags, savings_plan_amortized_upfront_commitment_for_billing_period, savings_plan_end_time, savings_plan_instance_type_family, savings_plan_net_amortized_upfront_commitment_for_billing_period, savings_plan_net_recurring_commitment_for_billing_period, savings_plan_net_savings_plan_effective_cost, savings_plan_offering_type, savings_plan_payment_option, savings_plan_purchase_term, savings_plan_recurring_commitment_for_billing_period, savings_plan_region, savings_plan_savings_plan_a_r_n, savings_plan_savings_plan_effective_cost, savings_plan_savings_plan_rate, savings_plan_start_time, savings_plan_total_commitment_to_date, savings_plan_used_commitment, split_line_item_actual_usage, split_line_item_net_split_cost, split_line_item_net_unused_cost, split_line_item_parent_resource_id, split_line_item_public_on_demand_split_cost, split_line_item_public_on_demand_unused_cost, split_line_item_reserved_usage, split_line_item_split_cost, split_line_item_split_usage, split_line_item_split_usage_ratio, split_line_item_unused_cost + FROM COST_AND_USAGE_REPORT + FOCUS: + DefaultQuery: >- + SELECT AvailabilityZone, BilledCost, BillingAccountId, BillingAccountName, BillingCurrency, BillingPeriodEnd, BillingPeriodStart, ChargeCategory, ChargeClass, ChargeDescription, ChargeFrequency, ChargePeriodEnd, ChargePeriodStart, CommitmentDiscountCategory, CommitmentDiscountId, CommitmentDiscountName, CommitmentDiscountType, CommitmentDiscountStatus, ConsumedQuantity, ConsumedUnit, ContractedCost, ContractedUnitPrice, EffectiveCost, InvoiceIssuerName, ListCost, ListUnitPrice, PricingCategory, PricingQuantity, PricingUnit, ProviderName, PublisherName, RegionId, RegionName, ResourceId, ResourceName, ResourceType, ServiceCategory, ServiceName, SkuId, SkuPriceId, SubAccountId, SubAccountName, Tags, x_CostCategories, x_Discounts, x_Operation, x_ServiceCode, x_UsageType + FROM FOCUS_1_0_AWS_PREVIEW + COH: + DefaultQuery: >- + SELECT account_id, action_type, currency_code, current_resource_details, current_resource_summary, current_resource_type, estimated_monthly_cost_after_discount, estimated_monthly_cost_before_discount, estimated_monthly_savings_after_discount, estimated_monthly_savings_before_discount, estimated_savings_percentage_after_discount, estimated_savings_percentage_before_discount, implementation_effort, last_refresh_timestamp, recommendation_id, recommendation_lookback_period_in_days, recommendation_source, recommended_resource_details, recommended_resource_summary, recommended_resource_type, region, resource_arn, restart_needed, rollback_possible, tags + FROM COST_OPTIMIZATION_RECOMMENDATIONS + +Resources: + ########################################################################### + # Destination Account Resources + ########################################################################### + + DestinationS3: + Type: AWS::S3::Bucket + Condition: IsDestinationAccount + DeletionPolicy: Retain + UpdateReplacePolicy: Retain + Properties: + BucketName: !Sub "${ResourcePrefix}-${AWS::AccountId}-data-exports" + + ## Uncomment following lines to enable bucket logging if needed. Please be careful with the cost of logging. + # LoggingConfiguration: + # DestinationBucketName: REPLACE_WITH_YOUR_LOGGING_BUCKET + # LogFilePrefix: REPLACE_WITH_YOUR_PREFIX + + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 ## Use AWS managed KMS + ## If you need Customer managed KMS key, yoy can do that using following parameters: + # SSEAlgorithm: aws:kms + # KMSMasterKeyID: "REPLACE_WITH_YOUR_KEY_ARN" + AccessControl: BucketOwnerFullControl + OwnershipControls: + Rules: + - ObjectOwnership: BucketOwnerEnforced + PublicAccessBlockConfiguration: + BlockPublicAcls: true + BlockPublicPolicy: true + IgnorePublicAcls: true + RestrictPublicBuckets: true + VersioningConfiguration: + Status: Enabled + LifecycleConfiguration: + Rules: + - Id: Object&Version Expiration + Status: Enabled + NoncurrentVersionExpirationInDays: 32 # 1 month + Metadata: + cfn_nag: + rules_to_suppress: + - id: "W35" + reason: "Data buckets would generate too much logs" + cfn-lint: + config: + ignore_checks: + - W3045 # Need to use AccessControl for replication + + DestinationS3BucketPolicy: + Type: "AWS::S3::BucketPolicy" + Condition: IsDestinationAccount + DeletionPolicy: Retain + UpdateReplacePolicy: Delete + Properties: + Bucket: + Ref: DestinationS3 + PolicyDocument: + Id: AllowReplication + Version: "2012-10-17" + Statement: + - Sid: AllowTLS12Only + Effect: Deny + Principal: "*" + Action: s3:* + Resource: + - !Sub "arn:${AWS::Partition}:s3:::${DestinationS3}" + - !Sub "arn:${AWS::Partition}:s3:::${DestinationS3}/*" + Condition: + NumericLessThan: + s3:TlsVersion: 1.2 + - Sid: AllowOnlyHTTPS + Effect: Deny + Principal: "*" + Action: s3:* + Resource: + - !Sub "arn:${AWS::Partition}:s3:::${DestinationS3}" + - !Sub "arn:${AWS::Partition}:s3:::${DestinationS3}/*" + Condition: + Bool: + aws:SecureTransport: false + - Sid: AllowReplicationWrite + Effect: Allow + Principal: + AWS: + Fn::If: + - EmptySourceAccountIds + - !Ref AWS::AccountId + - !Split [",", !Ref SourceAccountIds] + Action: + - s3:ReplicateDelete + - s3:ReplicateObject + Resource: !Sub "arn:${AWS::Partition}:s3:::${DestinationS3}/*" + - Sid: AllowReplicationRead + Effect: Allow + Principal: + AWS: + Fn::If: + - EmptySourceAccountIds + - !Ref AWS::AccountId + - !Split [",", !Ref SourceAccountIds] + Action: + - s3:ListBucket + - s3:ListBucketVersions + - s3:GetBucketVersioning + - s3:PutBucketVersioning + Resource: !Sub "arn:${AWS::Partition}:s3:::${DestinationS3}" + + SourceS3: + Type: AWS::S3::Bucket + Condition: DeployDataExport + DeletionPolicy: Delete + UpdateReplacePolicy: Delete + Properties: + BucketName: !Sub ${ResourcePrefix}-${AWS::AccountId}-data-local + + ## Uncomment following lines to enable bucket logging if needed. Please be careful with the cost of logging. + # LoggingConfiguration: + # DestinationBucketName: REPLACE_WITH_YOUR_LOGGING_BUCKET + # LogFilePrefix: REPLACE_WITH_YOUR_PREFIX + + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 ## Use AWS managed KMS + ## If you need Customer managed KMS key, yoy can do that using following parameters: + # SSEAlgorithm: aws:kms + # KMSMasterKeyID: "REPLACE_WITH_YOUR_KEY_ARN" + AccessControl: BucketOwnerFullControl + OwnershipControls: + Rules: + - ObjectOwnership: BucketOwnerEnforced + PublicAccessBlockConfiguration: + BlockPublicAcls: true + BlockPublicPolicy: true + IgnorePublicAcls: true + RestrictPublicBuckets: true + VersioningConfiguration: + Status: Enabled + ReplicationConfiguration: + Role: !GetAtt ReplicationRole.Arn + Rules: + - Destination: + Bucket: !Sub "arn:${AWS::Partition}:s3:::${ResourcePrefix}-${DestinationAccountId}-data-exports" + StorageClass: STANDARD + Id: ReplicateCUR2Data + Prefix: !Sub "cur2/${AWS::AccountId}/${ResourcePrefix}-cur2/data/" # Hardcoded export name + Status: Enabled + - Destination: + Bucket: !Sub "arn:${AWS::Partition}:s3:::${ResourcePrefix}-${DestinationAccountId}-data-exports" + StorageClass: STANDARD + Id: ReplicateFOCUSData + Prefix: !Sub "focus/${AWS::AccountId}/${ResourcePrefix}-focus/data/" # Hardcoded export name + Status: Enabled + - Destination: + Bucket: !Sub "arn:${AWS::Partition}:s3:::${ResourcePrefix}-${DestinationAccountId}-data-exports" + StorageClass: STANDARD + Id: ReplicateCOHData + Prefix: !Sub "coh/${AWS::AccountId}/${ResourcePrefix}-coh/data/" # Hardcoded export name + Status: Enabled + LifecycleConfiguration: + Rules: + - Id: Object&Version Expiration + Status: Enabled + NoncurrentVersionExpirationInDays: 32 # 1 month + ExpirationInDays: 64 # 2 months + Metadata: + cfn_nag: + rules_to_suppress: + - id: "W35" + reason: "Data buckets would generate too much logs" + cfn-lint: + config: + ignore_checks: + - W3045 # Need to use AccessControl for replication + + SourceS3BucketPolicy: + Type: "AWS::S3::BucketPolicy" + Condition: DeployDataExport + DeletionPolicy: Delete + UpdateReplacePolicy: Delete + Properties: + Bucket: !Ref SourceS3 + PolicyDocument: + Id: AllowBillingReadAndWrite + Version: "2012-10-17" + Statement: + - Sid: AllowTLS12Only + Effect: Deny + Principal: "*" + Action: s3:* + Resource: + - !Sub "arn:${AWS::Partition}:s3:::${SourceS3}" + - !Sub "arn:${AWS::Partition}:s3:::${SourceS3}/*" + Condition: + NumericLessThan: + s3:TlsVersion: 1.2 + - Sid: AllowOnlyHTTPS + Effect: Deny + Principal: "*" + Action: s3:* + Resource: + - !Sub "arn:${AWS::Partition}:s3:::${SourceS3}" + - !Sub "arn:${AWS::Partition}:s3:::${SourceS3}/*" + Condition: + Bool: + aws:SecureTransport: false + - Sid: AllowBillingReadAndWrite + Effect: Allow + Principal: + Service: bcm-data-exports.amazonaws.com + Action: + - s3:GetBucketAcl + - s3:GetBucketPolicy + - s3:PutObject + Resource: + - !Sub "arn:${AWS::Partition}:s3:::${SourceS3}" + - !Sub "arn:${AWS::Partition}:s3:::${SourceS3}/*" + Condition: + StringEquals: + aws:SourceAccount: !Ref AWS::AccountId + + ReplicationRole: + Condition: DeployDataExport + Type: AWS::IAM::Role + Properties: + Path: !Sub /${ResourcePrefix}/ + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: + - "s3.amazonaws.com" + Action: + - "sts:AssumeRole" + Policies: + - PolicyName: ReplicationPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - s3:GetReplicationConfiguration + - s3:ListBucket + Resource: !Sub "arn:${AWS::Partition}:s3:::${ResourcePrefix}-${AWS::AccountId}-data-local" + - Effect: Allow + Action: + - s3:GetObjectVersionForReplication + - s3:GetObjectVersionAcl + - s3:GetObjectVersionTagging + Resource: !Sub "arn:${AWS::Partition}:s3:::${ResourcePrefix}-${AWS::AccountId}-data-local/*" + - Effect: Allow + Action: + - s3:ReplicateObject + - s3:ReplicateDelete + - s3:ReplicateTags + Resource: !Sub "arn:${AWS::Partition}:s3:::${ResourcePrefix}-${DestinationAccountId}-data-exports/*/${AWS::AccountId}/*" + + # CUR2 + + ## Deploy Data Export natively via CFN resource in regions that support native CFN + LocalCUR2viaCFN: + Type: AWS::BCMDataExports::Export + Condition: DeployCUR2ViaCFN + DependsOn: + - SourceS3BucketPolicy + Properties: + Export: + DataQuery: + QueryStatement: + !If [ + EnableSCAD, + !FindInMap [DataExports, CUR2, SCADQuery], + !FindInMap [DataExports, CUR2, DefaultQuery], + ] + TableConfigurations: + COST_AND_USAGE_REPORT: + TIME_GRANULARITY: "HOURLY" + INCLUDE_RESOURCES: "TRUE" + INCLUDE_MANUAL_DISCOUNT_COMPATIBILITY: "FALSE" + INCLUDE_SPLIT_COST_ALLOCATION_DATA: + !If [EnableSCAD, "TRUE", "FALSE"] + Description: "CUR 2.0 export for aggregation in CID" + DestinationConfigurations: + S3Destination: + S3Bucket: !Ref SourceS3 + S3Prefix: !Sub "cur2/${AWS::AccountId}" # Explicitly hardcode name of folder (it will be the same as table name) + S3Region: !Ref AWS::Region + S3OutputConfigurations: + Overwrite: "OVERWRITE_REPORT" + Format: "PARQUET" + Compression: "PARQUET" + OutputType: "CUSTOM" + Name: !Sub "${ResourcePrefix}-cur2" + RefreshCadence: + Frequency: "SYNCHRONOUS" + + # Deploy Data Export via Lambda due to missing CFN resource definition AWS::BCMDataExports::Export outside us-east-1 or cn-northwest-1 + LocalCUR2viaCustomResource: + Type: Custom::CUR2Creator + Condition: DeployCUR2ViaLambda + Properties: + ServiceToken: !GetAtt CidDataExportCreatorLambda.Arn + BucketPolicyWait: !Ref SourceS3BucketPolicy + Export: + DataQuery: + QueryStatement: + !If [ + EnableSCAD, + !FindInMap [DataExports, CUR2, SCADQuery], + !FindInMap [DataExports, CUR2, DefaultQuery], + ] + TableConfigurations: + COST_AND_USAGE_REPORT: + TIME_GRANULARITY: "HOURLY" + INCLUDE_RESOURCES: "TRUE" + INCLUDE_MANUAL_DISCOUNT_COMPATIBILITY: "FALSE" + INCLUDE_SPLIT_COST_ALLOCATION_DATA: + !If [EnableSCAD, "TRUE", "FALSE"] + Description: "CUR 2.0 export for aggregation in CID" + DestinationConfigurations: + S3Destination: + S3Bucket: !Ref SourceS3 + S3Prefix: !Sub "cur2/${AWS::AccountId}" # Explicitly hardcode name of folder (it will be the same as table name) + S3Region: !Ref AWS::Region + S3OutputConfigurations: + Overwrite: "OVERWRITE_REPORT" + Format: "PARQUET" + Compression: "PARQUET" + OutputType: "CUSTOM" + Name: !Sub "${ResourcePrefix}-cur2" + RefreshCadence: + Frequency: "SYNCHRONOUS" + + # FOCUS + + ## Deploy Data Export natively via CFN resource in regions that support native CFN + LocalFOCUSviaCFN: + Type: AWS::BCMDataExports::Export + Condition: DeployFOCUSViaCFN + DependsOn: + - SourceS3BucketPolicy + Properties: + Export: + DataQuery: + QueryStatement: !FindInMap [DataExports, FOCUS, DefaultQuery] + Description: "FOCUS export for aggregation in CID" + DestinationConfigurations: + S3Destination: + S3Bucket: !Ref SourceS3 + S3Prefix: !Sub "focus/${AWS::AccountId}" # Explicitly hardcode name of folder (it will be the same as table name) + S3Region: !Ref AWS::Region + S3OutputConfigurations: + Overwrite: "OVERWRITE_REPORT" + Format: "PARQUET" + Compression: "PARQUET" + OutputType: "CUSTOM" + Name: !Sub "${ResourcePrefix}-focus" + RefreshCadence: + Frequency: "SYNCHRONOUS" + + # Deploy Data Export via Lambda due to missing CFN resource definition AWS::BCMDataExports::Export outside us-east-1 or cn-northwest-1 + LocalFOCUSviaCustomResource: + Type: Custom::CUR2Creator + Condition: DeployFOCUSViaLambda + Properties: + ServiceToken: !GetAtt CidDataExportCreatorLambda.Arn + BucketPolicyWait: !Ref SourceS3BucketPolicy + Export: + DataQuery: + QueryStatement: !FindInMap [DataExports, FOCUS, DefaultQuery] + Description: "FOCUS export for aggregation in CID" + DestinationConfigurations: + S3Destination: + S3Bucket: !Ref SourceS3 + S3Prefix: !Sub "focus/${AWS::AccountId}" # Explicitly hardcode name of folder (it will be the same as table name) + S3Region: !Ref AWS::Region + S3OutputConfigurations: + Overwrite: "OVERWRITE_REPORT" + Format: "PARQUET" + Compression: "PARQUET" + OutputType: "CUSTOM" + Name: !Sub "${ResourcePrefix}-focus" + RefreshCadence: + Frequency: "SYNCHRONOUS" + + # COH + + ## Deploy Data Export natively via CFN resource in regions that support native CFN + LocalCOHviaCFN: + Type: AWS::BCMDataExports::Export + Condition: DeployCOHViaCFN + DependsOn: + - SourceS3BucketPolicy + - CreateServiceLinkedRoleCustomResource + Properties: + Export: + DataQuery: + QueryStatement: !FindInMap [DataExports, COH, DefaultQuery] + TableConfigurations: + COST_OPTIMIZATION_RECOMMENDATIONS: + FILTER: "{}" + INCLUDE_ALL_RECOMMENDATIONS: "TRUE" + Description: "Cost Optimization Hub Recommendations export for aggregation in CID" + DestinationConfigurations: + S3Destination: + S3Bucket: !Ref SourceS3 + S3Prefix: !Sub "coh/${AWS::AccountId}" # Explicitly hardcode name of folder (it will be the same as table name) + S3Region: !Ref AWS::Region + S3OutputConfigurations: + Overwrite: "OVERWRITE_REPORT" + Format: "PARQUET" + Compression: "PARQUET" + OutputType: "CUSTOM" + Name: !Sub "${ResourcePrefix}-coh" + RefreshCadence: + Frequency: "SYNCHRONOUS" + + # Deploy Data Export via Lambda due to missing CFN resource definition AWS::BCMDataExports::Export outside us-east-1 or cn-northwest-1 + LocalCOHviaCustomResource: + Type: Custom::CUR2Creator + DependsOn: + - CreateServiceLinkedRoleCustomResource + Condition: DeployCOHViaLambda + Properties: + ServiceToken: !GetAtt CidDataExportCreatorLambda.Arn + BucketPolicyWait: !Ref SourceS3BucketPolicy + Export: + DataQuery: + QueryStatement: !FindInMap [DataExports, COH, DefaultQuery] + TableConfigurations: + COST_OPTIMIZATION_RECOMMENDATIONS: + FILTER: "{}" + INCLUDE_ALL_RECOMMENDATIONS: "TRUE" + Description: "Cost Optimization Hub Recommendations export for aggregation in CID" + DestinationConfigurations: + S3Destination: + S3Bucket: !Ref SourceS3 + S3Prefix: !Sub "coh/${AWS::AccountId}" # Explicitly hardcode name of folder (it will be the same as table name) + S3Region: !Ref AWS::Region + S3OutputConfigurations: + Overwrite: "OVERWRITE_REPORT" + Format: "PARQUET" + Compression: "PARQUET" + OutputType: "CUSTOM" + Name: !Sub "${ResourcePrefix}-coh" + RefreshCadence: + Frequency: "SYNCHRONOUS" + + # COH export requires a service linked role to be created BUT this role can be already created. Thus we need to create it via custom resource + LambdaServiceLinkedRoleExecutionRole: + Type: "AWS::IAM::Role" + Condition: DeployCOHServiceRole + Properties: + RoleName: !Sub "${AWS::StackName}-LambdaExecutionRole" + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: !Sub "${AWS::StackName}-LambdaPolicy" + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + Resource: + - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-CreateServiceLinkedRoleFunction" + - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-CreateServiceLinkedRoleFunction:*" + - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-CreateServiceLinkedRoleFunction:*:*" + - Effect: Allow + Action: + - iam:GetRole + - iam:CreateServiceLinkedRole + - iam:DeleteServiceLinkedRole + - iam:GetServiceLinkedRoleDeletionStatus + Resource: !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/aws-service-role/bcm-data-exports.amazonaws.com/AWSServiceRoleForBCMDataExports" + - Effect: Allow + Action: + - cost-optimization-hub:GetPreferences + Resource: "*" # Cannot restrict this + + Metadata: + cfn_nag: + rules_to_suppress: + - id: "W28" + reason: "Need an explicit name for reference" + - id: "W11" + reason: "Some COH resources cannot be restricted" + + CreateServiceLinkedRoleFunction: + Type: "AWS::Lambda::Function" + Condition: DeployCOHServiceRole + Properties: + FunctionName: !Sub "${ResourcePrefix}-CreateServiceLinkedRoleFunction" + Handler: index.handler + MemorySize: 128 + Runtime: python3.12 + Timeout: 15 + Role: !GetAtt LambdaServiceLinkedRoleExecutionRole.Arn + Code: + ZipFile: | + import json + import time + import boto3 + import cfnresponse + + def handler(event, context): + print(json.dumps(event)) + coh = boto3.client('cost-optimization-hub', region_name='us-east-1') + iam = boto3.client('iam') + try: + if event['RequestType'] in ['Create', 'Update']: + + print("Make sure CO hub is activated") + try: + coh.get_preferences() + except Exception as e: + if 'AWS account is not enrolled for recommendations' in str(e): + raise Exception('AWS account is not enrolled for recommendations. Please activate Cost Optimization Hub.') + raise + + print("Creating service linked role") + iam.create_service_linked_role( + AWSServiceName='bcm-data-exports.amazonaws.com', + Description='Service-linked role for bcm-data-exports.amazonaws.com' + ) + + print("Waiting for the role to be created") + for i in range(60): + try: + iam.get_role(RoleName='AWSServiceRoleForBCMDataExports') + print("Role is created") + break + except iam.exceptions.NoSuchEntityException: + time.sleep(1) + + print("Additional wait to make sure the role is available") + time.sleep(30) + + cfnresponse.send(event, context, cfnresponse.SUCCESS, {}) + + except Exception as e: + if 'has been taken in this account' in str(e): + print('the role AWSServiceRoleForBCMDataExports already exist') + cfnresponse.send(event, context, cfnresponse.SUCCESS, {}) + else: + print(e) + cfnresponse.send(event, context, cfnresponse.FAILED, {}, reason=str(e)) + Metadata: + cfn_nag: + rules_to_suppress: + - id: "W89" + reason: "This Lambda does not require VPC" + - id: "W92" + reason: "One Time execution. No need for ReservedConcurrentExecutions" + + CreateServiceLinkedRoleCustomResource: + Condition: DeployCOHServiceRole + Type: "AWS::CloudFormation::CustomResource" + Properties: + ServiceToken: !GetAtt CreateServiceLinkedRoleFunction.Arn + + ########################################################################### + # Lambda DataExport Creator: used to create DataExport from outside us-east-1 or cn-northwest-1 + ########################################################################### + + CidDataExportCreatorLambdaRole: + Type: AWS::IAM::Role + Condition: DeployAnyExportViaLambda + Properties: + Path: !Sub /${ResourcePrefix}/ + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + Policies: + - PolicyName: "ExecutionDefault" + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - logs:CreateLogStream + - logs:PutLogEvents + - logs:CreateLogGroup + Resource: + - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-DataExportCreator" + - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-DataExportCreator:*" + - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-DataExportCreator:*:*" + - PolicyName: "AllowDataExports" + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - bcm-data-exports:CreateExport + - bcm-data-exports:UpdateExport + - bcm-data-exports:DeleteExport + Resource: !Sub "arn:${AWS::Partition}:bcm-data-exports:*:${AWS::AccountId}:*" + - Effect: Allow + Action: + - cur:PutReportDefinition #need this permission for bcm-data-exports to work + Resource: !Sub "arn:${AWS::Partition}:cur:*:${AWS::AccountId}:*" + - Effect: Allow + Action: + - cost-optimization-hub:GetRecommendation #need this permission for bcm-data-exports to work + - cost-optimization-hub:ListRecommendations + Resource: "arn:aws:cost-optimization-hub:*" + + CidDataExportCreatorLambda: + Type: AWS::Lambda::Function + Condition: DeployAnyExportViaLambda + Properties: + Runtime: python3.12 + FunctionName: !Sub ${ResourcePrefix}-DataExportCreator + Handler: index.lambda_handler + MemorySize: 128 + Role: !GetAtt CidDataExportCreatorLambdaRole.Arn + Timeout: 15 + Code: + ZipFile: | + import os + import json + import uuid + + import boto3 + import cfnresponse + + # DataExports only exist in us-east-1 and cn-northwest-1 regions + region = 'us-east-1' if not os.environ['AWS_REGION'].startswith('cn-') else 'cn-northwest-1' + + client = boto3.client('bcm-data-exports', region_name=region) + + def lambda_handler(event, context): + + print(json.dumps(event)) + reason = "" + + try: + export = event['ResourceProperties']['Export'] + export_name = event['ResourceProperties']['Export']['Name'] + + if event['RequestType'] == 'Create': + res = client.create_export(Export=export) + print('created:', json.dumps(res)) + elif event['RequestType'] == 'Update': + old_export_name = event['OldResourceProperties']['Export']['Name'] + if export["Name"] != old_export_name: + res = client.create_export(Export=export) + print('created:', json.dumps(res)) + try: + res = client.delete_export(Name=old_export_name) + print('deleted:', json.dumps(res)) + except: + pass # Do not block deletion + else: + res = client.update_export(Name=old_export_name, Export=export) + print('updated:', json.dumps(res)) + elif event['RequestType'] == 'Delete': + try: + res = client.delete_export(Name=old_export_name) + print('deleted:', json.dumps(res)) + except: + pass # Do not block deletion + else: + raise Exception("Unknown operation: " + event['RequestType']) + + except Exception as e: + reason = str(e) + print(e) + finally: + physicalResourceId = event.get('ResourceProperties',{}).get('Export', {}).get('Name', None) or str(uuid.uuid1()) + if reason: + print("FAILURE") + cfnresponse.send(event, context, cfnresponse.FAILED, {}, physicalResourceId, reason=reason) + else: + print("SUCCESS") + cfnresponse.send(event, context, cfnresponse.SUCCESS, {}, physicalResourceId) + Metadata: + cfn_nag: + rules_to_suppress: + - id: "W89" + reason: "This Lambda does not require VPC" + - id: "W92" + reason: "One Time execution. No need for ReservedConcurrentExecutions" + + CIDDatabase: + Type: AWS::Glue::Database + Condition: DeployAnyTable + Properties: + DatabaseInput: + Name: !Join ["_", !Split ["-", !Sub "${ResourcePrefix}_data_export"]] # replace '-' to '_' + CatalogId: !Sub "${AWS::AccountId}" + + ########################################################################### + # CUR2 + ########################################################################### + + CURTable: # Initial creation of table. it will be updated by crawler later + Type: AWS::Glue::Table + Condition: DeployCUR2Table + Properties: + CatalogId: !Sub "${AWS::AccountId}" + DatabaseName: !Ref CIDDatabase + TableInput: + Name: cur2 + Owner: owner + Retention: 0 + TableType: EXTERNAL_TABLE + Parameters: + compressionType: none + classification: parquet + UPDATED_BY_CRAWLER: !Ref CURCrawler + StorageDescriptor: + BucketColumns: [] + Compressed: false + Location: !Sub "s3://${DestinationS3}/cur2/" + NumberOfBuckets: -1 + InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat + SerdeInfo: + Parameters: + serialization.format: "1" + SerializationLibrary: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe + StoredAsSubDirectories: false + Columns: # All fields required for CID + - { "Name": "bill_bill_type", "Type": "string" } + - { "Name": "bill_billing_entity", "Type": "string" } + - { "Name": "bill_billing_period_end_date", "Type": "timestamp" } + - { "Name": "bill_billing_period_start_date", "Type": "timestamp" } + - { "Name": "bill_invoice_id", "Type": "string" } + - { "Name": "bill_payer_account_id", "Type": "string" } + - { "Name": "bill_payer_account_name", "Type": "string" } + - { "Name": "cost_category", "Type": "map" } + - { "Name": "discount", "Type": "map" } + - { "Name": "identity_line_item_id", "Type": "string" } + - { "Name": "identity_time_interval", "Type": "string" } + - { "Name": "line_item_availability_zone", "Type": "string" } + - { "Name": "line_item_legal_entity", "Type": "string" } + - { "Name": "line_item_line_item_description", "Type": "string" } + - { "Name": "line_item_line_item_type", "Type": "string" } + - { "Name": "line_item_operation", "Type": "string" } + - { "Name": "line_item_product_code", "Type": "string" } + - { "Name": "line_item_resource_id", "Type": "string" } + - { "Name": "line_item_unblended_cost", "Type": "double" } + - { "Name": "line_item_usage_account_id", "Type": "string" } + - { "Name": "line_item_usage_account_name", "Type": "string" } + - { "Name": "line_item_usage_amount", "Type": "double" } + - { "Name": "line_item_usage_end_date", "Type": "timestamp" } + - { "Name": "line_item_usage_start_date", "Type": "timestamp" } + - { "Name": "line_item_usage_type", "Type": "string" } + - { "Name": "pricing_lease_contract_length", "Type": "string" } + - { "Name": "pricing_offering_class", "Type": "string" } + - { "Name": "pricing_public_on_demand_cost", "Type": "double" } + - { "Name": "pricing_purchase_option", "Type": "string" } + - { "Name": "pricing_term", "Type": "string" } + - { "Name": "pricing_unit", "Type": "string" } + - { "Name": "product", "Type": "map" } + - { "Name": "product_from_location", "Type": "string" } + - { "Name": "product_instance_type", "Type": "string" } + - { "Name": "product_product_family", "Type": "string" } + - { "Name": "product_servicecode", "Type": "string" } + - { "Name": "product_to_location", "Type": "string" } + - { + "Name": "reservation_amortized_upfront_fee_for_billing_period", + "Type": "double", + } + - { "Name": "reservation_effective_cost", "Type": "double" } + - { "Name": "reservation_end_time", "Type": "string" } + - { "Name": "reservation_reservation_a_r_n", "Type": "string" } + - { "Name": "reservation_start_time", "Type": "string" } + - { + "Name": "reservation_unused_amortized_upfront_fee_for_billing_period", + "Type": "double", + } + - { "Name": "reservation_unused_recurring_fee", "Type": "double" } + - { "Name": "resource_tags", "Type": "map" } + - { + "Name": "savings_plan_amortized_upfront_commitment_for_billing_period", + "Type": "double", + } + - { "Name": "savings_plan_end_time", "Type": "string" } + - { "Name": "savings_plan_offering_type", "Type": "string" } + - { "Name": "savings_plan_payment_option", "Type": "string" } + - { "Name": "savings_plan_purchase_term", "Type": "string" } + - { "Name": "savings_plan_savings_plan_a_r_n", "Type": "string" } + - { + "Name": "savings_plan_savings_plan_effective_cost", + "Type": "double", + } + - { "Name": "savings_plan_start_time", "Type": "string" } + - { + "Name": "savings_plan_total_commitment_to_date", + "Type": "double", + } + - { "Name": "savings_plan_used_commitment", "Type": "double" } + - { "Name": "split_line_item_parent_resource_id", "Type": "string" } + - { "Name": "split_line_item_reserved_usage", "Type": "double" } + - { "Name": "split_line_item_actual_usage", "Type": "double" } + - { "Name": "split_line_item_split_usage", "Type": "double" } + - { "Name": "split_line_item_split_usage_ratio", "Type": "double" } + - { "Name": "split_line_item_split_cost", "Type": "double" } + - { "Name": "split_line_item_unused_cost", "Type": "double" } + - { "Name": "split_line_item_net_split_cost", "Type": "double" } + - { "Name": "split_line_item_net_unused_cost", "Type": "double" } + - { + "Name": "split_line_item_public_on_demand_split_cost", + "Type": "double", + } + - { + "Name": "split_line_item_public_on_demand_unused_cost", + "Type": "double", + } + PartitionKeys: + - { "Name": "source_account_id", "Type": "string" } + - { "Name": "report_name", "Type": "string" } + - { "Name": "data", "Type": "string" } + - { "Name": "billing_period", "Type": "string" } + + CURCrawler: + Type: AWS::Glue::Crawler + Condition: DeployCUR2Table + Properties: + Name: !Sub "${ResourcePrefix}-DataExportCUR2Crawler" + Description: A recurring crawler that keeps your CUR table in Athena up-to-date. + Role: !GetAtt CidDataExportCrawlerRole.Arn + DatabaseName: !Ref CIDDatabase + Targets: + S3Targets: + - Path: !Sub "s3://${DestinationS3}/cur2/" + Exclusions: + - "**.json" + - "**.yml" + - "**.sql" + - "**.csv" + - "**.csv.metadata" + - "**.gz" + - "**.zip" + - "**/cost_and_usage_data_status/*" + - "aws-programmatic-access-test-object" + SchemaChangePolicy: + DeleteBehavior: LOG + RecrawlPolicy: + RecrawlBehavior: CRAWL_EVERYTHING + Schedule: + ScheduleExpression: cron(0 2 * * ? *) + Configuration: | + { + "Version":1.0, + "Grouping": { + "TableGroupingPolicy": "CombineCompatibleSchemas" + }, + "CrawlerOutput":{ + "Tables":{ + "AddOrUpdateBehavior":"MergeNewColumns" + } + } + } + + ########################################################################### + # FOCUS + ########################################################################### + + FOCUSTable: # Initial creation of table. it will be updated by crawler later + Type: AWS::Glue::Table + Condition: DeployFOCUSTable + Properties: + CatalogId: !Sub "${AWS::AccountId}" + DatabaseName: !Ref CIDDatabase + TableInput: + Name: focus + Owner: owner + Retention: 0 + TableType: EXTERNAL_TABLE + Parameters: + compressionType: none + classification: parquet + UPDATED_BY_CRAWLER: !Ref FOCUSCrawler + StorageDescriptor: + BucketColumns: [] + Compressed: false + Location: !Sub "s3://${DestinationS3}/focus/" + NumberOfBuckets: -1 + InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat + SerdeInfo: + Parameters: + serialization.format: "1" + SerializationLibrary: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe + StoredAsSubDirectories: false + Columns: + - { "Name": "availabilityzone", "Type": "string" } + - { "Name": "billedcost", "Type": "double" } + - { "Name": "billingaccountid", "Type": "string" } + - { "Name": "billingaccountname", "Type": "string" } + - { "Name": "billingcurrency", "Type": "string" } + - { "Name": "billingperiodend", "Type": "timestamp" } + - { "Name": "billingperiodstart", "Type": "timestamp" } + - { "Name": "chargecategory", "Type": "string" } + - { "Name": "chargeclass", "Type": "string" } + - { "Name": "chargedescription", "Type": "string" } + - { "Name": "chargefrequency", "Type": "string" } + - { "Name": "chargeperiodend", "Type": "timestamp" } + - { "Name": "chargeperiodstart", "Type": "timestamp" } + - { "Name": "commitmentdiscountcategory", "Type": "string" } + - { "Name": "commitmentdiscountid", "Type": "string" } + - { "Name": "commitmentdiscountname", "Type": "string" } + - { "Name": "commitmentdiscounttype", "Type": "string" } + - { "Name": "commitmentdiscountstatus", "Type": "string" } + - { "Name": "consumedquantity", "Type": "double" } + - { "Name": "consumedunit", "Type": "string" } + - { "Name": "contractedcost", "Type": "double" } + - { "Name": "contractedunitprice", "Type": "double" } + - { "Name": "effectivecost", "Type": "double" } + - { "Name": "invoiceissuername", "Type": "string" } + - { "Name": "listcost", "Type": "double" } + - { "Name": "listunitprice", "Type": "double" } + - { "Name": "pricingcategory", "Type": "string" } + - { "Name": "pricingquantity", "Type": "double" } + - { "Name": "pricingunit", "Type": "string" } + - { "Name": "providername", "Type": "string" } + - { "Name": "publishername", "Type": "string" } + - { "Name": "regionid", "Type": "string" } + - { "Name": "regionname", "Type": "string" } + - { "Name": "resourceid", "Type": "string" } + - { "Name": "resourcename", "Type": "string" } + - { "Name": "resourcetype", "Type": "string" } + - { "Name": "servicecategory", "Type": "string" } + - { "Name": "servicename", "Type": "string" } + - { "Name": "skuid", "Type": "string" } + - { "Name": "skupriceid", "Type": "string" } + - { "Name": "subaccountid", "Type": "string" } + - { "Name": "subaccountname", "Type": "string" } + - { "Name": "tags", "Type": "map" } + PartitionKeys: + - { "Name": "source_account_id", "Type": "string" } + - { "Name": "report_name", "Type": "string" } + - { "Name": "data", "Type": "string" } + - { "Name": "billing_period", "Type": "string" } + + FOCUSCrawler: + Type: AWS::Glue::Crawler + Condition: DeployFOCUSTable + Properties: + Name: !Sub "${ResourcePrefix}-DataExportFOCUSCrawler" + Description: A recurring crawler that keeps your FOCUS table in Athena up-to-date. + Role: !GetAtt CidDataExportCrawlerRole.Arn + DatabaseName: !Ref CIDDatabase + Targets: + S3Targets: + - Path: !Sub "s3://${DestinationS3}/focus/" + Exclusions: + - "**.json" + - "**.yml" + - "**.sql" + - "**.csv" + - "**.csv.metadata" + - "**.gz" + - "**.zip" + - "**/cost_and_usage_data_status/*" + - "aws-programmatic-access-test-object" + SchemaChangePolicy: + DeleteBehavior: LOG + RecrawlPolicy: + RecrawlBehavior: CRAWL_EVERYTHING + Schedule: + ScheduleExpression: cron(0 2 * * ? *) + Configuration: | + { + "Version":1.0, + "Grouping": { + "TableGroupingPolicy": "CombineCompatibleSchemas" + }, + "CrawlerOutput":{ + "Tables":{ + "AddOrUpdateBehavior":"MergeNewColumns" + } + } + } + + ########################################################################### + # COH + ########################################################################### + + COHTable: # Initial creation of table. it will be updated by crawler later + Type: AWS::Glue::Table + Condition: DeployCOHTable + Properties: + CatalogId: !Sub "${AWS::AccountId}" + DatabaseName: !Ref CIDDatabase + TableInput: + Name: coh + Owner: owner + Retention: 0 + TableType: EXTERNAL_TABLE + Parameters: + compressionType: none + classification: parquet + UPDATED_BY_CRAWLER: !Ref COHCrawler + StorageDescriptor: + BucketColumns: [] + Compressed: false + Location: !Sub "s3://${DestinationS3}/coh/" + NumberOfBuckets: -1 + InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat + OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat + SerdeInfo: + Parameters: + serialization.format: "1" + SerializationLibrary: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe + StoredAsSubDirectories: false + Columns: # All fields required for CID + - { "Name": "account_id", "Type": "string" } + - { "Name": "action_type", "Type": "string" } + - { "Name": "currency_code", "Type": "string" } + - { "Name": "current_resource_details", "Type": "string" } + - { "Name": "current_resource_summary", "Type": "string" } + - { "Name": "current_resource_type", "Type": "string" } + - { + "Name": "estimated_monthly_cost_after_discount", + "Type": "double", + } + - { + "Name": "estimated_monthly_cost_before_discount", + "Type": "double", + } + - { + "Name": "estimated_monthly_savings_after_discount", + "Type": "double", + } + - { + "Name": "estimated_monthly_savings_before_discount", + "Type": "double", + } + - { + "Name": "estimated_savings_percentage_after_discount", + "Type": "double", + } + - { + "Name": "estimated_savings_percentage_before_discount", + "Type": "double", + } + - { "Name": "implementation_effort", "Type": "string" } + - { "Name": "last_refresh_timestamp", "Type": "string" } + - { "Name": "recommendation_id", "Type": "string" } + - { + "Name": "recommendation_lookback_period_in_days", + "Type": "int", + } + - { "Name": "recommendation_source", "Type": "string" } + - { "Name": "recommended_resource_details", "Type": "string" } + - { "Name": "recommended_resource_summary", "Type": "string" } + - { "Name": "recommended_resource_type", "Type": "string" } + - { "Name": "region", "Type": "string" } + - { "Name": "resource_arn", "Type": "string" } + - { "Name": "restart_needed", "Type": "boolean" } + - { "Name": "rollback_possible", "Type": "boolean" } + - { "Name": "tags", "Type": "map" } + PartitionKeys: + - { "Name": "source_account_id", "Type": "string" } + - { "Name": "report_name", "Type": "string" } + - { "Name": "data", "Type": "string" } + - { "Name": "date", "Type": "string" } + + COHCrawler: + Type: AWS::Glue::Crawler + Condition: DeployCOHTable + Properties: + Name: !Sub "${ResourcePrefix}-DataExportCOHCrawler" + Description: A recurring crawler that keeps your COH table in Athena up-to-date. + Role: !GetAtt CidDataExportCrawlerRole.Arn + DatabaseName: !Ref CIDDatabase + Targets: + S3Targets: + - Path: !Sub "s3://${DestinationS3}/coh/" + Exclusions: + - "**.json" + - "**.yml" + - "**.sql" + - "**.csv" + - "**.csv.metadata" + - "**.gz" + - "**.zip" + - "**/cost_and_usage_data_status/*" + - "aws-programmatic-access-test-object" + SchemaChangePolicy: + DeleteBehavior: LOG + RecrawlPolicy: + RecrawlBehavior: CRAWL_EVERYTHING + Schedule: + ScheduleExpression: cron(0 2 * * ? *) + Configuration: | + { + "Version":1.0, + "Grouping": { + "TableGroupingPolicy": "CombineCompatibleSchemas" + }, + "CrawlerOutput":{ + "Tables":{ + "AddOrUpdateBehavior":"MergeNewColumns" + } + } + } + + ########################################################################### + # Generic Resources + ########################################################################### + + CidDataExportCrawlerRole: + Type: AWS::IAM::Role + Condition: DeployAnyTable + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - glue.amazonaws.com + Action: + - "sts:AssumeRole" + Path: !Ref RolePath + Policies: + - PolicyName: CrawlerPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - "s3:GetObject" + Resource: !Sub "arn:${AWS::Partition}:s3:::${DestinationS3}/*" + - Effect: Allow + Action: + - "s3:ListBucket" + Resource: !Sub "arn:${AWS::Partition}:s3:::${DestinationS3}" + - Effect: Allow + Action: + - glue:GetDatabase + - glue:GetDatabases + - glue:UpdateDatabase + - glue:CreateTable + - glue:UpdateTable + - glue:GetTable + - glue:GetTables + - glue:BatchCreatePartition + - glue:CreatePartition + - glue:DeletePartition + - glue:BatchDeletePartition + - glue:UpdatePartition + - glue:GetPartition + - glue:GetPartitions + - glue:BatchGetPartition + - glue:ImportCatalogToGlue + Resource: + - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:catalog + - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:database/${CIDDatabase} + - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:table/${CIDDatabase}/* + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + Resource: + - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws-glue/crawlers:*" + - Effect: Allow + Action: + - logs:PutLogEvents + Resource: + - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws-glue/crawlers:log-stream:*" + + ########################################################################### + # Analytics: used by CID team to track adoption, by retrieving AWS AccountId + ########################################################################### + + CidLambdaAnalyticsRole: #Execution role for the custom resource CidLambdaAnalyticsExecutor + Type: AWS::IAM::Role + Properties: + Path: !Sub /${ResourcePrefix}/ + #RoleName: CID-Analytics + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + Policies: + - PolicyName: "ExecutionDefault" + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - logs:CreateLogStream + - logs:PutLogEvents + - logs:CreateLogGroup + Resource: + - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-CID-Analytics-DataExports" + - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-CID-Analytics-DataExports:*" + - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-CID-Analytics-DataExports:*:*" + + CidLambdaAnalytics: + Type: AWS::Lambda::Function + Properties: + Runtime: python3.12 + FunctionName: !Sub ${ResourcePrefix}-CID-Analytics-DataExports + Handler: index.lambda_handler + MemorySize: 128 + Role: !GetAtt CidLambdaAnalyticsRole.Arn + Timeout: 15 + Environment: + Variables: + API_ENDPOINT: https://okakvoavfg.execute-api.eu-west-1.amazonaws.com/ + Code: + ZipFile: | + import os + import json + import uuid + import urllib3 + + import boto3 + import cfnresponse + + http = urllib3.PoolManager() + endpoint = os.environ["API_ENDPOINT"] + account_id = boto3.client("sts").get_caller_identity()["Account"] + + def execute_request(action, product_id, via_key): + try: + payload = {'dashboard_id': product_id, 'account_id': account_id, via_key: 'CFN'} + encoded_data = json.dumps(payload).encode('utf-8') + r = http.request(action, endpoint, body=encoded_data, headers={'Content-Type': 'application/json'}) + if r.status != 200: + return f"This will not fail the deployment. There has been an issue logging action {action} for product {product_id} and account {account_id}, server did not respond with a 200 response, status: {r.status}, response data {r.data.decode('utf-8')}. This issue will be ignored" + except urllib3.exceptions.HTTPError as e: + return f"Issue logging action {action} for product {product_id} and account {account_id}, due to a urllib3 exception {str(e)}. This issue will be ignored" + return None + + def register_deployment(action, products): + message = f"Successfully logged {action} for {products}" + for product_id in products: + if action == 'Create': + message = execute_request('PUT', product_id, 'created_via') + elif action == 'Update': + message = execute_request('PATCH', product_id, 'updated_via') + elif action == 'Delete': + message = execute_request('DELETE', product_id, 'deleted_via') + if not message: + message = f"Successfully logged {action} for {products} " + #Do not stop deployment if we're not able to successfully record this deployment, still return true + return (True, message) + + def lambda_handler(event, context): + if event['RequestType'] in ['Create', 'Update', 'Delete']: + res, reason = register_deployment(event['RequestType'], event['ResourceProperties']['DeploymentType']) + else: + res, reason = False, "Unknown operation: " + event['RequestType'] + + physicalResourceId = event.get('ResourceProperties', {}).get('Export', {}).get('Name') or str(uuid.uuid1()) + if res: + cfnresponse.send(event, context, cfnresponse.SUCCESS, {}, physicalResourceId ) + else: + cfnresponse.send(event, context, cfnresponse.FAILED, {}, physicalResourceId, reason=reason) + Metadata: + cfn_nag: + rules_to_suppress: + - id: "W89" + reason: "This Lambda does not require VPC" + - id: "W92" + reason: "One Time execution. No need for ReservedConcurrentExecutions" + + CidLambdaAnalyticsExecutorForDataExports: + Type: Custom::CidLambdaAnalyticsExecutorForDataExports + Properties: + ServiceToken: !GetAtt CidLambdaAnalytics.Arn + DeploymentType: + - !If [ + IsDestinationAccount, + "cid-dataexport-destination", + !Ref "AWS::NoValue", + ] + - !If [IsSourceAccount, "cid-dataexport-source", !Ref "AWS::NoValue"] + + DataExportsReadAccess: + Type: AWS::IAM::ManagedPolicy + Condition: DeployAnyTable + Properties: + ManagedPolicyName: !Sub ${ResourcePrefix}DataExportsReadAccess + Description: "Policy for QuickSight to allow DataExports access" + PolicyDocument: + Version: "2012-10-17" + Statement: + - Sid: AllowGlue + Effect: Allow + Action: + - glue:GetPartition + - glue:GetPartitions + - glue:GetDatabase + - glue:GetDatabases + - glue:GetTable + - glue:GetTables + Resource: + - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:catalog + - Fn::Join: + - "" + - - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:table/ + - !Join [ + "_", + !Split ["-", !Sub "${ResourcePrefix}_data_export"], + ] # replace '-' to '_' + - "/*" + - Fn::Join: + - "" + - - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:database/ + - !Join [ + "_", + !Split ["-", !Sub "${ResourcePrefix}_data_export"], + ] # replace '-' to '_' + - Sid: AllowListBucket + Effect: Allow + Action: s3:ListBucket + Resource: + - !Sub arn:${AWS::Partition}:s3:::${ResourcePrefix}-${DestinationAccountId}-data-exports + - Sid: AllowReadBucket + Effect: Allow + Action: + - s3:GetObject + - s3:GetObjectVersion + Resource: + - !Sub arn:${AWS::Partition}:s3:::${ResourcePrefix}-${DestinationAccountId}-data-exports/* + Metadata: + cfn_nag: + rules_to_suppress: + - id: "W28" + reason: "Need an explicit name for reference" + +Outputs: + DestinationBucketName: + Description: Bucket with aggregate Data Exports + Value: !Sub ${ResourcePrefix}-${DestinationAccountId}-data-exports + Export: { Name: "cid-DataExports-Bucket" } + Database: + Description: Database for Data Exports + Value: !Join ["_", !Split ["-", !Sub "${ResourcePrefix}_data_export"]] # replace '-' to '_' + Export: { Name: "cid-DataExports-Database" } + LocalAccountBucket: + Condition: DeployDataExport + Description: Local Bucket Name which replicate objects to centralized bucket + Value: !Sub ${ResourcePrefix}-${AWS::AccountId}-data-local + ReadAccessPolicyARN: + Condition: DeployAnyTable + Description: Policy to allow read access DataExports in S3 and Athena. Attach it to QuickSight role. + Value: !Ref DataExportsReadAccess + Export: { Name: "cid-DataExports-ReadAccessPolicyARN" } diff --git a/modules/destination/README.md b/modules/destination/README.md index 3621c0a..c67b50e 100644 --- a/modules/destination/README.md +++ b/modules/destination/README.md @@ -48,7 +48,7 @@ | [saml\_metadata](#input\_saml\_metadata) | The configuration for the SAML identity provider | `string` | `null` | no | | [stack\_name\_cloud\_intelligence](#input\_stack\_name\_cloud\_intelligence) | The name of the CloudFormation stack to create the dashboards | `string` | `"CI-Cloud-Intelligence-Dashboards"` | no | | [stack\_name\_collectors](#input\_stack\_name\_collectors) | The name of the CloudFormation stack to create the collectors | `string` | `"CidDataCollectionStack"` | no | -| [stack\_name\_cora\_data\_exports\_destination](#input\_stack\_name\_cora\_data\_exports\_destination) | The name of the CloudFormation stack to create the CORA Data Exports | `string` | `"CidCoraCoraDataExportsDestinationStack"` | no | +| [stack\_name\_cora\_data\_exports](#input\_stack\_name\_cora\_data\_exports) | The name of the CloudFormation stack to create the CORA Data Exports | `string` | `"CidCoraCoraDataExportsDestinationStack"` | no | ## Outputs diff --git a/modules/destination/assets/cloudformation/data-exports-aggregation.yaml b/modules/destination/assets/cloudformation/data-exports-aggregation.yaml new file mode 120000 index 0000000..759e345 --- /dev/null +++ b/modules/destination/assets/cloudformation/data-exports-aggregation.yaml @@ -0,0 +1 @@ +../../../../assets/cloudformation/cudos/data-exports-aggregation.yaml \ No newline at end of file diff --git a/modules/destination/main.tf b/modules/destination/main.tf index 15e286c..88b2b85 100644 --- a/modules/destination/main.tf +++ b/modules/destination/main.tf @@ -1,5 +1,5 @@ -## Craft and IAM policy that allows the account to access the bucket +## Craft and IAM policy that allows the account to access the bucket data "aws_iam_policy_document" "bucket_policy" { statement { effect = "Allow" @@ -36,7 +36,7 @@ data "aws_iam_policy_document" "bucket_policy" { } } -## Provision a bucket used to contain the cloudformation templates +## Provision a bucket used to contain the cloudformation templates # tfsec:ignore:aws-s3-enable-bucket-logging module "cloudformation" { source = "terraform-aws-modules/s3-bucket/aws" @@ -67,7 +67,7 @@ module "cloudformation" { } } -## Create a lifecycle rule to old versions of the objects in the bucket +## Create a lifecycle rule to old versions of the objects in the bucket resource "aws_s3_bucket_lifecycle_configuration" "bucket_lifecycle" { bucket = module.cloudformation.s3_bucket_id @@ -81,7 +81,7 @@ resource "aws_s3_bucket_lifecycle_configuration" "bucket_lifecycle" { } } -## Upload the cloudformation templates to the bucket +## Upload the cloudformation templates to the bucket resource "aws_s3_object" "cloudformation_templates" { for_each = fileset("${path.module}/assets/cloudformation/", "**/*.yaml") @@ -92,7 +92,7 @@ resource "aws_s3_object" "cloudformation_templates" { source = "${path.module}/assets/cloudformation/${each.value}" } -## Provision enterprise quicksight if enabled +## Provision enterprise quicksight if enabled resource "aws_quicksight_account_subscription" "subscription" { count = var.enable_quicksight_subscription ? 1 : 0 @@ -102,8 +102,8 @@ resource "aws_quicksight_account_subscription" "subscription" { notification_email = var.quicksight_subscription_email } -## Provision a SAML identity provider in the data collection account - this will be -## used to authenticate sso users into quicksights +## Provision a SAML identity provider in the data collection account - this will be +## used to authenticate sso users into quicksights resource "aws_iam_saml_provider" "saml" { count = var.enable_sso ? 1 : 0 @@ -112,7 +112,7 @@ resource "aws_iam_saml_provider" "saml" { tags = var.tags } -## Provision a trust policy for the above SAML identity provider +## Provision a trust policy for the above SAML identity provider data "aws_iam_policy_document" "cudos_sso" { count = var.enable_sso ? 1 : 0 @@ -133,9 +133,9 @@ data "aws_iam_policy_document" "cudos_sso" { } } -## Provision an IAM policy which will be attached to the IAM role and has the -## necessary permissions to access quicksight users, whom have authenticated via -## the SAML identity provider +## Provision an IAM policy which will be attached to the IAM role and has the +## necessary permissions to access quicksight users, whom have authenticated via +## the SAML identity provider data "aws_iam_policy_document" "cudos_sso_permissions" { count = var.enable_sso ? 1 : 0 @@ -147,7 +147,7 @@ data "aws_iam_policy_document" "cudos_sso_permissions" { } ## Provision and IAM role to be assumed by the SAML identity provider; this role will -## be used to authenticate users into quicksights +## be used to authenticate users into quicksights resource "aws_iam_role" "cudos_sso" { count = var.enable_sso ? 1 : 0 @@ -156,7 +156,7 @@ resource "aws_iam_role" "cudos_sso" { tags = var.tags } -## Attach an inline policy to the IAM role +## Attach an inline policy to the IAM role resource "aws_iam_role_policy" "cudos_sso" { count = var.enable_sso ? 1 : 0 @@ -219,7 +219,7 @@ module "dashboard_bucket" { } } -## First we configure the collector to accept the CUR (Cost and Usage Report) from the source account +## First we configure the collector to accept the CUR (Cost and Usage Report) from the source account # tfsec:ignore:aws-s3-enable-bucket-logging module "collector" { source = "github.com/aws-samples/aws-cudos-framework-deployment//terraform-modules/cur-setup-destination?ref=4.0.2" @@ -259,7 +259,7 @@ module "dashboards" { ] } -## We need to provision the data collection stack in the colletor account +## We need to provision the data collection stack in the colletor account resource "aws_cloudformation_stack" "cudos_data_collection" { name = var.stack_name_collectors capabilities = ["CAPABILITY_NAMED_IAM", "CAPABILITY_AUTO_EXPAND"] @@ -287,3 +287,30 @@ resource "aws_cloudformation_stack" "cudos_data_collection" { module.dashboards, ] } + +## Provision the stack contain the cora data exports in the management account +## Deployment of same stack the management account +resource "aws_cloudformation_stack" "core_data_export_destination" { + count = var.enable_cora_data_exports ? 1 : 0 + + capabilities = ["CAPABILITY_NAMED_IAM", "CAPABILITY_AUTO_EXPAND"] + name = var.stack_name_cora_data_exports + on_failure = "ROLLBACK" + tags = var.tags + template_url = format("%s/cudos/%s", local.bucket_url, "data-exports-aggregation.yaml") + + parameters = { + "DestinationAccountId" = local.account_id, + "EnableSCAD" = var.enable_scad ? "yes" : "no", + "ManageCOH" = "yes", + "ManageCUR2" = "no", + "SourceAccountIds" = join(",", local.payer_account_ids), + } + + lifecycle { + ignore_changes = [ + capabilities, + ] + } +} + diff --git a/modules/destination/variables.tf b/modules/destination/variables.tf index a49b837..83c6c1d 100644 --- a/modules/destination/variables.tf +++ b/modules/destination/variables.tf @@ -57,7 +57,7 @@ variable "stack_name_collectors" { default = "CidDataCollectionStack" } -variable "stack_name_cora_data_exports_destination" { +variable "stack_name_cora_data_exports" { description = "The name of the CloudFormation stack to create the CORA Data Exports" type = string default = "CidCoraCoraDataExportsDestinationStack" diff --git a/modules/source/README.md b/modules/source/README.md index ff7bad6..2b2391c 100644 --- a/modules/source/README.md +++ b/modules/source/README.md @@ -25,6 +25,7 @@ | [enable\_scad](#input\_enable\_scad) | Indicates if the SCAD module should be enabled, only available when Cora enabled | `bool` | `false` | no | | [enable\_tao\_module](#input\_enable\_tao\_module) | Indicates if the TAO module should be enabled | `bool` | `true` | no | | [enable\_transit\_gateway\_module](#input\_enable\_transit\_gateway\_module) | Indicates if the Transit Gateway module should be enabled | `bool` | `true` | no | +| [organization\_unit\_ids](#input\_organization\_unit\_ids) | List of organization units where the read permissions stack will be deployed | `list(string)` | `[]` | no | | [stack\_name\_cora\_data\_exports\_source](#input\_stack\_name\_cora\_data\_exports\_source) | The name of the CloudFormation stack to create the CORA Data Exports | `string` | `"CidCoraCoraDataExportsSourceStack"` | no | | [stack\_name\_read\_permissions](#input\_stack\_name\_read\_permissions) | The name of the CloudFormation stack to create the collectors | `string` | `"CidDataCollectionReadPermissionsStack"` | no | | [stacks\_bucket\_name](#input\_stacks\_bucket\_name) | The name of the bucket to store the CloudFormation templates | `string` | `"cid-cloudformation-templates"` | no | diff --git a/modules/source/assets/cloudformation/cudos/data-exports-aggregation.yaml b/modules/source/assets/cloudformation/cudos/data-exports-aggregation.yaml deleted file mode 100644 index e5caa7f..0000000 --- a/modules/source/assets/cloudformation/cudos/data-exports-aggregation.yaml +++ /dev/null @@ -1,1564 +0,0 @@ -# -## https://raw.githubusercontent.com/aws-samples/aws-cudos-framework-deployment/refs/heads/main/cfn-templates/data-exports-aggregation.yaml -# -AWSTemplateFormatVersion: "2010-09-09" -Description: AWS Billing Data Export Aggregation v0.1.4 -Metadata: - AWS::CloudFormation::Interface: - ParameterGroups: - - Label: - default: "Common Parameters Configuration" - Parameters: - - DestinationAccountId - - ResourcePrefix - - ManageCUR2 - - ManageFOCUS - - ManageCOH - - Label: - default: "Parameters needed in Destination (Data Collection) Account only" - Parameters: - - SourceAccountIds - - Label: - default: "Technical Parameters. Please do not change." - Parameters: - - EnableSCAD - - RolePath - - ParameterLabels: - ManageCOH: - default: "Cost Optimization Recommendations" - ManageCUR2: - default: "CUR 2.0" - ManageFOCUS: - default: "FOCUS" - DestinationAccountId: - default: "Destination (Data Collection) Account Id" - ResourcePrefix: - default: "Resource Prefix" - SourceAccountIds: - default: "Source Account Ids (Comma separated list)" - EnableSCAD: - default: "Enable Split Cost Allocation Data (SCAD) in CUR 2.0" - -Parameters: - ## - # Common params - ## - DestinationAccountId: - Type: String - Description: "AWS Account Id where DataExport will be replicated to (Where you deploy CID Quicksight Dashboards or Data Collection)" - AllowedPattern: '\d{12}' - ResourcePrefix: - Type: String - Default: "cid" - Description: "Prefix used for all named resources, including S3 Bucket. Must be the same in destination and source stacks" - MaxLength: 37 # = 63 - len('-123456789012-data-exports') - AllowedPattern: "^[a-z0-9]+[a-z0-9-]{1,61}[a-z0-9]+$" - ManageCUR2: - Type: String - Description: "" - AllowedValues: ["yes", "no"] - Default: "no" - ManageFOCUS: - Type: String - Description: "NOTE: you can have only one export of this type" - AllowedValues: ["yes", "no"] - Default: "no" - ManageCOH: - Type: String - Description: "NOTE: you must have Cost Optimization Hub Enabled" - AllowedValues: ["yes", "no"] - Default: "no" - - ## - # Destination specific params - ## - SourceAccountIds: - Type: String - AllowedPattern: "^((\\d{12})\\,?)*$" - Default: "" - Description: "Ex: 12345678912,98745612312,.... If you install all in the same account (Source=Destination) please put Destination Account Id first in the list. " - - ## - # Technical params - ## - EnableSCAD: - Type: String - Description: Whether to enable Split Cost Allocation Data (Scad). Set this to 'No', if you experience performance issues due to dataset size. - AllowedValues: ["yes", "no"] - Default: "yes" - RolePath: - Type: String - Default: "/" - Description: Path for roles where PermissionBoundaries can limit location - -Conditions: - EmptySourceAccountIds: !Equals [!Ref SourceAccountIds, ""] - IsDestinationAccount: - !Equals [!Ref DestinationAccountId, !Ref "AWS::AccountId"] - IsSourceAccount: - # it is Source account if it is not a destination or if it is a destination and it is listed in Source Accounts (as the list one). - # Unfortunately, there no 'Fn::Contains' in Conditions, so we need to request user setting Dest account as the first. - Fn::Or: - - !Not [!Condition IsDestinationAccount] - - !Equals [ - !Ref "AWS::AccountId", - !Select [0, !Split [",", !Sub "${SourceAccountIds},"]], - ] - RegionSupportsDataExportsViaCFN: # CFN supports DataExports only in us-east-1 and cn-northwest-1. Other regions must use lambda. - Fn::Or: - - !Equals [!Ref "AWS::Region", "us-east-1"] - - !Equals [!Ref "AWS::Region", "cn-northwest-1"] - ManageCUR2: !Equals [!Ref ManageCUR2, "yes"] - ManageFOCUS: !Equals [!Ref ManageFOCUS, "yes"] - ManageCOH: !Equals [!Ref ManageCOH, "yes"] - EnableSCAD: !Equals [!Ref EnableSCAD, "yes"] - DeployDataExport: - Fn::Or: - - !Condition ManageCUR2 - - !Condition ManageFOCUS - - !Condition ManageCOH - DeployCOHServiceRole: !And [!Condition IsSourceAccount, !Condition ManageCOH] - DeployCUR2ViaCFN: - !And [ - !Condition IsSourceAccount, - !Condition ManageCUR2, - !Condition RegionSupportsDataExportsViaCFN, - ] - DeployFOCUSViaCFN: - !And [ - !Condition IsSourceAccount, - !Condition ManageFOCUS, - !Condition RegionSupportsDataExportsViaCFN, - ] - DeployCOHViaCFN: - !And [ - !Condition IsSourceAccount, - !Condition ManageCOH, - !Condition RegionSupportsDataExportsViaCFN, - ] - DeployCUR2ViaLambda: - !And [ - !Condition IsSourceAccount, - !Condition ManageCUR2, - !Not [!Condition RegionSupportsDataExportsViaCFN], - ] - DeployFOCUSViaLambda: - !And [ - !Condition IsSourceAccount, - !Condition ManageFOCUS, - !Not [!Condition RegionSupportsDataExportsViaCFN], - ] - DeployCOHViaLambda: - !And [ - !Condition IsSourceAccount, - !Condition ManageCOH, - !Not [!Condition RegionSupportsDataExportsViaCFN], - ] - DeployCUR2Table: !And [!Condition IsDestinationAccount, !Condition ManageCUR2] - DeployFOCUSTable: - !And [!Condition IsDestinationAccount, !Condition ManageFOCUS] - DeployCOHTable: !And [!Condition IsDestinationAccount, !Condition ManageCOH] - DeployAnyExportViaLambda: - !Or [ - !Condition DeployCUR2ViaLambda, - !Condition DeployFOCUSViaLambda, - !Condition DeployCOHViaLambda, - ] - DeployAnyTable: - !Or [ - !Condition DeployFOCUSTable, - !Condition DeployCUR2Table, - !Condition DeployCOHTable, - ] - -Mappings: - DataExports: - #Mappings for storing values for different Data Exports tables - CUR2: - DefaultQuery: >- - SELECT bill_bill_type, bill_billing_entity, bill_billing_period_end_date, bill_billing_period_start_date, bill_invoice_id, bill_invoicing_entity, bill_payer_account_id, bill_payer_account_name, cost_category, discount, discount_bundled_discount, discount_total_discount, identity_line_item_id, identity_time_interval, line_item_availability_zone, line_item_blended_cost, line_item_blended_rate, line_item_currency_code, line_item_legal_entity, line_item_line_item_description, line_item_line_item_type, line_item_net_unblended_cost, line_item_net_unblended_rate, line_item_normalization_factor, line_item_normalized_usage_amount, line_item_operation, line_item_product_code, line_item_resource_id, line_item_tax_type, line_item_unblended_cost, line_item_unblended_rate, line_item_usage_account_id, line_item_usage_account_name, line_item_usage_amount, line_item_usage_end_date, line_item_usage_start_date, line_item_usage_type, pricing_currency, pricing_lease_contract_length, pricing_offering_class, pricing_public_on_demand_cost, pricing_public_on_demand_rate, pricing_purchase_option, pricing_rate_code, pricing_rate_id, pricing_term, pricing_unit, product, product_comment, product_fee_code, product_fee_description, product_from_location, product_from_location_type, product_from_region_code, product_instance_family, product_instance_type, product_instancesku, product_location, product_location_type, product_operation, product_pricing_unit, product_product_family, product_region_code, product_servicecode, product_sku, product_to_location, product_to_location_type, product_to_region_code, product_usagetype, reservation_amortized_upfront_cost_for_usage, reservation_amortized_upfront_fee_for_billing_period, reservation_availability_zone, reservation_effective_cost, reservation_end_time, reservation_modification_status, reservation_net_amortized_upfront_cost_for_usage, reservation_net_amortized_upfront_fee_for_billing_period, reservation_net_effective_cost, reservation_net_recurring_fee_for_usage, reservation_net_unused_amortized_upfront_fee_for_billing_period, reservation_net_unused_recurring_fee, reservation_net_upfront_value, reservation_normalized_units_per_reservation, reservation_number_of_reservations, reservation_recurring_fee_for_usage, reservation_reservation_a_r_n, reservation_start_time, reservation_subscription_id, reservation_total_reserved_normalized_units, reservation_total_reserved_units, reservation_units_per_reservation, reservation_unused_amortized_upfront_fee_for_billing_period, reservation_unused_normalized_unit_quantity, reservation_unused_quantity, reservation_unused_recurring_fee, reservation_upfront_value, resource_tags, savings_plan_amortized_upfront_commitment_for_billing_period, savings_plan_end_time, savings_plan_instance_type_family, savings_plan_net_amortized_upfront_commitment_for_billing_period, savings_plan_net_recurring_commitment_for_billing_period, savings_plan_net_savings_plan_effective_cost, savings_plan_offering_type, savings_plan_payment_option, savings_plan_purchase_term, savings_plan_recurring_commitment_for_billing_period, savings_plan_region, savings_plan_savings_plan_a_r_n, savings_plan_savings_plan_effective_cost, savings_plan_savings_plan_rate, savings_plan_start_time, savings_plan_total_commitment_to_date, savings_plan_used_commitment - FROM COST_AND_USAGE_REPORT - SCADQuery: >- - SELECT bill_bill_type, bill_billing_entity, bill_billing_period_end_date, bill_billing_period_start_date, bill_invoice_id, bill_invoicing_entity, bill_payer_account_id, bill_payer_account_name, cost_category, discount, discount_bundled_discount, discount_total_discount, identity_line_item_id, identity_time_interval, line_item_availability_zone, line_item_blended_cost, line_item_blended_rate, line_item_currency_code, line_item_legal_entity, line_item_line_item_description, line_item_line_item_type, line_item_net_unblended_cost, line_item_net_unblended_rate, line_item_normalization_factor, line_item_normalized_usage_amount, line_item_operation, line_item_product_code, line_item_resource_id, line_item_tax_type, line_item_unblended_cost, line_item_unblended_rate, line_item_usage_account_id, line_item_usage_account_name, line_item_usage_amount, line_item_usage_end_date, line_item_usage_start_date, line_item_usage_type, pricing_currency, pricing_lease_contract_length, pricing_offering_class, pricing_public_on_demand_cost, pricing_public_on_demand_rate, pricing_purchase_option, pricing_rate_code, pricing_rate_id, pricing_term, pricing_unit, product, product_comment, product_fee_code, product_fee_description, product_from_location, product_from_location_type, product_from_region_code, product_instance_family, product_instance_type, product_instancesku, product_location, product_location_type, product_operation, product_pricing_unit, product_product_family, product_region_code, product_servicecode, product_sku, product_to_location, product_to_location_type, product_to_region_code, product_usagetype, reservation_amortized_upfront_cost_for_usage, reservation_amortized_upfront_fee_for_billing_period, reservation_availability_zone, reservation_effective_cost, reservation_end_time, reservation_modification_status, reservation_net_amortized_upfront_cost_for_usage, reservation_net_amortized_upfront_fee_for_billing_period, reservation_net_effective_cost, reservation_net_recurring_fee_for_usage, reservation_net_unused_amortized_upfront_fee_for_billing_period, reservation_net_unused_recurring_fee, reservation_net_upfront_value, reservation_normalized_units_per_reservation, reservation_number_of_reservations, reservation_recurring_fee_for_usage, reservation_reservation_a_r_n, reservation_start_time, reservation_subscription_id, reservation_total_reserved_normalized_units, reservation_total_reserved_units, reservation_units_per_reservation, reservation_unused_amortized_upfront_fee_for_billing_period, reservation_unused_normalized_unit_quantity, reservation_unused_quantity, reservation_unused_recurring_fee, reservation_upfront_value, resource_tags, savings_plan_amortized_upfront_commitment_for_billing_period, savings_plan_end_time, savings_plan_instance_type_family, savings_plan_net_amortized_upfront_commitment_for_billing_period, savings_plan_net_recurring_commitment_for_billing_period, savings_plan_net_savings_plan_effective_cost, savings_plan_offering_type, savings_plan_payment_option, savings_plan_purchase_term, savings_plan_recurring_commitment_for_billing_period, savings_plan_region, savings_plan_savings_plan_a_r_n, savings_plan_savings_plan_effective_cost, savings_plan_savings_plan_rate, savings_plan_start_time, savings_plan_total_commitment_to_date, savings_plan_used_commitment, split_line_item_actual_usage, split_line_item_net_split_cost, split_line_item_net_unused_cost, split_line_item_parent_resource_id, split_line_item_public_on_demand_split_cost, split_line_item_public_on_demand_unused_cost, split_line_item_reserved_usage, split_line_item_split_cost, split_line_item_split_usage, split_line_item_split_usage_ratio, split_line_item_unused_cost - FROM COST_AND_USAGE_REPORT - FOCUS: - DefaultQuery: >- - SELECT AvailabilityZone, BilledCost, BillingAccountId, BillingAccountName, BillingCurrency, BillingPeriodEnd, BillingPeriodStart, ChargeCategory, ChargeClass, ChargeDescription, ChargeFrequency, ChargePeriodEnd, ChargePeriodStart, CommitmentDiscountCategory, CommitmentDiscountId, CommitmentDiscountName, CommitmentDiscountType, CommitmentDiscountStatus, ConsumedQuantity, ConsumedUnit, ContractedCost, ContractedUnitPrice, EffectiveCost, InvoiceIssuerName, ListCost, ListUnitPrice, PricingCategory, PricingQuantity, PricingUnit, ProviderName, PublisherName, RegionId, RegionName, ResourceId, ResourceName, ResourceType, ServiceCategory, ServiceName, SkuId, SkuPriceId, SubAccountId, SubAccountName, Tags, x_CostCategories, x_Discounts, x_Operation, x_ServiceCode, x_UsageType - FROM FOCUS_1_0_AWS_PREVIEW - COH: - DefaultQuery: >- - SELECT account_id, action_type, currency_code, current_resource_details, current_resource_summary, current_resource_type, estimated_monthly_cost_after_discount, estimated_monthly_cost_before_discount, estimated_monthly_savings_after_discount, estimated_monthly_savings_before_discount, estimated_savings_percentage_after_discount, estimated_savings_percentage_before_discount, implementation_effort, last_refresh_timestamp, recommendation_id, recommendation_lookback_period_in_days, recommendation_source, recommended_resource_details, recommended_resource_summary, recommended_resource_type, region, resource_arn, restart_needed, rollback_possible, tags - FROM COST_OPTIMIZATION_RECOMMENDATIONS - -Resources: - ########################################################################### - # Destination Account Resources - ########################################################################### - - DestinationS3: - Type: AWS::S3::Bucket - Condition: IsDestinationAccount - DeletionPolicy: Retain - UpdateReplacePolicy: Retain - Properties: - BucketName: !Sub "${ResourcePrefix}-${AWS::AccountId}-data-exports" - - ## Uncomment following lines to enable bucket logging if needed. Please be careful with the cost of logging. - # LoggingConfiguration: - # DestinationBucketName: REPLACE_WITH_YOUR_LOGGING_BUCKET - # LogFilePrefix: REPLACE_WITH_YOUR_PREFIX - - BucketEncryption: - ServerSideEncryptionConfiguration: - - ServerSideEncryptionByDefault: - SSEAlgorithm: AES256 ## Use AWS managed KMS - ## If you need Customer managed KMS key, yoy can do that using following parameters: - # SSEAlgorithm: aws:kms - # KMSMasterKeyID: "REPLACE_WITH_YOUR_KEY_ARN" - AccessControl: BucketOwnerFullControl - OwnershipControls: - Rules: - - ObjectOwnership: BucketOwnerEnforced - PublicAccessBlockConfiguration: - BlockPublicAcls: true - BlockPublicPolicy: true - IgnorePublicAcls: true - RestrictPublicBuckets: true - VersioningConfiguration: - Status: Enabled - LifecycleConfiguration: - Rules: - - Id: Object&Version Expiration - Status: Enabled - NoncurrentVersionExpirationInDays: 32 # 1 month - Metadata: - cfn_nag: - rules_to_suppress: - - id: "W35" - reason: "Data buckets would generate too much logs" - cfn-lint: - config: - ignore_checks: - - W3045 # Need to use AccessControl for replication - - DestinationS3BucketPolicy: - Type: "AWS::S3::BucketPolicy" - Condition: IsDestinationAccount - DeletionPolicy: Retain - UpdateReplacePolicy: Delete - Properties: - Bucket: - Ref: DestinationS3 - PolicyDocument: - Id: AllowReplication - Version: "2012-10-17" - Statement: - - Sid: AllowTLS12Only - Effect: Deny - Principal: "*" - Action: s3:* - Resource: - - !Sub "arn:${AWS::Partition}:s3:::${DestinationS3}" - - !Sub "arn:${AWS::Partition}:s3:::${DestinationS3}/*" - Condition: - NumericLessThan: - s3:TlsVersion: 1.2 - - Sid: AllowOnlyHTTPS - Effect: Deny - Principal: "*" - Action: s3:* - Resource: - - !Sub "arn:${AWS::Partition}:s3:::${DestinationS3}" - - !Sub "arn:${AWS::Partition}:s3:::${DestinationS3}/*" - Condition: - Bool: - aws:SecureTransport: false - - Sid: AllowReplicationWrite - Effect: Allow - Principal: - AWS: - Fn::If: - - EmptySourceAccountIds - - !Ref AWS::AccountId - - !Split [",", !Ref SourceAccountIds] - Action: - - s3:ReplicateDelete - - s3:ReplicateObject - Resource: !Sub "arn:${AWS::Partition}:s3:::${DestinationS3}/*" - - Sid: AllowReplicationRead - Effect: Allow - Principal: - AWS: - Fn::If: - - EmptySourceAccountIds - - !Ref AWS::AccountId - - !Split [",", !Ref SourceAccountIds] - Action: - - s3:ListBucket - - s3:ListBucketVersions - - s3:GetBucketVersioning - - s3:PutBucketVersioning - Resource: !Sub "arn:${AWS::Partition}:s3:::${DestinationS3}" - - SourceS3: - Type: AWS::S3::Bucket - Condition: DeployDataExport - DeletionPolicy: Delete - UpdateReplacePolicy: Delete - Properties: - BucketName: !Sub ${ResourcePrefix}-${AWS::AccountId}-data-local - - ## Uncomment following lines to enable bucket logging if needed. Please be careful with the cost of logging. - # LoggingConfiguration: - # DestinationBucketName: REPLACE_WITH_YOUR_LOGGING_BUCKET - # LogFilePrefix: REPLACE_WITH_YOUR_PREFIX - - BucketEncryption: - ServerSideEncryptionConfiguration: - - ServerSideEncryptionByDefault: - SSEAlgorithm: AES256 ## Use AWS managed KMS - ## If you need Customer managed KMS key, yoy can do that using following parameters: - # SSEAlgorithm: aws:kms - # KMSMasterKeyID: "REPLACE_WITH_YOUR_KEY_ARN" - AccessControl: BucketOwnerFullControl - OwnershipControls: - Rules: - - ObjectOwnership: BucketOwnerEnforced - PublicAccessBlockConfiguration: - BlockPublicAcls: true - BlockPublicPolicy: true - IgnorePublicAcls: true - RestrictPublicBuckets: true - VersioningConfiguration: - Status: Enabled - ReplicationConfiguration: - Role: !GetAtt ReplicationRole.Arn - Rules: - - Destination: - Bucket: !Sub "arn:${AWS::Partition}:s3:::${ResourcePrefix}-${DestinationAccountId}-data-exports" - StorageClass: STANDARD - Id: ReplicateCUR2Data - Prefix: !Sub "cur2/${AWS::AccountId}/${ResourcePrefix}-cur2/data/" # Hardcoded export name - Status: Enabled - - Destination: - Bucket: !Sub "arn:${AWS::Partition}:s3:::${ResourcePrefix}-${DestinationAccountId}-data-exports" - StorageClass: STANDARD - Id: ReplicateFOCUSData - Prefix: !Sub "focus/${AWS::AccountId}/${ResourcePrefix}-focus/data/" # Hardcoded export name - Status: Enabled - - Destination: - Bucket: !Sub "arn:${AWS::Partition}:s3:::${ResourcePrefix}-${DestinationAccountId}-data-exports" - StorageClass: STANDARD - Id: ReplicateCOHData - Prefix: !Sub "coh/${AWS::AccountId}/${ResourcePrefix}-coh/data/" # Hardcoded export name - Status: Enabled - LifecycleConfiguration: - Rules: - - Id: Object&Version Expiration - Status: Enabled - NoncurrentVersionExpirationInDays: 32 # 1 month - ExpirationInDays: 64 # 2 months - Metadata: - cfn_nag: - rules_to_suppress: - - id: "W35" - reason: "Data buckets would generate too much logs" - cfn-lint: - config: - ignore_checks: - - W3045 # Need to use AccessControl for replication - - SourceS3BucketPolicy: - Type: "AWS::S3::BucketPolicy" - Condition: DeployDataExport - DeletionPolicy: Delete - UpdateReplacePolicy: Delete - Properties: - Bucket: !Ref SourceS3 - PolicyDocument: - Id: AllowBillingReadAndWrite - Version: "2012-10-17" - Statement: - - Sid: AllowTLS12Only - Effect: Deny - Principal: "*" - Action: s3:* - Resource: - - !Sub "arn:${AWS::Partition}:s3:::${SourceS3}" - - !Sub "arn:${AWS::Partition}:s3:::${SourceS3}/*" - Condition: - NumericLessThan: - s3:TlsVersion: 1.2 - - Sid: AllowOnlyHTTPS - Effect: Deny - Principal: "*" - Action: s3:* - Resource: - - !Sub "arn:${AWS::Partition}:s3:::${SourceS3}" - - !Sub "arn:${AWS::Partition}:s3:::${SourceS3}/*" - Condition: - Bool: - aws:SecureTransport: false - - Sid: AllowBillingReadAndWrite - Effect: Allow - Principal: - Service: bcm-data-exports.amazonaws.com - Action: - - s3:GetBucketAcl - - s3:GetBucketPolicy - - s3:PutObject - Resource: - - !Sub "arn:${AWS::Partition}:s3:::${SourceS3}" - - !Sub "arn:${AWS::Partition}:s3:::${SourceS3}/*" - Condition: - StringEquals: - aws:SourceAccount: !Ref AWS::AccountId - - ReplicationRole: - Condition: DeployDataExport - Type: AWS::IAM::Role - Properties: - Path: !Sub /${ResourcePrefix}/ - AssumeRolePolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Principal: - Service: - - "s3.amazonaws.com" - Action: - - "sts:AssumeRole" - Policies: - - PolicyName: ReplicationPolicy - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: - - s3:GetReplicationConfiguration - - s3:ListBucket - Resource: !Sub "arn:${AWS::Partition}:s3:::${ResourcePrefix}-${AWS::AccountId}-data-local" - - Effect: Allow - Action: - - s3:GetObjectVersionForReplication - - s3:GetObjectVersionAcl - - s3:GetObjectVersionTagging - Resource: !Sub "arn:${AWS::Partition}:s3:::${ResourcePrefix}-${AWS::AccountId}-data-local/*" - - Effect: Allow - Action: - - s3:ReplicateObject - - s3:ReplicateDelete - - s3:ReplicateTags - Resource: !Sub "arn:${AWS::Partition}:s3:::${ResourcePrefix}-${DestinationAccountId}-data-exports/*/${AWS::AccountId}/*" - - # CUR2 - - ## Deploy Data Export natively via CFN resource in regions that support native CFN - LocalCUR2viaCFN: - Type: AWS::BCMDataExports::Export - Condition: DeployCUR2ViaCFN - DependsOn: - - SourceS3BucketPolicy - Properties: - Export: - DataQuery: - QueryStatement: - !If [ - EnableSCAD, - !FindInMap [DataExports, CUR2, SCADQuery], - !FindInMap [DataExports, CUR2, DefaultQuery], - ] - TableConfigurations: - COST_AND_USAGE_REPORT: - TIME_GRANULARITY: "HOURLY" - INCLUDE_RESOURCES: "TRUE" - INCLUDE_MANUAL_DISCOUNT_COMPATIBILITY: "FALSE" - INCLUDE_SPLIT_COST_ALLOCATION_DATA: - !If [EnableSCAD, "TRUE", "FALSE"] - Description: "CUR 2.0 export for aggregation in CID" - DestinationConfigurations: - S3Destination: - S3Bucket: !Ref SourceS3 - S3Prefix: !Sub "cur2/${AWS::AccountId}" # Explicitly hardcode name of folder (it will be the same as table name) - S3Region: !Ref AWS::Region - S3OutputConfigurations: - Overwrite: "OVERWRITE_REPORT" - Format: "PARQUET" - Compression: "PARQUET" - OutputType: "CUSTOM" - Name: !Sub "${ResourcePrefix}-cur2" - RefreshCadence: - Frequency: "SYNCHRONOUS" - - # Deploy Data Export via Lambda due to missing CFN resource definition AWS::BCMDataExports::Export outside us-east-1 or cn-northwest-1 - LocalCUR2viaCustomResource: - Type: Custom::CUR2Creator - Condition: DeployCUR2ViaLambda - Properties: - ServiceToken: !GetAtt CidDataExportCreatorLambda.Arn - BucketPolicyWait: !Ref SourceS3BucketPolicy - Export: - DataQuery: - QueryStatement: - !If [ - EnableSCAD, - !FindInMap [DataExports, CUR2, SCADQuery], - !FindInMap [DataExports, CUR2, DefaultQuery], - ] - TableConfigurations: - COST_AND_USAGE_REPORT: - TIME_GRANULARITY: "HOURLY" - INCLUDE_RESOURCES: "TRUE" - INCLUDE_MANUAL_DISCOUNT_COMPATIBILITY: "FALSE" - INCLUDE_SPLIT_COST_ALLOCATION_DATA: - !If [EnableSCAD, "TRUE", "FALSE"] - Description: "CUR 2.0 export for aggregation in CID" - DestinationConfigurations: - S3Destination: - S3Bucket: !Ref SourceS3 - S3Prefix: !Sub "cur2/${AWS::AccountId}" # Explicitly hardcode name of folder (it will be the same as table name) - S3Region: !Ref AWS::Region - S3OutputConfigurations: - Overwrite: "OVERWRITE_REPORT" - Format: "PARQUET" - Compression: "PARQUET" - OutputType: "CUSTOM" - Name: !Sub "${ResourcePrefix}-cur2" - RefreshCadence: - Frequency: "SYNCHRONOUS" - - # FOCUS - - ## Deploy Data Export natively via CFN resource in regions that support native CFN - LocalFOCUSviaCFN: - Type: AWS::BCMDataExports::Export - Condition: DeployFOCUSViaCFN - DependsOn: - - SourceS3BucketPolicy - Properties: - Export: - DataQuery: - QueryStatement: !FindInMap [DataExports, FOCUS, DefaultQuery] - Description: "FOCUS export for aggregation in CID" - DestinationConfigurations: - S3Destination: - S3Bucket: !Ref SourceS3 - S3Prefix: !Sub "focus/${AWS::AccountId}" # Explicitly hardcode name of folder (it will be the same as table name) - S3Region: !Ref AWS::Region - S3OutputConfigurations: - Overwrite: "OVERWRITE_REPORT" - Format: "PARQUET" - Compression: "PARQUET" - OutputType: "CUSTOM" - Name: !Sub "${ResourcePrefix}-focus" - RefreshCadence: - Frequency: "SYNCHRONOUS" - - # Deploy Data Export via Lambda due to missing CFN resource definition AWS::BCMDataExports::Export outside us-east-1 or cn-northwest-1 - LocalFOCUSviaCustomResource: - Type: Custom::CUR2Creator - Condition: DeployFOCUSViaLambda - Properties: - ServiceToken: !GetAtt CidDataExportCreatorLambda.Arn - BucketPolicyWait: !Ref SourceS3BucketPolicy - Export: - DataQuery: - QueryStatement: !FindInMap [DataExports, FOCUS, DefaultQuery] - Description: "FOCUS export for aggregation in CID" - DestinationConfigurations: - S3Destination: - S3Bucket: !Ref SourceS3 - S3Prefix: !Sub "focus/${AWS::AccountId}" # Explicitly hardcode name of folder (it will be the same as table name) - S3Region: !Ref AWS::Region - S3OutputConfigurations: - Overwrite: "OVERWRITE_REPORT" - Format: "PARQUET" - Compression: "PARQUET" - OutputType: "CUSTOM" - Name: !Sub "${ResourcePrefix}-focus" - RefreshCadence: - Frequency: "SYNCHRONOUS" - - # COH - - ## Deploy Data Export natively via CFN resource in regions that support native CFN - LocalCOHviaCFN: - Type: AWS::BCMDataExports::Export - Condition: DeployCOHViaCFN - DependsOn: - - SourceS3BucketPolicy - - CreateServiceLinkedRoleCustomResource - Properties: - Export: - DataQuery: - QueryStatement: !FindInMap [DataExports, COH, DefaultQuery] - TableConfigurations: - COST_OPTIMIZATION_RECOMMENDATIONS: - FILTER: "{}" - INCLUDE_ALL_RECOMMENDATIONS: "TRUE" - Description: "Cost Optimization Hub Recommendations export for aggregation in CID" - DestinationConfigurations: - S3Destination: - S3Bucket: !Ref SourceS3 - S3Prefix: !Sub "coh/${AWS::AccountId}" # Explicitly hardcode name of folder (it will be the same as table name) - S3Region: !Ref AWS::Region - S3OutputConfigurations: - Overwrite: "OVERWRITE_REPORT" - Format: "PARQUET" - Compression: "PARQUET" - OutputType: "CUSTOM" - Name: !Sub "${ResourcePrefix}-coh" - RefreshCadence: - Frequency: "SYNCHRONOUS" - - # Deploy Data Export via Lambda due to missing CFN resource definition AWS::BCMDataExports::Export outside us-east-1 or cn-northwest-1 - LocalCOHviaCustomResource: - Type: Custom::CUR2Creator - DependsOn: - - CreateServiceLinkedRoleCustomResource - Condition: DeployCOHViaLambda - Properties: - ServiceToken: !GetAtt CidDataExportCreatorLambda.Arn - BucketPolicyWait: !Ref SourceS3BucketPolicy - Export: - DataQuery: - QueryStatement: !FindInMap [DataExports, COH, DefaultQuery] - TableConfigurations: - COST_OPTIMIZATION_RECOMMENDATIONS: - FILTER: "{}" - INCLUDE_ALL_RECOMMENDATIONS: "TRUE" - Description: "Cost Optimization Hub Recommendations export for aggregation in CID" - DestinationConfigurations: - S3Destination: - S3Bucket: !Ref SourceS3 - S3Prefix: !Sub "coh/${AWS::AccountId}" # Explicitly hardcode name of folder (it will be the same as table name) - S3Region: !Ref AWS::Region - S3OutputConfigurations: - Overwrite: "OVERWRITE_REPORT" - Format: "PARQUET" - Compression: "PARQUET" - OutputType: "CUSTOM" - Name: !Sub "${ResourcePrefix}-coh" - RefreshCadence: - Frequency: "SYNCHRONOUS" - - # COH export requires a service linked role to be created BUT this role can be already created. Thus we need to create it via custom resource - LambdaServiceLinkedRoleExecutionRole: - Type: "AWS::IAM::Role" - Condition: DeployCOHServiceRole - Properties: - RoleName: !Sub "${AWS::StackName}-LambdaExecutionRole" - AssumeRolePolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Principal: - Service: - - lambda.amazonaws.com - Action: sts:AssumeRole - Policies: - - PolicyName: !Sub "${AWS::StackName}-LambdaPolicy" - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: - - logs:CreateLogGroup - - logs:CreateLogStream - - logs:PutLogEvents - Resource: - - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-CreateServiceLinkedRoleFunction" - - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-CreateServiceLinkedRoleFunction:*" - - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-CreateServiceLinkedRoleFunction:*:*" - - Effect: Allow - Action: - - iam:GetRole - - iam:CreateServiceLinkedRole - - iam:DeleteServiceLinkedRole - - iam:GetServiceLinkedRoleDeletionStatus - Resource: !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/aws-service-role/bcm-data-exports.amazonaws.com/AWSServiceRoleForBCMDataExports" - - Effect: Allow - Action: - - cost-optimization-hub:GetPreferences - Resource: "*" # Cannot restrict this - - Metadata: - cfn_nag: - rules_to_suppress: - - id: "W28" - reason: "Need an explicit name for reference" - - id: "W11" - reason: "Some COH resources cannot be restricted" - - CreateServiceLinkedRoleFunction: - Type: "AWS::Lambda::Function" - Condition: DeployCOHServiceRole - Properties: - FunctionName: !Sub "${ResourcePrefix}-CreateServiceLinkedRoleFunction" - Handler: index.handler - MemorySize: 128 - Runtime: python3.12 - Timeout: 15 - Role: !GetAtt LambdaServiceLinkedRoleExecutionRole.Arn - Code: - ZipFile: | - import json - import time - import boto3 - import cfnresponse - - def handler(event, context): - print(json.dumps(event)) - coh = boto3.client('cost-optimization-hub', region_name='us-east-1') - iam = boto3.client('iam') - try: - if event['RequestType'] in ['Create', 'Update']: - - print("Make sure CO hub is activated") - try: - coh.get_preferences() - except Exception as e: - if 'AWS account is not enrolled for recommendations' in str(e): - raise Exception('AWS account is not enrolled for recommendations. Please activate Cost Optimization Hub.') - raise - - print("Creating service linked role") - iam.create_service_linked_role( - AWSServiceName='bcm-data-exports.amazonaws.com', - Description='Service-linked role for bcm-data-exports.amazonaws.com' - ) - - print("Waiting for the role to be created") - for i in range(60): - try: - iam.get_role(RoleName='AWSServiceRoleForBCMDataExports') - print("Role is created") - break - except iam.exceptions.NoSuchEntityException: - time.sleep(1) - - print("Additional wait to make sure the role is available") - time.sleep(30) - - cfnresponse.send(event, context, cfnresponse.SUCCESS, {}) - - except Exception as e: - if 'has been taken in this account' in str(e): - print('the role AWSServiceRoleForBCMDataExports already exist') - cfnresponse.send(event, context, cfnresponse.SUCCESS, {}) - else: - print(e) - cfnresponse.send(event, context, cfnresponse.FAILED, {}, reason=str(e)) - Metadata: - cfn_nag: - rules_to_suppress: - - id: "W89" - reason: "This Lambda does not require VPC" - - id: "W92" - reason: "One Time execution. No need for ReservedConcurrentExecutions" - - CreateServiceLinkedRoleCustomResource: - Condition: DeployCOHServiceRole - Type: "AWS::CloudFormation::CustomResource" - Properties: - ServiceToken: !GetAtt CreateServiceLinkedRoleFunction.Arn - - ########################################################################### - # Lambda DataExport Creator: used to create DataExport from outside us-east-1 or cn-northwest-1 - ########################################################################### - - CidDataExportCreatorLambdaRole: - Type: AWS::IAM::Role - Condition: DeployAnyExportViaLambda - Properties: - Path: !Sub /${ResourcePrefix}/ - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Principal: - Service: - - lambda.amazonaws.com - Action: - - sts:AssumeRole - Policies: - - PolicyName: "ExecutionDefault" - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: - - logs:CreateLogStream - - logs:PutLogEvents - - logs:CreateLogGroup - Resource: - - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-DataExportCreator" - - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-DataExportCreator:*" - - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-DataExportCreator:*:*" - - PolicyName: "AllowDataExports" - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: - - bcm-data-exports:CreateExport - - bcm-data-exports:UpdateExport - - bcm-data-exports:DeleteExport - Resource: !Sub "arn:${AWS::Partition}:bcm-data-exports:*:${AWS::AccountId}:*" - - Effect: Allow - Action: - - cur:PutReportDefinition #need this permission for bcm-data-exports to work - Resource: !Sub "arn:${AWS::Partition}:cur:*:${AWS::AccountId}:*" - - Effect: Allow - Action: - - cost-optimization-hub:GetRecommendation #need this permission for bcm-data-exports to work - - cost-optimization-hub:ListRecommendations - Resource: "arn:aws:cost-optimization-hub:*" - - CidDataExportCreatorLambda: - Type: AWS::Lambda::Function - Condition: DeployAnyExportViaLambda - Properties: - Runtime: python3.12 - FunctionName: !Sub ${ResourcePrefix}-DataExportCreator - Handler: index.lambda_handler - MemorySize: 128 - Role: !GetAtt CidDataExportCreatorLambdaRole.Arn - Timeout: 15 - Code: - ZipFile: | - import os - import json - import uuid - - import boto3 - import cfnresponse - - # DataExports only exist in us-east-1 and cn-northwest-1 regions - region = 'us-east-1' if not os.environ['AWS_REGION'].startswith('cn-') else 'cn-northwest-1' - - client = boto3.client('bcm-data-exports', region_name=region) - - def lambda_handler(event, context): - - print(json.dumps(event)) - reason = "" - - try: - export = event['ResourceProperties']['Export'] - export_name = event['ResourceProperties']['Export']['Name'] - - if event['RequestType'] == 'Create': - res = client.create_export(Export=export) - print('created:', json.dumps(res)) - elif event['RequestType'] == 'Update': - old_export_name = event['OldResourceProperties']['Export']['Name'] - if export["Name"] != old_export_name: - res = client.create_export(Export=export) - print('created:', json.dumps(res)) - try: - res = client.delete_export(Name=old_export_name) - print('deleted:', json.dumps(res)) - except: - pass # Do not block deletion - else: - res = client.update_export(Name=old_export_name, Export=export) - print('updated:', json.dumps(res)) - elif event['RequestType'] == 'Delete': - try: - res = client.delete_export(Name=old_export_name) - print('deleted:', json.dumps(res)) - except: - pass # Do not block deletion - else: - raise Exception("Unknown operation: " + event['RequestType']) - - except Exception as e: - reason = str(e) - print(e) - finally: - physicalResourceId = event.get('ResourceProperties',{}).get('Export', {}).get('Name', None) or str(uuid.uuid1()) - if reason: - print("FAILURE") - cfnresponse.send(event, context, cfnresponse.FAILED, {}, physicalResourceId, reason=reason) - else: - print("SUCCESS") - cfnresponse.send(event, context, cfnresponse.SUCCESS, {}, physicalResourceId) - Metadata: - cfn_nag: - rules_to_suppress: - - id: "W89" - reason: "This Lambda does not require VPC" - - id: "W92" - reason: "One Time execution. No need for ReservedConcurrentExecutions" - - CIDDatabase: - Type: AWS::Glue::Database - Condition: DeployAnyTable - Properties: - DatabaseInput: - Name: !Join ["_", !Split ["-", !Sub "${ResourcePrefix}_data_export"]] # replace '-' to '_' - CatalogId: !Sub "${AWS::AccountId}" - - ########################################################################### - # CUR2 - ########################################################################### - - CURTable: # Initial creation of table. it will be updated by crawler later - Type: AWS::Glue::Table - Condition: DeployCUR2Table - Properties: - CatalogId: !Sub "${AWS::AccountId}" - DatabaseName: !Ref CIDDatabase - TableInput: - Name: cur2 - Owner: owner - Retention: 0 - TableType: EXTERNAL_TABLE - Parameters: - compressionType: none - classification: parquet - UPDATED_BY_CRAWLER: !Ref CURCrawler - StorageDescriptor: - BucketColumns: [] - Compressed: false - Location: !Sub "s3://${DestinationS3}/cur2/" - NumberOfBuckets: -1 - InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat - OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat - SerdeInfo: - Parameters: - serialization.format: "1" - SerializationLibrary: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe - StoredAsSubDirectories: false - Columns: # All fields required for CID - - { "Name": "bill_bill_type", "Type": "string" } - - { "Name": "bill_billing_entity", "Type": "string" } - - { "Name": "bill_billing_period_end_date", "Type": "timestamp" } - - { "Name": "bill_billing_period_start_date", "Type": "timestamp" } - - { "Name": "bill_invoice_id", "Type": "string" } - - { "Name": "bill_payer_account_id", "Type": "string" } - - { "Name": "bill_payer_account_name", "Type": "string" } - - { "Name": "cost_category", "Type": "map" } - - { "Name": "discount", "Type": "map" } - - { "Name": "identity_line_item_id", "Type": "string" } - - { "Name": "identity_time_interval", "Type": "string" } - - { "Name": "line_item_availability_zone", "Type": "string" } - - { "Name": "line_item_legal_entity", "Type": "string" } - - { "Name": "line_item_line_item_description", "Type": "string" } - - { "Name": "line_item_line_item_type", "Type": "string" } - - { "Name": "line_item_operation", "Type": "string" } - - { "Name": "line_item_product_code", "Type": "string" } - - { "Name": "line_item_resource_id", "Type": "string" } - - { "Name": "line_item_unblended_cost", "Type": "double" } - - { "Name": "line_item_usage_account_id", "Type": "string" } - - { "Name": "line_item_usage_account_name", "Type": "string" } - - { "Name": "line_item_usage_amount", "Type": "double" } - - { "Name": "line_item_usage_end_date", "Type": "timestamp" } - - { "Name": "line_item_usage_start_date", "Type": "timestamp" } - - { "Name": "line_item_usage_type", "Type": "string" } - - { "Name": "pricing_lease_contract_length", "Type": "string" } - - { "Name": "pricing_offering_class", "Type": "string" } - - { "Name": "pricing_public_on_demand_cost", "Type": "double" } - - { "Name": "pricing_purchase_option", "Type": "string" } - - { "Name": "pricing_term", "Type": "string" } - - { "Name": "pricing_unit", "Type": "string" } - - { "Name": "product", "Type": "map" } - - { "Name": "product_from_location", "Type": "string" } - - { "Name": "product_instance_type", "Type": "string" } - - { "Name": "product_product_family", "Type": "string" } - - { "Name": "product_servicecode", "Type": "string" } - - { "Name": "product_to_location", "Type": "string" } - - { - "Name": "reservation_amortized_upfront_fee_for_billing_period", - "Type": "double", - } - - { "Name": "reservation_effective_cost", "Type": "double" } - - { "Name": "reservation_end_time", "Type": "string" } - - { "Name": "reservation_reservation_a_r_n", "Type": "string" } - - { "Name": "reservation_start_time", "Type": "string" } - - { - "Name": "reservation_unused_amortized_upfront_fee_for_billing_period", - "Type": "double", - } - - { "Name": "reservation_unused_recurring_fee", "Type": "double" } - - { "Name": "resource_tags", "Type": "map" } - - { - "Name": "savings_plan_amortized_upfront_commitment_for_billing_period", - "Type": "double", - } - - { "Name": "savings_plan_end_time", "Type": "string" } - - { "Name": "savings_plan_offering_type", "Type": "string" } - - { "Name": "savings_plan_payment_option", "Type": "string" } - - { "Name": "savings_plan_purchase_term", "Type": "string" } - - { "Name": "savings_plan_savings_plan_a_r_n", "Type": "string" } - - { - "Name": "savings_plan_savings_plan_effective_cost", - "Type": "double", - } - - { "Name": "savings_plan_start_time", "Type": "string" } - - { - "Name": "savings_plan_total_commitment_to_date", - "Type": "double", - } - - { "Name": "savings_plan_used_commitment", "Type": "double" } - - { "Name": "split_line_item_parent_resource_id", "Type": "string" } - - { "Name": "split_line_item_reserved_usage", "Type": "double" } - - { "Name": "split_line_item_actual_usage", "Type": "double" } - - { "Name": "split_line_item_split_usage", "Type": "double" } - - { "Name": "split_line_item_split_usage_ratio", "Type": "double" } - - { "Name": "split_line_item_split_cost", "Type": "double" } - - { "Name": "split_line_item_unused_cost", "Type": "double" } - - { "Name": "split_line_item_net_split_cost", "Type": "double" } - - { "Name": "split_line_item_net_unused_cost", "Type": "double" } - - { - "Name": "split_line_item_public_on_demand_split_cost", - "Type": "double", - } - - { - "Name": "split_line_item_public_on_demand_unused_cost", - "Type": "double", - } - PartitionKeys: - - { "Name": "source_account_id", "Type": "string" } - - { "Name": "report_name", "Type": "string" } - - { "Name": "data", "Type": "string" } - - { "Name": "billing_period", "Type": "string" } - - CURCrawler: - Type: AWS::Glue::Crawler - Condition: DeployCUR2Table - Properties: - Name: !Sub "${ResourcePrefix}-DataExportCUR2Crawler" - Description: A recurring crawler that keeps your CUR table in Athena up-to-date. - Role: !GetAtt CidDataExportCrawlerRole.Arn - DatabaseName: !Ref CIDDatabase - Targets: - S3Targets: - - Path: !Sub "s3://${DestinationS3}/cur2/" - Exclusions: - - "**.json" - - "**.yml" - - "**.sql" - - "**.csv" - - "**.csv.metadata" - - "**.gz" - - "**.zip" - - "**/cost_and_usage_data_status/*" - - "aws-programmatic-access-test-object" - SchemaChangePolicy: - DeleteBehavior: LOG - RecrawlPolicy: - RecrawlBehavior: CRAWL_EVERYTHING - Schedule: - ScheduleExpression: cron(0 2 * * ? *) - Configuration: | - { - "Version":1.0, - "Grouping": { - "TableGroupingPolicy": "CombineCompatibleSchemas" - }, - "CrawlerOutput":{ - "Tables":{ - "AddOrUpdateBehavior":"MergeNewColumns" - } - } - } - - ########################################################################### - # FOCUS - ########################################################################### - - FOCUSTable: # Initial creation of table. it will be updated by crawler later - Type: AWS::Glue::Table - Condition: DeployFOCUSTable - Properties: - CatalogId: !Sub "${AWS::AccountId}" - DatabaseName: !Ref CIDDatabase - TableInput: - Name: focus - Owner: owner - Retention: 0 - TableType: EXTERNAL_TABLE - Parameters: - compressionType: none - classification: parquet - UPDATED_BY_CRAWLER: !Ref FOCUSCrawler - StorageDescriptor: - BucketColumns: [] - Compressed: false - Location: !Sub "s3://${DestinationS3}/focus/" - NumberOfBuckets: -1 - InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat - OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat - SerdeInfo: - Parameters: - serialization.format: "1" - SerializationLibrary: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe - StoredAsSubDirectories: false - Columns: - - { "Name": "availabilityzone", "Type": "string" } - - { "Name": "billedcost", "Type": "double" } - - { "Name": "billingaccountid", "Type": "string" } - - { "Name": "billingaccountname", "Type": "string" } - - { "Name": "billingcurrency", "Type": "string" } - - { "Name": "billingperiodend", "Type": "timestamp" } - - { "Name": "billingperiodstart", "Type": "timestamp" } - - { "Name": "chargecategory", "Type": "string" } - - { "Name": "chargeclass", "Type": "string" } - - { "Name": "chargedescription", "Type": "string" } - - { "Name": "chargefrequency", "Type": "string" } - - { "Name": "chargeperiodend", "Type": "timestamp" } - - { "Name": "chargeperiodstart", "Type": "timestamp" } - - { "Name": "commitmentdiscountcategory", "Type": "string" } - - { "Name": "commitmentdiscountid", "Type": "string" } - - { "Name": "commitmentdiscountname", "Type": "string" } - - { "Name": "commitmentdiscounttype", "Type": "string" } - - { "Name": "commitmentdiscountstatus", "Type": "string" } - - { "Name": "consumedquantity", "Type": "double" } - - { "Name": "consumedunit", "Type": "string" } - - { "Name": "contractedcost", "Type": "double" } - - { "Name": "contractedunitprice", "Type": "double" } - - { "Name": "effectivecost", "Type": "double" } - - { "Name": "invoiceissuername", "Type": "string" } - - { "Name": "listcost", "Type": "double" } - - { "Name": "listunitprice", "Type": "double" } - - { "Name": "pricingcategory", "Type": "string" } - - { "Name": "pricingquantity", "Type": "double" } - - { "Name": "pricingunit", "Type": "string" } - - { "Name": "providername", "Type": "string" } - - { "Name": "publishername", "Type": "string" } - - { "Name": "regionid", "Type": "string" } - - { "Name": "regionname", "Type": "string" } - - { "Name": "resourceid", "Type": "string" } - - { "Name": "resourcename", "Type": "string" } - - { "Name": "resourcetype", "Type": "string" } - - { "Name": "servicecategory", "Type": "string" } - - { "Name": "servicename", "Type": "string" } - - { "Name": "skuid", "Type": "string" } - - { "Name": "skupriceid", "Type": "string" } - - { "Name": "subaccountid", "Type": "string" } - - { "Name": "subaccountname", "Type": "string" } - - { "Name": "tags", "Type": "map" } - PartitionKeys: - - { "Name": "source_account_id", "Type": "string" } - - { "Name": "report_name", "Type": "string" } - - { "Name": "data", "Type": "string" } - - { "Name": "billing_period", "Type": "string" } - - FOCUSCrawler: - Type: AWS::Glue::Crawler - Condition: DeployFOCUSTable - Properties: - Name: !Sub "${ResourcePrefix}-DataExportFOCUSCrawler" - Description: A recurring crawler that keeps your FOCUS table in Athena up-to-date. - Role: !GetAtt CidDataExportCrawlerRole.Arn - DatabaseName: !Ref CIDDatabase - Targets: - S3Targets: - - Path: !Sub "s3://${DestinationS3}/focus/" - Exclusions: - - "**.json" - - "**.yml" - - "**.sql" - - "**.csv" - - "**.csv.metadata" - - "**.gz" - - "**.zip" - - "**/cost_and_usage_data_status/*" - - "aws-programmatic-access-test-object" - SchemaChangePolicy: - DeleteBehavior: LOG - RecrawlPolicy: - RecrawlBehavior: CRAWL_EVERYTHING - Schedule: - ScheduleExpression: cron(0 2 * * ? *) - Configuration: | - { - "Version":1.0, - "Grouping": { - "TableGroupingPolicy": "CombineCompatibleSchemas" - }, - "CrawlerOutput":{ - "Tables":{ - "AddOrUpdateBehavior":"MergeNewColumns" - } - } - } - - ########################################################################### - # COH - ########################################################################### - - COHTable: # Initial creation of table. it will be updated by crawler later - Type: AWS::Glue::Table - Condition: DeployCOHTable - Properties: - CatalogId: !Sub "${AWS::AccountId}" - DatabaseName: !Ref CIDDatabase - TableInput: - Name: coh - Owner: owner - Retention: 0 - TableType: EXTERNAL_TABLE - Parameters: - compressionType: none - classification: parquet - UPDATED_BY_CRAWLER: !Ref COHCrawler - StorageDescriptor: - BucketColumns: [] - Compressed: false - Location: !Sub "s3://${DestinationS3}/coh/" - NumberOfBuckets: -1 - InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat - OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat - SerdeInfo: - Parameters: - serialization.format: "1" - SerializationLibrary: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe - StoredAsSubDirectories: false - Columns: # All fields required for CID - - { "Name": "account_id", "Type": "string" } - - { "Name": "action_type", "Type": "string" } - - { "Name": "currency_code", "Type": "string" } - - { "Name": "current_resource_details", "Type": "string" } - - { "Name": "current_resource_summary", "Type": "string" } - - { "Name": "current_resource_type", "Type": "string" } - - { - "Name": "estimated_monthly_cost_after_discount", - "Type": "double", - } - - { - "Name": "estimated_monthly_cost_before_discount", - "Type": "double", - } - - { - "Name": "estimated_monthly_savings_after_discount", - "Type": "double", - } - - { - "Name": "estimated_monthly_savings_before_discount", - "Type": "double", - } - - { - "Name": "estimated_savings_percentage_after_discount", - "Type": "double", - } - - { - "Name": "estimated_savings_percentage_before_discount", - "Type": "double", - } - - { "Name": "implementation_effort", "Type": "string" } - - { "Name": "last_refresh_timestamp", "Type": "string" } - - { "Name": "recommendation_id", "Type": "string" } - - { - "Name": "recommendation_lookback_period_in_days", - "Type": "int", - } - - { "Name": "recommendation_source", "Type": "string" } - - { "Name": "recommended_resource_details", "Type": "string" } - - { "Name": "recommended_resource_summary", "Type": "string" } - - { "Name": "recommended_resource_type", "Type": "string" } - - { "Name": "region", "Type": "string" } - - { "Name": "resource_arn", "Type": "string" } - - { "Name": "restart_needed", "Type": "boolean" } - - { "Name": "rollback_possible", "Type": "boolean" } - - { "Name": "tags", "Type": "map" } - PartitionKeys: - - { "Name": "source_account_id", "Type": "string" } - - { "Name": "report_name", "Type": "string" } - - { "Name": "data", "Type": "string" } - - { "Name": "date", "Type": "string" } - - COHCrawler: - Type: AWS::Glue::Crawler - Condition: DeployCOHTable - Properties: - Name: !Sub "${ResourcePrefix}-DataExportCOHCrawler" - Description: A recurring crawler that keeps your COH table in Athena up-to-date. - Role: !GetAtt CidDataExportCrawlerRole.Arn - DatabaseName: !Ref CIDDatabase - Targets: - S3Targets: - - Path: !Sub "s3://${DestinationS3}/coh/" - Exclusions: - - "**.json" - - "**.yml" - - "**.sql" - - "**.csv" - - "**.csv.metadata" - - "**.gz" - - "**.zip" - - "**/cost_and_usage_data_status/*" - - "aws-programmatic-access-test-object" - SchemaChangePolicy: - DeleteBehavior: LOG - RecrawlPolicy: - RecrawlBehavior: CRAWL_EVERYTHING - Schedule: - ScheduleExpression: cron(0 2 * * ? *) - Configuration: | - { - "Version":1.0, - "Grouping": { - "TableGroupingPolicy": "CombineCompatibleSchemas" - }, - "CrawlerOutput":{ - "Tables":{ - "AddOrUpdateBehavior":"MergeNewColumns" - } - } - } - - ########################################################################### - # Generic Resources - ########################################################################### - - CidDataExportCrawlerRole: - Type: AWS::IAM::Role - Condition: DeployAnyTable - Properties: - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Principal: - Service: - - glue.amazonaws.com - Action: - - "sts:AssumeRole" - Path: !Ref RolePath - Policies: - - PolicyName: CrawlerPolicy - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: - - "s3:GetObject" - Resource: !Sub "arn:${AWS::Partition}:s3:::${DestinationS3}/*" - - Effect: Allow - Action: - - "s3:ListBucket" - Resource: !Sub "arn:${AWS::Partition}:s3:::${DestinationS3}" - - Effect: Allow - Action: - - glue:GetDatabase - - glue:GetDatabases - - glue:UpdateDatabase - - glue:CreateTable - - glue:UpdateTable - - glue:GetTable - - glue:GetTables - - glue:BatchCreatePartition - - glue:CreatePartition - - glue:DeletePartition - - glue:BatchDeletePartition - - glue:UpdatePartition - - glue:GetPartition - - glue:GetPartitions - - glue:BatchGetPartition - - glue:ImportCatalogToGlue - Resource: - - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:catalog - - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:database/${CIDDatabase} - - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:table/${CIDDatabase}/* - - Effect: Allow - Action: - - logs:CreateLogGroup - - logs:CreateLogStream - Resource: - - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws-glue/crawlers:*" - - Effect: Allow - Action: - - logs:PutLogEvents - Resource: - - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws-glue/crawlers:log-stream:*" - - ########################################################################### - # Analytics: used by CID team to track adoption, by retrieving AWS AccountId - ########################################################################### - - CidLambdaAnalyticsRole: #Execution role for the custom resource CidLambdaAnalyticsExecutor - Type: AWS::IAM::Role - Properties: - Path: !Sub /${ResourcePrefix}/ - #RoleName: CID-Analytics - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Principal: - Service: - - lambda.amazonaws.com - Action: - - sts:AssumeRole - Policies: - - PolicyName: "ExecutionDefault" - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: - - logs:CreateLogStream - - logs:PutLogEvents - - logs:CreateLogGroup - Resource: - - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-CID-Analytics-DataExports" - - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-CID-Analytics-DataExports:*" - - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-CID-Analytics-DataExports:*:*" - - CidLambdaAnalytics: - Type: AWS::Lambda::Function - Properties: - Runtime: python3.12 - FunctionName: !Sub ${ResourcePrefix}-CID-Analytics-DataExports - Handler: index.lambda_handler - MemorySize: 128 - Role: !GetAtt CidLambdaAnalyticsRole.Arn - Timeout: 15 - Environment: - Variables: - API_ENDPOINT: https://okakvoavfg.execute-api.eu-west-1.amazonaws.com/ - Code: - ZipFile: | - import os - import json - import uuid - import urllib3 - - import boto3 - import cfnresponse - - http = urllib3.PoolManager() - endpoint = os.environ["API_ENDPOINT"] - account_id = boto3.client("sts").get_caller_identity()["Account"] - - def execute_request(action, product_id, via_key): - try: - payload = {'dashboard_id': product_id, 'account_id': account_id, via_key: 'CFN'} - encoded_data = json.dumps(payload).encode('utf-8') - r = http.request(action, endpoint, body=encoded_data, headers={'Content-Type': 'application/json'}) - if r.status != 200: - return f"This will not fail the deployment. There has been an issue logging action {action} for product {product_id} and account {account_id}, server did not respond with a 200 response, status: {r.status}, response data {r.data.decode('utf-8')}. This issue will be ignored" - except urllib3.exceptions.HTTPError as e: - return f"Issue logging action {action} for product {product_id} and account {account_id}, due to a urllib3 exception {str(e)}. This issue will be ignored" - return None - - def register_deployment(action, products): - message = f"Successfully logged {action} for {products}" - for product_id in products: - if action == 'Create': - message = execute_request('PUT', product_id, 'created_via') - elif action == 'Update': - message = execute_request('PATCH', product_id, 'updated_via') - elif action == 'Delete': - message = execute_request('DELETE', product_id, 'deleted_via') - if not message: - message = f"Successfully logged {action} for {products} " - #Do not stop deployment if we're not able to successfully record this deployment, still return true - return (True, message) - - def lambda_handler(event, context): - if event['RequestType'] in ['Create', 'Update', 'Delete']: - res, reason = register_deployment(event['RequestType'], event['ResourceProperties']['DeploymentType']) - else: - res, reason = False, "Unknown operation: " + event['RequestType'] - - physicalResourceId = event.get('ResourceProperties', {}).get('Export', {}).get('Name') or str(uuid.uuid1()) - if res: - cfnresponse.send(event, context, cfnresponse.SUCCESS, {}, physicalResourceId ) - else: - cfnresponse.send(event, context, cfnresponse.FAILED, {}, physicalResourceId, reason=reason) - Metadata: - cfn_nag: - rules_to_suppress: - - id: "W89" - reason: "This Lambda does not require VPC" - - id: "W92" - reason: "One Time execution. No need for ReservedConcurrentExecutions" - - CidLambdaAnalyticsExecutorForDataExports: - Type: Custom::CidLambdaAnalyticsExecutorForDataExports - Properties: - ServiceToken: !GetAtt CidLambdaAnalytics.Arn - DeploymentType: - - !If [ - IsDestinationAccount, - "cid-dataexport-destination", - !Ref "AWS::NoValue", - ] - - !If [IsSourceAccount, "cid-dataexport-source", !Ref "AWS::NoValue"] - - DataExportsReadAccess: - Type: AWS::IAM::ManagedPolicy - Condition: DeployAnyTable - Properties: - ManagedPolicyName: !Sub ${ResourcePrefix}DataExportsReadAccess - Description: "Policy for QuickSight to allow DataExports access" - PolicyDocument: - Version: "2012-10-17" - Statement: - - Sid: AllowGlue - Effect: Allow - Action: - - glue:GetPartition - - glue:GetPartitions - - glue:GetDatabase - - glue:GetDatabases - - glue:GetTable - - glue:GetTables - Resource: - - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:catalog - - Fn::Join: - - "" - - - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:table/ - - !Join [ - "_", - !Split ["-", !Sub "${ResourcePrefix}_data_export"], - ] # replace '-' to '_' - - "/*" - - Fn::Join: - - "" - - - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:database/ - - !Join [ - "_", - !Split ["-", !Sub "${ResourcePrefix}_data_export"], - ] # replace '-' to '_' - - Sid: AllowListBucket - Effect: Allow - Action: s3:ListBucket - Resource: - - !Sub arn:${AWS::Partition}:s3:::${ResourcePrefix}-${DestinationAccountId}-data-exports - - Sid: AllowReadBucket - Effect: Allow - Action: - - s3:GetObject - - s3:GetObjectVersion - Resource: - - !Sub arn:${AWS::Partition}:s3:::${ResourcePrefix}-${DestinationAccountId}-data-exports/* - Metadata: - cfn_nag: - rules_to_suppress: - - id: "W28" - reason: "Need an explicit name for reference" - -Outputs: - DestinationBucketName: - Description: Bucket with aggregate Data Exports - Value: !Sub ${ResourcePrefix}-${DestinationAccountId}-data-exports - Export: { Name: "cid-DataExports-Bucket" } - Database: - Description: Database for Data Exports - Value: !Join ["_", !Split ["-", !Sub "${ResourcePrefix}_data_export"]] # replace '-' to '_' - Export: { Name: "cid-DataExports-Database" } - LocalAccountBucket: - Condition: DeployDataExport - Description: Local Bucket Name which replicate objects to centralized bucket - Value: !Sub ${ResourcePrefix}-${AWS::AccountId}-data-local - ReadAccessPolicyARN: - Condition: DeployAnyTable - Description: Policy to allow read access DataExports in S3 and Athena. Attach it to QuickSight role. - Value: !Ref DataExportsReadAccess - Export: { Name: "cid-DataExports-ReadAccessPolicyARN" } diff --git a/modules/source/assets/cloudformation/cudos/data-exports-aggregation.yaml b/modules/source/assets/cloudformation/cudos/data-exports-aggregation.yaml new file mode 120000 index 0000000..e94b02f --- /dev/null +++ b/modules/source/assets/cloudformation/cudos/data-exports-aggregation.yaml @@ -0,0 +1 @@ +../../../../../assets/cloudformation/cudos/data-exports-aggregation.yaml \ No newline at end of file diff --git a/modules/source/data.tf b/modules/source/data.tf index 40ca1e9..f4244e9 100644 --- a/modules/source/data.tf +++ b/modules/source/data.tf @@ -1,9 +1,6 @@ -## Find the account id for the management account +## Find the account id for the management account data "aws_caller_identity" "current" {} -## Find the current region +## Find the current region data "aws_region" "current" {} - -## Find the current organization -data "aws_organizations_organization" "current" {} diff --git a/modules/source/locals.tf b/modules/source/locals.tf index cb8b415..048a4b2 100644 --- a/modules/source/locals.tf +++ b/modules/source/locals.tf @@ -1,14 +1,14 @@ locals { - ## The region where the stack is being deployed + ## The region where the stack is being deployed region = data.aws_region.current.name - ## Is the management account id + ## Is the management account id management_account_id = data.aws_caller_identity.current.account_id ## Is the organization root id organization_root_id = data.aws_organizations_organization.current.roots[0].id - ## The s3 bucket name for the cloudformation scripts + ## The s3 bucket name for the cloudformation scripts stacks_base_url = format("https://%s.s3.%s.amazonaws.com", var.stacks_bucket_name, local.region) - ## The account id where the dashboard is being deployed + ## The account id where the dashboard is being deployed destination_account_id = var.destination_account_id } diff --git a/modules/source/main.tf b/modules/source/main.tf index 1553046..9395776 100644 --- a/modules/source/main.tf +++ b/modules/source/main.tf @@ -1,5 +1,5 @@ -## Craft and IAM policy that allows the account to access the bucket +## Craft and IAM policy that allows the account to access the bucket data "aws_iam_policy_document" "stack_bucket_policy" { statement { effect = "Allow" @@ -36,7 +36,7 @@ data "aws_iam_policy_document" "stack_bucket_policy" { } } -## Provision a bucket used to contain the cloudformation templates +## Provision a bucket used to contain the cloudformation templates # tfsec:ignore:aws-s3-enable-bucket-logging module "cloudformation_bucket" { source = "terraform-aws-modules/s3-bucket/aws" @@ -67,7 +67,7 @@ module "cloudformation_bucket" { } } -## Upload the cloudformation templates to the bucket +## Upload the cloudformation templates to the bucket resource "aws_s3_object" "cloudformation_templates" { for_each = fileset("${path.module}/assets/cloudformation/", "**/*.yaml") @@ -78,8 +78,8 @@ resource "aws_s3_object" "cloudformation_templates" { source = "${path.module}/assets/cloudformation/${each.value}" } -## Setup the replication from the management account to the collector account -## to receive the CUR data +## Setup the replication from the management account to the collector account +## to receive the CUR data # tfsec:ignore:aws-s3-enable-bucket-logging # tfsec:ignore:aws-iam-no-policy-wildcards module "source" { @@ -93,7 +93,7 @@ module "source" { } } -## Provision the stack contain the cora data exports in the management account +## Provision the stack contain the cora data exports in the management account ## Deployment of same stack the management account resource "aws_cloudformation_stack" "core_data_export_management" { count = var.enable_cora_data_exports ? 1 : 0 @@ -119,7 +119,9 @@ resource "aws_cloudformation_stack" "core_data_export_management" { } } -## We need to provision the read permissions stack in the management account +## We need to provision the read permissions stack within the management account, note +## this effectively creates a stackset which is deployed to all accounts within the +## organization resource "aws_cloudformation_stack" "cudos_read_permissions" { name = var.stack_name_read_permissions capabilities = ["CAPABILITY_NAMED_IAM", "CAPABILITY_AUTO_EXPAND"] @@ -139,7 +141,7 @@ resource "aws_cloudformation_stack" "cudos_read_permissions" { "IncludeRightsizingModule" = var.enable_rightsizing_module ? "yes" : "no", "IncludeTAModule" = var.enable_tao_module ? "yes" : "no", "IncludeTransitGatewayModule" = var.enable_transit_gateway_module ? "yes" : "no", - "OrganizationalUnitIds" = local.organization_root_id, + "OrganizationalUnitIds" = var.organizational_unit_ids, } depends_on = [ diff --git a/modules/source/variables.tf b/modules/source/variables.tf index 0c9ff8f..13ef52c 100644 --- a/modules/source/variables.tf +++ b/modules/source/variables.tf @@ -109,3 +109,9 @@ variable "enable_rightsizing_module" { type = bool default = true } + +variable "organization_unit_ids" { + description = "List of organization units where the read permissions stack will be deployed" + type = list(string) + default = [] +} From 09deff1cc978107bb356bd4b9ab099f135d66908 Mon Sep 17 00:00:00 2001 From: Rohith Jayawardene Date: Wed, 27 Nov 2024 10:36:10 +0000 Subject: [PATCH 15/19] feat: adding back the cora export to the data account, as this is required and providing a toggle the data read permissions --- modules/source/README.md | 1 + modules/source/main.tf | 2 ++ modules/source/variables.tf | 6 ++++++ 3 files changed, 9 insertions(+) diff --git a/modules/source/README.md b/modules/source/README.md index 2b2391c..a03c3ea 100644 --- a/modules/source/README.md +++ b/modules/source/README.md @@ -21,6 +21,7 @@ | [enable\_health\_events\_module](#input\_enable\_health\_events\_module) | Indicates if the Health Events module should be enabled | `bool` | `true` | no | | [enable\_inventory\_module](#input\_enable\_inventory\_module) | Indicates if the Inventory module should be enabled | `bool` | `true` | no | | [enable\_rds\_utilization\_module](#input\_enable\_rds\_utilization\_module) | Indicates if the RDS Utilization module should be enabled | `bool` | `true` | no | +| [enable\_read\_permissions\_stack](#input\_enable\_read\_permissions\_stack) | Indicates if the read permissions are deployed to all linked accounts | `bool` | `false` | no | | [enable\_rightsizing\_module](#input\_enable\_rightsizing\_module) | Indicates if the Rightsizing module should be enabled | `bool` | `true` | no | | [enable\_scad](#input\_enable\_scad) | Indicates if the SCAD module should be enabled, only available when Cora enabled | `bool` | `false` | no | | [enable\_tao\_module](#input\_enable\_tao\_module) | Indicates if the TAO module should be enabled | `bool` | `true` | no | diff --git a/modules/source/main.tf b/modules/source/main.tf index 9395776..fde4ef0 100644 --- a/modules/source/main.tf +++ b/modules/source/main.tf @@ -123,6 +123,8 @@ resource "aws_cloudformation_stack" "core_data_export_management" { ## this effectively creates a stackset which is deployed to all accounts within the ## organization resource "aws_cloudformation_stack" "cudos_read_permissions" { + count = var.enable_read_permissions_stack ? 1 : 0 + name = var.stack_name_read_permissions capabilities = ["CAPABILITY_NAMED_IAM", "CAPABILITY_AUTO_EXPAND"] template_url = format("%s/cudos/%s", local.stacks_base_url, "deploy-data-read-permissions.yaml") diff --git a/modules/source/variables.tf b/modules/source/variables.tf index 13ef52c..06d437c 100644 --- a/modules/source/variables.tf +++ b/modules/source/variables.tf @@ -32,6 +32,12 @@ variable "stack_name_cora_data_exports_source" { default = "CidCoraCoraDataExportsSourceStack" } +variable "enable_read_permissions_stack" { + description = "Indicates if the read permissions are deployed to all linked accounts" + type = bool + default = false +} + variable "enable_cost_anomaly_module" { description = "Indicates if the Cost Anomaly module should be enabled" type = bool From 7e947853903b6876047ad11c611fe0dc4b28d7bd Mon Sep 17 00:00:00 2001 From: Rohith Jayawardene Date: Wed, 27 Nov 2024 10:37:07 +0000 Subject: [PATCH 16/19] chore: removing the read permission variable and using the lack of organizational id --- modules/source/README.md | 1 - modules/source/main.tf | 2 -- modules/source/variables.tf | 6 ------ 3 files changed, 9 deletions(-) diff --git a/modules/source/README.md b/modules/source/README.md index a03c3ea..2b2391c 100644 --- a/modules/source/README.md +++ b/modules/source/README.md @@ -21,7 +21,6 @@ | [enable\_health\_events\_module](#input\_enable\_health\_events\_module) | Indicates if the Health Events module should be enabled | `bool` | `true` | no | | [enable\_inventory\_module](#input\_enable\_inventory\_module) | Indicates if the Inventory module should be enabled | `bool` | `true` | no | | [enable\_rds\_utilization\_module](#input\_enable\_rds\_utilization\_module) | Indicates if the RDS Utilization module should be enabled | `bool` | `true` | no | -| [enable\_read\_permissions\_stack](#input\_enable\_read\_permissions\_stack) | Indicates if the read permissions are deployed to all linked accounts | `bool` | `false` | no | | [enable\_rightsizing\_module](#input\_enable\_rightsizing\_module) | Indicates if the Rightsizing module should be enabled | `bool` | `true` | no | | [enable\_scad](#input\_enable\_scad) | Indicates if the SCAD module should be enabled, only available when Cora enabled | `bool` | `false` | no | | [enable\_tao\_module](#input\_enable\_tao\_module) | Indicates if the TAO module should be enabled | `bool` | `true` | no | diff --git a/modules/source/main.tf b/modules/source/main.tf index fde4ef0..9395776 100644 --- a/modules/source/main.tf +++ b/modules/source/main.tf @@ -123,8 +123,6 @@ resource "aws_cloudformation_stack" "core_data_export_management" { ## this effectively creates a stackset which is deployed to all accounts within the ## organization resource "aws_cloudformation_stack" "cudos_read_permissions" { - count = var.enable_read_permissions_stack ? 1 : 0 - name = var.stack_name_read_permissions capabilities = ["CAPABILITY_NAMED_IAM", "CAPABILITY_AUTO_EXPAND"] template_url = format("%s/cudos/%s", local.stacks_base_url, "deploy-data-read-permissions.yaml") diff --git a/modules/source/variables.tf b/modules/source/variables.tf index 06d437c..13ef52c 100644 --- a/modules/source/variables.tf +++ b/modules/source/variables.tf @@ -32,12 +32,6 @@ variable "stack_name_cora_data_exports_source" { default = "CidCoraCoraDataExportsSourceStack" } -variable "enable_read_permissions_stack" { - description = "Indicates if the read permissions are deployed to all linked accounts" - type = bool - default = false -} - variable "enable_cost_anomaly_module" { description = "Indicates if the Cost Anomaly module should be enabled" type = bool From 181a900e19e72824391706c108881d8df3f278b7 Mon Sep 17 00:00:00 2001 From: Rohith Jayawardene Date: Wed, 27 Nov 2024 13:26:15 +0000 Subject: [PATCH 17/19] fix: messed up on the organizational id variable --- modules/source/README.md | 2 +- modules/source/locals.tf | 2 -- modules/source/main.tf | 2 +- modules/source/variables.tf | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/modules/source/README.md b/modules/source/README.md index 2b2391c..d27eee3 100644 --- a/modules/source/README.md +++ b/modules/source/README.md @@ -25,7 +25,7 @@ | [enable\_scad](#input\_enable\_scad) | Indicates if the SCAD module should be enabled, only available when Cora enabled | `bool` | `false` | no | | [enable\_tao\_module](#input\_enable\_tao\_module) | Indicates if the TAO module should be enabled | `bool` | `true` | no | | [enable\_transit\_gateway\_module](#input\_enable\_transit\_gateway\_module) | Indicates if the Transit Gateway module should be enabled | `bool` | `true` | no | -| [organization\_unit\_ids](#input\_organization\_unit\_ids) | List of organization units where the read permissions stack will be deployed | `list(string)` | `[]` | no | +| [organizational\_unit\_ids](#input\_organizational\_unit\_ids) | List of organization units where the read permissions stack will be deployed | `list(string)` | `[]` | no | | [stack\_name\_cora\_data\_exports\_source](#input\_stack\_name\_cora\_data\_exports\_source) | The name of the CloudFormation stack to create the CORA Data Exports | `string` | `"CidCoraCoraDataExportsSourceStack"` | no | | [stack\_name\_read\_permissions](#input\_stack\_name\_read\_permissions) | The name of the CloudFormation stack to create the collectors | `string` | `"CidDataCollectionReadPermissionsStack"` | no | | [stacks\_bucket\_name](#input\_stacks\_bucket\_name) | The name of the bucket to store the CloudFormation templates | `string` | `"cid-cloudformation-templates"` | no | diff --git a/modules/source/locals.tf b/modules/source/locals.tf index 048a4b2..ed3eebe 100644 --- a/modules/source/locals.tf +++ b/modules/source/locals.tf @@ -4,8 +4,6 @@ locals { region = data.aws_region.current.name ## Is the management account id management_account_id = data.aws_caller_identity.current.account_id - ## Is the organization root id - organization_root_id = data.aws_organizations_organization.current.roots[0].id ## The s3 bucket name for the cloudformation scripts stacks_base_url = format("https://%s.s3.%s.amazonaws.com", var.stacks_bucket_name, local.region) ## The account id where the dashboard is being deployed diff --git a/modules/source/main.tf b/modules/source/main.tf index 9395776..43f51fe 100644 --- a/modules/source/main.tf +++ b/modules/source/main.tf @@ -141,7 +141,7 @@ resource "aws_cloudformation_stack" "cudos_read_permissions" { "IncludeRightsizingModule" = var.enable_rightsizing_module ? "yes" : "no", "IncludeTAModule" = var.enable_tao_module ? "yes" : "no", "IncludeTransitGatewayModule" = var.enable_transit_gateway_module ? "yes" : "no", - "OrganizationalUnitIds" = var.organizational_unit_ids, + "OrganizationalUnitIds" = join(",", var.organizational_unit_ids) } depends_on = [ diff --git a/modules/source/variables.tf b/modules/source/variables.tf index 13ef52c..975ad58 100644 --- a/modules/source/variables.tf +++ b/modules/source/variables.tf @@ -110,7 +110,7 @@ variable "enable_rightsizing_module" { default = true } -variable "organization_unit_ids" { +variable "organizational_unit_ids" { description = "List of organization units where the read permissions stack will be deployed" type = list(string) default = [] From 7318b2e50fbd994ada3d311947cc1f9921e08f3b Mon Sep 17 00:00:00 2001 From: Rohith Jayawardene Date: Wed, 27 Nov 2024 14:03:38 +0000 Subject: [PATCH 18/19] fix: moving the files to the correct place --- .../assets/cloudformation/cudos/data-exports-aggregation.yaml | 1 + .../cloudformation/{ => cudos}/deploy-data-collection.yaml | 0 .../assets/cloudformation/data-exports-aggregation.yaml | 1 - 3 files changed, 1 insertion(+), 1 deletion(-) create mode 120000 modules/destination/assets/cloudformation/cudos/data-exports-aggregation.yaml rename modules/destination/assets/cloudformation/{ => cudos}/deploy-data-collection.yaml (100%) delete mode 120000 modules/destination/assets/cloudformation/data-exports-aggregation.yaml diff --git a/modules/destination/assets/cloudformation/cudos/data-exports-aggregation.yaml b/modules/destination/assets/cloudformation/cudos/data-exports-aggregation.yaml new file mode 120000 index 0000000..e94b02f --- /dev/null +++ b/modules/destination/assets/cloudformation/cudos/data-exports-aggregation.yaml @@ -0,0 +1 @@ +../../../../../assets/cloudformation/cudos/data-exports-aggregation.yaml \ No newline at end of file diff --git a/modules/destination/assets/cloudformation/deploy-data-collection.yaml b/modules/destination/assets/cloudformation/cudos/deploy-data-collection.yaml similarity index 100% rename from modules/destination/assets/cloudformation/deploy-data-collection.yaml rename to modules/destination/assets/cloudformation/cudos/deploy-data-collection.yaml diff --git a/modules/destination/assets/cloudformation/data-exports-aggregation.yaml b/modules/destination/assets/cloudformation/data-exports-aggregation.yaml deleted file mode 120000 index 759e345..0000000 --- a/modules/destination/assets/cloudformation/data-exports-aggregation.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../../assets/cloudformation/cudos/data-exports-aggregation.yaml \ No newline at end of file From d0dd299485014abbadaf62163576f48401387410 Mon Sep 17 00:00:00 2001 From: Rohith Jayawardene Date: Wed, 27 Nov 2024 14:10:34 +0000 Subject: [PATCH 19/19] fix: ensuring the dependency of the bucket --- modules/destination/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/destination/main.tf b/modules/destination/main.tf index 88b2b85..b241f7f 100644 --- a/modules/destination/main.tf +++ b/modules/destination/main.tf @@ -283,6 +283,7 @@ resource "aws_cloudformation_stack" "cudos_data_collection" { } depends_on = [ + aws_s3_object.cloudformation_templates, module.collector, module.dashboards, ]