feat: Added SQS queue creation with events for Karpenter (aws-ia#1458)

examples/karpenter/main.tf

@@ -171,7 +171,6 @@ module "eks_blueprints_kubernetes_addons" {
   }
   karpenter_node_iam_instance_profile        = module.karpenter.instance_profile_name
   karpenter_enable_spot_termination_handling = true
-  karpenter_sqs_queue_arn                    = module.karpenter.queue_arn
 
   tags = local.tags
 }

modules/kubernetes-addons/README.md

@@ -277,7 +277,6 @@
 | <a name="input_karpenter_helm_config"></a> [karpenter\_helm\_config](#input\_karpenter\_helm\_config) | Karpenter autoscaler add-on config | `any` | `{}` | no |
 | <a name="input_karpenter_irsa_policies"></a> [karpenter\_irsa\_policies](#input\_karpenter\_irsa\_policies) | Additional IAM policies for a IAM role for service accounts | `list(string)` | `[]` | no |
 | <a name="input_karpenter_node_iam_instance_profile"></a> [karpenter\_node\_iam\_instance\_profile](#input\_karpenter\_node\_iam\_instance\_profile) | Karpenter Node IAM Instance profile id | `string` | `""` | no |
-| <a name="input_karpenter_sqs_queue_arn"></a> [karpenter\_sqs\_queue\_arn](#input\_karpenter\_sqs\_queue\_arn) | (Optional) ARN of SQS used by Karpenter when native node termination handling is enabled | `string` | `""` | no |
 | <a name="input_keda_helm_config"></a> [keda\_helm\_config](#input\_keda\_helm\_config) | KEDA Event-based autoscaler add-on config | `any` | `{}` | no |
 | <a name="input_keda_irsa_policies"></a> [keda\_irsa\_policies](#input\_keda\_irsa\_policies) | Additional IAM policies for a IAM role for service accounts | `list(string)` | `[]` | no |
 | <a name="input_kube_prometheus_stack_helm_config"></a> [kube\_prometheus\_stack\_helm\_config](#input\_kube\_prometheus\_stack\_helm\_config) | Community kube-prometheus-stack Helm Chart config | `any` | `{}` | no |
@@ -314,6 +313,9 @@
 | <a name="input_spark_history_server_irsa_policies"></a> [spark\_history\_server\_irsa\_policies](#input\_spark\_history\_server\_irsa\_policies) | Additional IAM policies for a IAM role for service accounts | `list(string)` | `[]` | no |
 | <a name="input_spark_history_server_s3a_path"></a> [spark\_history\_server\_s3a\_path](#input\_spark\_history\_server\_s3a\_path) | s3a path with prefix for Spark history server e.g., s3a://<bucket\_name>/<spark\_event\_logs> | `string` | `""` | no |
 | <a name="input_spark_k8s_operator_helm_config"></a> [spark\_k8s\_operator\_helm\_config](#input\_spark\_k8s\_operator\_helm\_config) | Spark on K8s Operator Helm Chart config | `any` | `{}` | no |
+| <a name="input_sqs_queue_kms_data_key_reuse_period_seconds"></a> [sqs\_queue\_kms\_data\_key\_reuse\_period\_seconds](#input\_sqs\_queue\_kms\_data\_key\_reuse\_period\_seconds) | The length of time, in seconds, for which Amazon SQS can reuse a data key to encrypt or decrypt messages before calling AWS KMS again | `number` | `null` | no |
+| <a name="input_sqs_queue_kms_master_key_id"></a> [sqs\_queue\_kms\_master\_key\_id](#input\_sqs\_queue\_kms\_master\_key\_id) | The ID of an AWS-managed customer master key (CMK) for Amazon SQS or a custom CMK | `string` | `null` | no |
+| <a name="input_sqs_queue_managed_sse_enabled"></a> [sqs\_queue\_managed\_sse\_enabled](#input\_sqs\_queue\_managed\_sse\_enabled) | Enable server-side encryption (SSE) for a SQS queue | `bool` | `true` | no |
 | <a name="input_strimzi_kafka_operator_helm_config"></a> [strimzi\_kafka\_operator\_helm\_config](#input\_strimzi\_kafka\_operator\_helm\_config) | Kafka Strimzi Helm Chart config | `any` | `{}` | no |
 | <a name="input_sysdig_agent_helm_config"></a> [sysdig\_agent\_helm\_config](#input\_sysdig\_agent\_helm\_config) | Sysdig Helm Chart config | `any` | `{}` | no |
 | <a name="input_tags"></a> [tags](#input\_tags) | Additional tags (e.g. `map('BusinessUnit`,`XYZ`) | `map(string)` | `{}` | no |

modules/kubernetes-addons/karpenter/README.md

@@ -28,30 +28,41 @@ For more details checkout [Karpenter](https://karpenter.sh/docs/getting-started/
 
 | Name | Type |
 |------|------|
+| [aws_cloudwatch_event_rule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource |
+| [aws_cloudwatch_event_target.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource |
 | [aws_iam_policy.karpenter](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
-| [aws_arn.queue](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source |
+| [aws_sqs_queue.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource |
+| [aws_sqs_queue_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource |
 | [aws_iam_policy_document.karpenter](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.sqs_queue](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source |
 
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_addon_context"></a> [addon\_context](#input\_addon\_context) | Input configuration for the addon | <pre>object({<br>    aws_caller_identity_account_id = string<br>    aws_caller_identity_arn        = string<br>    aws_eks_cluster_endpoint       = string<br>    aws_partition_id               = string<br>    aws_region_name                = string<br>    eks_cluster_id                 = string<br>    eks_oidc_issuer_url            = string<br>    eks_oidc_provider_arn          = string<br>    tags                           = map(string)<br>    irsa_iam_role_path             = string<br>    irsa_iam_permissions_boundary  = string<br>  })</pre> | n/a | yes |
-| <a name="input_enable_spot_termination_handling"></a> [enable\_spot\_termination\_handling](#input\_enable\_spot\_termination\_handling) | Determines whether to enable native spot termination handling | `bool` | `false` | no |
+| <a name="input_enable_spot_termination"></a> [enable\_spot\_termination](#input\_enable\_spot\_termination) | Determines whether to enable native spot termination handling | `bool` | `false` | no |
 | <a name="input_helm_config"></a> [helm\_config](#input\_helm\_config) | Helm provider config for the Karpenter | `any` | `{}` | no |
 | <a name="input_irsa_policies"></a> [irsa\_policies](#input\_irsa\_policies) | Additional IAM policies for a IAM role for service accounts | `list(string)` | `[]` | no |
 | <a name="input_manage_via_gitops"></a> [manage\_via\_gitops](#input\_manage\_via\_gitops) | Determines if the add-on should be managed via GitOps. | `bool` | `false` | no |
 | <a name="input_node_iam_instance_profile"></a> [node\_iam\_instance\_profile](#input\_node\_iam\_instance\_profile) | Karpenter Node IAM Instance profile id | `string` | `""` | no |
 | <a name="input_path"></a> [path](#input\_path) | Path in which to create the Karpenter policy | `string` | `"/"` | no |
-| <a name="input_sqs_queue_arn"></a> [sqs\_queue\_arn](#input\_sqs\_queue\_arn) | (Optional) ARN of SQS used by Karpenter when native node termination handling is enabled | `string` | `""` | no |
+| <a name="input_sqs_queue_kms_data_key_reuse_period_seconds"></a> [sqs\_queue\_kms\_data\_key\_reuse\_period\_seconds](#input\_sqs\_queue\_kms\_data\_key\_reuse\_period\_seconds) | The length of time, in seconds, for which Amazon SQS can reuse a data key to encrypt or decrypt messages before calling AWS KMS again | `number` | `null` | no |
+| <a name="input_sqs_queue_kms_master_key_id"></a> [sqs\_queue\_kms\_master\_key\_id](#input\_sqs\_queue\_kms\_master\_key\_id) | The ID of an AWS-managed customer master key (CMK) for Amazon SQS or a custom CMK | `string` | `null` | no |
+| <a name="input_sqs_queue_managed_sse_enabled"></a> [sqs\_queue\_managed\_sse\_enabled](#input\_sqs\_queue\_managed\_sse\_enabled) | Enable server-side encryption (SSE) for a SQS queue | `bool` | `true` | no |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
 | <a name="output_argocd_gitops_config"></a> [argocd\_gitops\_config](#output\_argocd\_gitops\_config) | Configuration used for managing the add-on with ArgoCD |
+| <a name="output_event_rules"></a> [event\_rules](#output\_event\_rules) | Map of the event rules created and their attributes |
 | <a name="output_irsa_arn"></a> [irsa\_arn](#output\_irsa\_arn) | IAM role ARN for the service account |
 | <a name="output_irsa_name"></a> [irsa\_name](#output\_irsa\_name) | IAM role name for the service account |
 | <a name="output_release_metadata"></a> [release\_metadata](#output\_release\_metadata) | Map of attributes of the Helm release metadata |
 | <a name="output_service_account"></a> [service\_account](#output\_service\_account) | Name of Kubernetes service account |
+| <a name="output_sqs_queue_arn"></a> [sqs\_queue\_arn](#output\_sqs\_queue\_arn) | The ARN of the SQS queue |
+| <a name="output_sqs_queue_name"></a> [sqs\_queue\_name](#output\_sqs\_queue\_name) | The name of the created Amazon SQS queue |
+| <a name="output_sqs_queue_url"></a> [sqs\_queue\_url](#output\_sqs\_queue\_url) | The URL for the created Amazon SQS queue |
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

modules/kubernetes-addons/karpenter/data.tf

@@ -1,8 +1,4 @@
-data "aws_arn" "queue" {
-  count = var.enable_spot_termination_handling ? 1 : 0
-
-  arn = var.sqs_queue_arn
-}
+data "aws_partition" "current" {}
 
 data "aws_iam_policy_document" "karpenter" {
   statement {
@@ -89,7 +85,7 @@ data "aws_iam_policy_document" "karpenter" {
   }
 
   dynamic "statement" {
-    for_each = var.sqs_queue_arn != "" ? [1] : []
+    for_each = var.enable_spot_termination ? [1] : []
 
     content {
       actions = [
@@ -98,7 +94,24 @@ data "aws_iam_policy_document" "karpenter" {
         "sqs:GetQueueUrl",
         "sqs:ReceiveMessage",
       ]
-      resources = [var.sqs_queue_arn]
+      resources = [aws_sqs_queue.this[0].arn]
+    }
+  }
+}
+
+data "aws_iam_policy_document" "sqs_queue" {
+  count = var.enable_spot_termination ? 1 : 0
+
+  statement {
+    sid     = "SqsWrite"
+    actions = ["sqs:SendMessage"]
+    principals {
+      type = "Service"
+      identifiers = [
+        "events.${local.dns_suffix}",
+        "sqs.${local.dns_suffix}"
+      ]
     }
+    resources = [aws_sqs_queue.this[0].arn]
   }
 }

modules/kubernetes-addons/karpenter/locals.tf

@@ -26,7 +26,7 @@ locals {
               clusterName: ${var.addon_context.eks_cluster_id}
               clusterEndpoint: ${var.addon_context.aws_eks_cluster_endpoint}
               defaultInstanceProfile: ${var.node_iam_instance_profile}
-              interruptionQueueName: ${try(data.aws_arn.queue[0].resource, "")}
+              interruptionQueueName: ${try(aws_sqs_queue.this[0].name, "")}
         EOT
       ]
       description = "karpenter Helm Chart for Node Autoscaling"
@@ -48,6 +48,50 @@ locals {
     serviceAccountName        = local.service_account
     controllerClusterEndpoint = var.addon_context.aws_eks_cluster_endpoint
     awsDefaultInstanceProfile = var.node_iam_instance_profile
-    awsInterruptionQueueName  = try(data.aws_arn.queue[0].resource, "")
+    awsInterruptionQueueName  = try(aws_sqs_queue.this[0].name, "")
+  }
+
+  dns_suffix = data.aws_partition.current.dns_suffix
+
+  # Karpenter Spot Interruption Event rules
+  event_rules = {
+    health_event = {
+      name        = "HealthEvent"
+      description = "Karpenter Interrupt - AWS health event for EC2"
+      event_pattern = {
+        source      = ["aws.health"]
+        detail-type = ["AWS Health Event"]
+        detail = {
+          service = ["EC2"]
+        }
+      }
+    }
+    spot_interupt = {
+      name        = "SpotInterrupt"
+      description = "Karpenter Interrupt - A spot interruption warning was triggered for the node"
+      event_pattern = {
+        source      = ["aws.ec2"]
+        detail-type = ["EC2 Spot Instance Interruption Warning"]
+      }
+    }
+    instance_rebalance = {
+      name        = "InstanceRebalance"
+      description = "Karpenter Interrupt - A spot rebalance recommendation was triggered for the node"
+      event_pattern = {
+        source      = ["aws.ec2"]
+        detail-type = ["EC2 Instance Rebalance Recommendation"]
+      }
+    }
+    instance_state_change = {
+      name        = "InstanceStateChange"
+      description = "Karpenter interrupt - EC2 instance state-change notification"
+      event_pattern = {
+        source      = ["aws.ec2"]
+        detail-type = ["EC2 Instance State-change Notification"]
+        detail = {
+          state = ["stopping", "terminated", "shutting-down", "stopped"] #ignored pending and running
+        }
+      }
+    }
   }
 }

modules/kubernetes-addons/karpenter/main.tf

@@ -13,3 +13,43 @@ resource "aws_iam_policy" "karpenter" {
   policy      = data.aws_iam_policy_document.karpenter.json
   path        = var.path
 }
+
+#tfsec:ignore:aws-sqs-enable-queue-encryption
+resource "aws_sqs_queue" "this" {
+  count = var.enable_spot_termination ? 1 : 0
+
+  name                              = "karpenter-${var.addon_context.eks_cluster_id}"
+  message_retention_seconds         = 300
+  sqs_managed_sse_enabled           = var.sqs_queue_managed_sse_enabled
+  kms_master_key_id                 = var.sqs_queue_kms_master_key_id
+  kms_data_key_reuse_period_seconds = var.sqs_queue_kms_data_key_reuse_period_seconds
+
+  tags = var.addon_context.tags
+}
+
+resource "aws_sqs_queue_policy" "this" {
+  count = var.enable_spot_termination ? 1 : 0
+
+  queue_url = aws_sqs_queue.this[0].id
+  policy    = data.aws_iam_policy_document.sqs_queue[0].json
+}
+
+resource "aws_cloudwatch_event_rule" "this" {
+  for_each = { for k, v in local.event_rules : k => v if var.enable_spot_termination }
+
+  name          = each.value.name
+  description   = each.value.description
+  event_pattern = jsonencode(each.value.event_pattern)
+  tags = merge(
+    { "ClusterName" : var.addon_context.eks_cluster_id },
+    var.addon_context.tags,
+  )
+}
+
+resource "aws_cloudwatch_event_target" "this" {
+  for_each = { for k, v in local.event_rules : k => v if var.enable_spot_termination }
+
+  rule      = aws_cloudwatch_event_rule.this[each.key].name
+  arn       = aws_sqs_queue.this[0].arn
+  target_id = "KarpenterInterruptionQueueTarget"
+}

modules/kubernetes-addons/karpenter/outputs.tf

@@ -22,3 +22,23 @@ output "service_account" {
   description = "Name of Kubernetes service account"
   value       = module.helm_addon.service_account
 }
+
+output "sqs_queue_arn" {
+  description = "The ARN of the SQS queue"
+  value       = try(aws_sqs_queue.this[0].arn, null)
+}
+
+output "sqs_queue_name" {
+  description = "The name of the created Amazon SQS queue"
+  value       = try(aws_sqs_queue.this[0].name, null)
+}
+
+output "sqs_queue_url" {
+  description = "The URL for the created Amazon SQS queue"
+  value       = try(aws_sqs_queue.this[0].url, null)
+}
+
+output "event_rules" {
+  description = "Map of the event rules created and their attributes"
+  value       = aws_cloudwatch_event_rule.this
+}

modules/kubernetes-addons/karpenter/variables.tf

@@ -22,19 +22,12 @@ variable "node_iam_instance_profile" {
   default     = ""
 }
 
-# tflint-ignore: terraform_unused_declarations
-variable "enable_spot_termination_handling" {
+variable "enable_spot_termination" {
   description = "Determines whether to enable native spot termination handling"
   type        = bool
   default     = false
 }
 
-variable "sqs_queue_arn" {
-  description = "(Optional) ARN of SQS used by Karpenter when native node termination handling is enabled"
-  type        = string
-  default     = ""
-}
-
 variable "addon_context" {
   description = "Input configuration for the addon"
   type = object({
@@ -57,3 +50,21 @@ variable "path" {
   type        = string
   default     = "/"
 }
+
+variable "sqs_queue_managed_sse_enabled" {
+  description = "Enable server-side encryption (SSE) for a SQS queue"
+  type        = bool
+  default     = true
+}
+
+variable "sqs_queue_kms_master_key_id" {
+  description = "The ID of an AWS-managed customer master key (CMK) for Amazon SQS or a custom CMK"
+  type        = string
+  default     = null
+}
+
+variable "sqs_queue_kms_data_key_reuse_period_seconds" {
+  description = "The length of time, in seconds, for which Amazon SQS can reuse a data key to encrypt or decrypt messages before calling AWS KMS again"
+  type        = number
+  default     = null
+}

modules/kubernetes-addons/main.tf

@@ -317,13 +317,15 @@ module "karpenter" {
 
   count = var.enable_karpenter ? 1 : 0
 
-  helm_config                      = var.karpenter_helm_config
-  irsa_policies                    = var.karpenter_irsa_policies
-  node_iam_instance_profile        = var.karpenter_node_iam_instance_profile
-  enable_spot_termination_handling = var.karpenter_enable_spot_termination_handling
-  sqs_queue_arn                    = var.karpenter_sqs_queue_arn
-  manage_via_gitops                = var.argocd_manage_add_ons
-  addon_context                    = local.addon_context
+  helm_config                                 = var.karpenter_helm_config
+  irsa_policies                               = var.karpenter_irsa_policies
+  node_iam_instance_profile                   = var.karpenter_node_iam_instance_profile
+  enable_spot_termination                     = var.karpenter_enable_spot_termination_handling
+  manage_via_gitops                           = var.argocd_manage_add_ons
+  addon_context                               = local.addon_context
+  sqs_queue_managed_sse_enabled               = var.sqs_queue_managed_sse_enabled
+  sqs_queue_kms_master_key_id                 = var.sqs_queue_kms_master_key_id
+  sqs_queue_kms_data_key_reuse_period_seconds = var.sqs_queue_kms_data_key_reuse_period_seconds
 }
 
 module "keda" {
@@ -530,6 +532,7 @@ module "secrets_store_csi_driver" {
   manage_via_gitops = var.argocd_manage_add_ons
   addon_context     = local.addon_context
 }
+
 module "aws_privateca_issuer" {
   count                   = var.enable_aws_privateca_issuer ? 1 : 0
   source                  = "./aws-privateca-issuer"

modules/kubernetes-addons/variables.tf

@@ -911,10 +911,22 @@ variable "karpenter_enable_spot_termination_handling" {
   default     = false
 }
 
-variable "karpenter_sqs_queue_arn" {
-  description = "(Optional) ARN of SQS used by Karpenter when native node termination handling is enabled"
+variable "sqs_queue_managed_sse_enabled" {
+  description = "Enable server-side encryption (SSE) for a SQS queue"
+  type        = bool
+  default     = true
+}
+
+variable "sqs_queue_kms_master_key_id" {
+  description = "The ID of an AWS-managed customer master key (CMK) for Amazon SQS or a custom CMK"
   type        = string
-  default     = ""
+  default     = null
+}
+
+variable "sqs_queue_kms_data_key_reuse_period_seconds" {
+  description = "The length of time, in seconds, for which Amazon SQS can reuse a data key to encrypt or decrypt messages before calling AWS KMS again"
+  type        = number
+  default     = null
 }
 
 #-----------KEDA ADDON-------------