Opensearch Module (#422)

terraform-modules/aws/opensearch/README.md

@@ -0,0 +1,42 @@
+ Terraform module for setting up an AWS OpenSearch domain, This module supports both public and VPC-based deployments, depending on the value of the `vpc_enabled` variable. The module covers the creation of an OpenSearch domain, security group, and associated ingress and egress rules.
+
+**Key Features**
+
+- OpenSearch domain creation with `aws_opensearch_domain` resource.
+- Security group creation with `aws_security_group` resource, including ingress and egress rules.
+- Ingress and egress rules support for CIDR blocks and IPv6 CIDR blocks.
+- Cloudwatch Log groups - To publish slow logs to CloudWatch Log Groups for monitoring and analysis
+
+**Input Variables**
+
+- domain_name (Optional, string): The user-friendly name for the OpenSearch domain. If not provided, Terraform will generate a default domain name.
+- aws_region (Optional, string, default: "us-west-2"): The AWS region where the OpenSearch domain will be created.
+- account_id (Optional, string): The AWS account ID of your AWS account.
+- tags (Optional, any, default: {}): AWS tags that will be applied to the OpenSearch domain and related resources.
+- subnet_ids (Required, list(string)): A list of private subnet IDs within your VPC where the OpenSearch domain will be created.
+- vpc_id (Required, string): The ID of the VPC where the OpenSearch domain will be created.
+- instance_count (Optional, number, default: 2): The number of instances in the OpenSearch domain cluster.
+- ingress_rule (Optional, list(any), default: provided): A list of ingress rules for the OpenSearch domain security group.
+- egress_rule (Optional, list(any), default: provided): A list of egress rules for the OpenSearch domain security group.
+
+**Domain Configuration**
+The OpenSearch domain is created using the aws_opensearch_domain resource with the following settings:
+
+- Engine version: OpenSearch 2.5
+- Cluster instance type: r4.large.search
+- Zone awareness enabled : By Default 2 availability zones
+- EBS storage enabled with 10GB volume size
+- [Encryption at rest](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/opensearch_domain#encrypt_at_rest) and [node-to-node encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/opensearch_domain#node_to_node_encryption) enabled
+- HTTPS enforced with TLS security policy: [Policy-Min-TLS-1-2-2019-07](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/opensearch_domain#tls_security_policy)
+
+- vpc_options: A dynamic block that conditionally creates a VPC configuration for the domain, based on the value of vpc_enabled.
+- access_policies: JSON-encoded access policies for the domain, with a conditional policy that enforces secure transport (HTTPS) if the domain is deployed within a VPC.
+- Log publishing options for index slow logs, search slow logs.
+
+**Security Group Configuration**
+- The security group is created using the aws_security_group resource, and it includes ingress and egress rules for controlling access to the OpenSearch domain.
+
+  - The ingress rules are defined in the ingress_rule variable.
+  - The egress rules are defined in the egress_rule variable.
+
+Both ingress and egress rules support CIDR blocks and IPv6 CIDR blocks.

terraform-modules/aws/opensearch/main.tf

@@ -0,0 +1,143 @@
+resource "aws_cloudwatch_log_group" "opensearch_slow_logs" {
+  name              = "${var.domain_name}-slow-logs"
+  retention_in_days = var.retention_in_days
+}
+
+resource "aws_cloudwatch_log_resource_policy" "opensearch_slow_logs_policy" {
+  policy_name = "${var.domain_name}-policy"
+  policy_document = jsonencode({
+    "Version" : "2012-10-17",
+    "Statement" : [
+      {
+        "Effect" : "Allow",
+        "Principal" : {
+          "Service" : "es.amazonaws.com"
+        },
+        "Action" : [
+          "logs:PutLogEvents",
+          "logs:CreateLogStream",
+          "logs:PutLogEventsBatch"
+        ],
+        "Resource" : "arn:aws:logs:${var.aws_region}:${var.account_id}:log-group:${aws_cloudwatch_log_group.opensearch_slow_logs.name}:*"
+      }
+    ]
+  })
+}
+
+resource "aws_opensearch_domain" "this" {
+  domain_name    = var.domain_name
+  engine_version = "OpenSearch_2.5"
+
+  cluster_config {
+    instance_type          = var.instance_type
+    zone_awareness_enabled = var.zone_awareness_enabled
+    instance_count         = var.instance_count
+  }
+
+  ebs_options {
+    ebs_enabled = var.ebs_enabled
+    volume_size = var.volume_size
+  }
+
+  encrypt_at_rest {
+    enabled = true
+  }
+
+  node_to_node_encryption {
+    enabled = true
+  }
+
+  domain_endpoint_options {
+    enforce_https       = var.enforce_https
+    tls_security_policy = var.tls_security_policy
+  }
+
+  # the dynamic block creates a vpc_options block with the specified security group and subnet IDs.
+  # If the variable vpc_enabled is set to false, the dynamic block is not created, 
+  # and the aws_opensearch_domain resource will not include a vpc_options block, 
+  # creating the OpenSearch domain publicly.
+  dynamic "vpc_options" {
+    for_each = var.vpc_enabled ? [1] : []
+    content {
+      security_group_ids = concat([aws_security_group.opensearch_sg.id], var.additional_security_group_ids)
+      subnet_ids         = var.subnet_ids
+    }
+  }
+
+  # The current configuration with Principal = { AWS = "*" } allows any authenticated AWS user or role to access the OpenSearch domain but the policy enforces that the access must be over a secure transport (HTTPS),
+  # as specified in the Condition block.
+  # Plese refer the documentation for access policies https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/opensearch_domain#access-policy
+  # use variable allowed_roles to update the policy with a user role who can access this
+  access_policies = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "es:*"
+        Effect = "Allow"
+        Principal = {
+          AWS = var.allowed_roles
+        }
+        Resource = "arn:aws:es:${var.aws_region}:${var.account_id}:domain/${var.domain_name}/*"
+        Condition = var.vpc_enabled ? {
+          Bool = {
+            "aws:SecureTransport" = "true"
+          }
+        } : {}
+      }
+    ]
+  })
+
+  # Index and search slow logs are logs generated by OpenSearch (formerly Elasticsearch) to record operations that take longer than a specified threshold. 
+  # These logs help identify performance issues and bottlenecks within the OpenSearch cluster
+
+  # Index Slow Logs: These logs record indexing operations that exceed the specified threshold, usually in terms of time taken to process the operation. 
+  # Indexing operations involve adding, updating, or deleting documents in the OpenSearch index. By analyzing index slow logs, 
+  # you can identify slow indexing operations and investigate potential causes, such as complex mappings, resource constraints, or heavy indexing loads.
+
+  log_publishing_options {
+    cloudwatch_log_group_arn = aws_cloudwatch_log_group.opensearch_slow_logs.arn
+    log_type                 = "INDEX_SLOW_LOGS"
+  }
+
+  # These logs record search and query operations that exceed the specified threshold, usually in terms of time taken to process the operation.
+  # Search operations involve running queries against the OpenSearch index to retrieve documents. By analyzing search slow logs, 
+  # you can identify slow search operations and investigate potential causes, such as inefficient queries, resource constraints, or heavy search loads.
+  log_publishing_options {
+    cloudwatch_log_group_arn = aws_cloudwatch_log_group.opensearch_slow_logs.arn
+    log_type                 = "SEARCH_SLOW_LOGS"
+  }
+
+  depends_on = [aws_security_group.opensearch_sg]
+}
+
+resource "aws_security_group" "opensearch_sg" {
+  name        = "${var.domain_name}-sg"
+  description = "Security group for OpenSearch domain"
+  vpc_id      = var.vpc_id
+
+  dynamic "ingress" {
+    for_each = var.ingress_rule
+    content {
+      description      = ingress.value["description"]
+      from_port        = ingress.value["from_port"]
+      to_port          = ingress.value["to_port"]
+      protocol         = ingress.value["protocol"]
+      cidr_blocks      = ingress.value["cidr_blocks"]
+      ipv6_cidr_blocks = ingress.value["ipv6_cidr_blocks"]
+    }
+  }
+
+  dynamic "egress" {
+    for_each = var.egress_rule
+    content {
+      description      = egress.value["description"]
+      from_port        = egress.value["from_port"]
+      to_port          = egress.value["to_port"]
+      protocol         = egress.value["protocol"]
+      cidr_blocks      = egress.value["cidr_blocks"]
+      ipv6_cidr_blocks = egress.value["ipv6_cidr_blocks"]
+    }
+  }
+
+  tags = var.tags
+}

terraform-modules/aws/opensearch/outputs.tf

@@ -0,0 +1,11 @@
+output "opensearch_domain_endpoint" {
+  value = aws_opensearch_domain.this.endpoint
+}
+
+output "opensearch_security_group_id" {
+  value = aws_security_group.opensearch_sg.id
+}
+
+output "aws_cloudwatch_log_group_arn" {
+  value = aws_cloudwatch_log_group.opensearch_slow_logs.arn
+}

terraform-modules/aws/opensearch/variables.tf

@@ -0,0 +1,133 @@
+variable "domain_name" {
+  type        = string
+  default     = ""
+  description = "(Optional) The friendly name for the Open Search Domain. By default generated by Terraform."
+}
+
+variable "aws_region" {
+  type        = string
+  default     = "us-west-2"
+  description = "The AWS region"
+}
+
+variable "account_id" {
+  type        = string
+  default     = null
+  description = "The account_id of your AWS Account. This allows sure the use of the account number in the role to mitigate issue of aws_caller_id showing *** by obtaining the value of account_id "
+}
+
+variable "enforce_https" {
+  type        = bool
+  default     = true
+  description = "Whether or not to require HTTPS. Defaults to true"
+}
+
+variable "vpc_enabled" {
+  type        = bool
+  description = "enable vpc based open search, the dynamic block creates a vpc_options block with the specified security group and subnet IDs. If the variable is set to false, the dynamic block is not created, and the aws_opensearch_domain resource will not include a vpc_options block, creating the OpenSearch domain publicly."
+  default     = true
+}
+
+variable "zone_awareness_enabled" {
+  type        = bool
+  description = "By Default 2 availability zones"
+  default     = true
+}
+
+variable "ebs_enabled" {
+  type        = bool
+  description = "Ebs Storage enabled or disabled"
+  default     = true
+}
+
+variable "volume_size" {
+  type        = number
+  default     = 10
+  description = "Size of EBS volumes attached to data nodes (in GiB)"
+}
+
+variable "retention_in_days" {
+  type        = number
+  default     = 14
+  description = "Specifies the number of days you want to retain log events in the specified log group"
+}
+
+variable "tags" {
+  type        = any
+  default     = {}
+  description = "AWS Tags"
+}
+
+variable "subnet_ids" {
+  type        = list(string)
+  default     = []
+  description = "(Required) The private subnet IDs in which the environment should be created."
+}
+
+variable "vpc_id" {
+  type        = string
+  default     = ""
+  description = "The vpc ID"
+}
+
+variable "instance_count" {
+  type        = number
+  default     = 2
+  description = "Number of instances in the cluster"
+}
+
+variable "instance_type" {
+  type        = string
+  default     = "r4.large.search"
+  description = "Instance type of data nodes in the cluster."
+}
+
+variable "tls_security_policy" {
+  type        = string
+  default     = "Policy-Min-TLS-1-2-2019-07"
+  description = "HTTPS enforced with TLS security policy"
+}
+
+variable "additional_security_group_ids" {
+  type        = list(string)
+  default     = []
+  description = "Additional security group IDs to be attached to the OpenSearch domain."
+}
+
+variable "allowed_roles" {
+  type        = string
+  description = "A list of AWS role ARNs allowed to access the OpenSearch domain"
+  default     = "*"
+}
+
+variable "ingress_rule" {
+  type        = list(any)
+  description = "A list of ingress rules"
+  default = [
+    {
+      description = "TLS from VPC"
+      //Port 443 is commonly used port for secure HTTPS traffic
+      from_port        = 443
+      to_port          = 443
+      protocol         = "tcp"
+      cidr_blocks      = ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]
+      ipv6_cidr_blocks = []
+    },
+  ]
+}
+
+variable "egress_rule" {
+  type        = list(any)
+  description = "A list of egress rules"
+  default = [
+    {
+      description = "Allow outbound HTTPS traffic to VPC"
+      //Port 443 is commonly used port for secure HTTPS traffic
+      from_port        = 443
+      to_port          = 443
+      protocol         = "tcp"
+      cidr_blocks      = ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]
+      ipv6_cidr_blocks = ["::/0"]
+    },
+  ]
+}