Make Prometheus SSL optional (#98)

* Allow disabling SSL for Prometheus * Update dependencies * Install nano by default * Use RedHat Enterprise Linux 9 for Prometheus repository * Add tests for input validation * Install locust by default * Run terraform test on GitHub
crate · Dec 18, 2023 · 9bcc3c1 · 9bcc3c1
1 parent ad05840
commit 9bcc3c1
Show file tree

Hide file tree

Showing 13 changed files with 287 additions and 155 deletions.
diff --git a/.github/workflows/tflint.yml b/.github/workflows/tflint.yml
@@ -19,6 +19,22 @@ jobs:
       - run: tflint --init
       - run: tflint --format compact
 
+  test_aws:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: aws
+    steps:
+      - uses: actions/checkout@v4
+      - uses: hashicorp/setup-terraform@v3
+      - run: terraform init
+      - shell: bash
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: |
+          terraform test
+
   lint_azure:
     runs-on: ubuntu-latest
     defaults:

diff --git a/aws/.terraform.lock.hcl b/aws/.terraform.lock.hcl
diff --git a/aws/.tflint.hcl b/aws/.tflint.hcl
@@ -1,6 +1,6 @@
 plugin "aws" {
   enabled = true
-  version = "0.24.1"
+  version = "0.28.0"
   source  = "github.com/terraform-linters/tflint-ruleset-aws"
 }
 

diff --git a/aws/README.md b/aws/README.md
@@ -1,96 +1,108 @@
 # CrateDB cluster on EC2 instances using Terraform
+
 This Terraform configuration will launch a CrateDB cluster on AWS. It consists of a public-facing load lancer with and a set of EC2 instances.
 
 ![AWS architecture](aws_architecture.png)
 
 The provided configuration is meant as an easy way to get started. It is not necessarily production-ready in all aspects, such as backups, high availability, and security. Please clone and extend the configuration to fit your individual needs, if needed.
 
 ## Setup
+
 The Terraform configuration generates by default an individual self-signed SSL certificate. If `crate.ssl_enable` is set to false, SSL will be disabled.
 For a full list of available variables (including disk configuration), please see [variables.tf](variables.tf).
 
 The main setup consists of the following steps:
+
 1. Crate a new `main.tf` Terraform configuration, referencing the CrateDB module:
 
-  ```hcl
-  module "cratedb-cluster" {
-    source = "[email protected]:crate/crate-terraform.git//aws"
+    ```hcl
+    module "cratedb-cluster" {
+      source = "[email protected]:crate/crate-terraform.git//aws"
 
-    # Global configuration items for naming/tagging resources
-    config = {
-      project_name = "example-project"
-      environment  = "test"
-      owner        = "Crate.IO"
-      team         = "Customer Engineering"
-    }
+      # Global configuration items for naming/tagging resources
+      config = {
+        project_name = "example-project"
+        environment  = "test"
+        owner        = "Crate.IO"
+        team         = "Customer Engineering"
+      }
 
-    # CrateDB-specific configuration
-    crate = {
-      # Java Heap size in GB available to CrateDB
-      heap_size_gb = 2
+      # CrateDB-specific configuration
+      crate = {
+        # Java Heap size in GB available to CrateDB
+        heap_size_gb = 2
 
-      cluster_name = "crate-cluster"
+        cluster_name = "crate-cluster"
 
-      # The number of nodes the cluster will consist of
-      cluster_size = 2
+        # The number of nodes the cluster will consist of
+        cluster_size = 2
 
-      # Enables a self-signed SSL certificate
-      ssl_enable = true
-    }
+        # Enables a self-signed SSL certificate
+        ssl_enable = true
+      }
 
-    # The disk size in GB to use for CrateDB's data directory
-    disk_size_gb = 512
+      # The disk size in GB to use for CrateDB's data directory
+      disk_size_gb = 512
 
-    # The AWS region
-    region = "eu-central-1"
+      # The AWS region
+      region = "eu-central-1"
 
-    # The VPC to deploy to
-    vpc_id = "vpc-1234567"
+      # The VPC to deploy to
+      vpc_id = "vpc-1234567"
 
-    # Applicable subnets of the VPC
-    subnet_ids = ["subnet-123456", "subnet-123457"]
+      # Applicable subnets of the VPC
+      subnet_ids = ["subnet-123456", "subnet-123457"]
 
-    # The corresponding availability zones of above subnets
-    availability_zones = ["eu-central-1b", "eu-central-1a"]
+      # The corresponding availability zones of above subnets
+      availability_zones = ["eu-central-1b", "eu-central-1a"]
 
-    # The SSH key pair for EC2 instances
-    ssh_keypair = "cratedb-cluster"
+      # The SSH key pair for EC2 instances
+      ssh_keypair = "cratedb-cluster"
 
-    # Enable SSH access to EC2 instances
-    ssh_access = true
-  }
+      # Enable SSH access to EC2 instances
+      ssh_access = true
+    }
 
-  output "cratedb" {
-    value     = module.cratedb-cluster
-    sensitive = true
-  }
-```
+    output "cratedb" {
+      value     = module.cratedb-cluster
+      sensitive = true
+    }
+    ```
 
 2. Run `terraform init` to download and install all needed providers.
 
 ## Execution
+
 To run the Terraform configuration:
+
 1. Run `terraform plan` to validate the planned resource creation
 2. Run `terraform apply` to execute the plan
 3. Run `terraform output -json` to view the cleartext output, such as the CrateDB URL and login credentials
 
 ## Accessing CrateDB
+
 The above last-mentioned step will output all needed information to connect to CrateDB. This includes the publicly accessible URL of the load balancer, as well as login credentials. On opening this URL in a browser, an HTTP Basic Auth appears.
 
 Please note that it might take a couple of minutes before instances are fully provisioned and CrateDB becomes accessible.
 
 ## Accessing EC2 instances
+
 Your EC2 instances will only have a public IP address if the corresponding VPC subnet is configured to [auto-assign](https://docs.aws.amazon.com/vpc/latest/userguide/vpc-ip-addressing.html) public IP addresses.
 
 Connecting via SSH can be done using the `ec2-user` account and the configured key pair. In the default configuration, SSH access is enabled in the security group. It can be disabled if needed via the `ssh_access` variable.
 
 ## Utility EC2 instance
+
 Optionally, you can provision an additional EC2 instance that will not run CrateDB. Instead, it can be used to run benchmarks or other client applications. It is located in the same VPC and subnet as the CrateDB nodes for optimal network latency.
 
 Connect to the EC2 instance using the `ec2-user` account and the configured key pair. The host and port for SSH connections is available via the output variables `utility_vm_host` and `utility_vm_port`.
 
 ## Crate JMX Exporter
+
 The [Crate JMX Exporter](https://github.com/crate/jmx_exporter) exposes monitoring metrics in the Prometheus format. It is available through the load balancer on port 8080. Independent of the `crate.ssl_enable` setting, the endpoint is always accessible through `http`.
 
 ## Prometheus
-[Prometheus](https://prometheus.io) is capturing the export of the Crate JMX Exporter. It is available through the load balancer on port 9090 through `https` with a self-signed certificate. Basic authentication is in place with the user `admin` and the password provided in the output variable `utility_vm_prometheus_password`.
+
+[Prometheus](https://prometheus.io) is capturing the export of the Crate JMX Exporter. It is available through the load balancer on port 9090 through `https` with a self-signed certificate. Basic authentication is in place with the user `admin` and the password provided in the output variable `utility_vm_prometheus_password`.
+
+Specify `prometheus_ssl = false` if you prefer Prometheus not to use SSL.
diff --git a/aws/ec2.tf b/aws/ec2.tf
@@ -40,16 +40,18 @@ data "cloudinit_config" "config" {
     content_type = "text/cloud-config"
     content = templatefile("${path.module}/scripts/cloud-init-cratedb-${var.cratedb_tar_download_url == null ? "rpm" : "tar"}.tftpl",
       {
-        crate_download_url    = var.cratedb_tar_download_url
-        crate_user            = local.config.crate_username
-        crate_pass            = local.cratedb_password
-        crate_heap_size       = var.crate.heap_size_gb
-        crate_cluster_name    = var.crate.cluster_name
-        crate_cluster_size    = var.crate.cluster_size
-        crate_nodes_ips       = indent(12, yamlencode(aws_network_interface.interface[*].private_ip))
-        crate_ssl_enable      = var.crate.ssl_enable
-        crate_ssl_certificate = base64encode(tls_self_signed_cert.ssl.cert_pem)
-        crate_ssl_private_key = base64encode(tls_private_key.ssl.private_key_pem)
+        user_provisioning_file = indent(6, file(("${path.module}/scripts/user_provisioning.sh")))
+        crate_download_url     = var.cratedb_tar_download_url
+        crate_user             = local.config.crate_username
+        crate_pass             = local.cratedb_password
+        crate_heap_size        = var.crate.heap_size_gb
+        crate_cluster_name     = var.crate.cluster_name
+        crate_cluster_size     = var.crate.cluster_size
+        crate_nodes_ips        = indent(12, yamlencode(aws_network_interface.interface[*].private_ip))
+        crate_ssl_enable       = var.crate.ssl_enable
+        crate_protocol         = var.crate.ssl_enable ? "https" : "http"
+        crate_ssl_certificate  = base64encode(tls_self_signed_cert.ssl.cert_pem)
+        crate_ssl_private_key  = base64encode(tls_private_key.ssl.private_key_pem)
       }
     )
   }