Merge pull request #13 from aigisuk/develop

☁️ add cluster/database firewalls; pre-install cert-manager

README.md

@@ -1,19 +1,22 @@
 # Terraform DigitalOcean HA K3S Module
 An opinionated Terraform module to provision a high availability [K3s](https://k3s.io/) cluster with external database on the DigitalOcean cloud platform. Perfect for development or testing.
 
-![Terraform, DigitalOcean, K3s illustration](https://res.cloudinary.com/qunux/image/upload/v1618967113/terraform-digitalocean-k3s-repo-logo_f2zyoz.svg)
+![k3s_cluster_project_on_digitalocean](https://user-images.githubusercontent.com/12916656/118027873-5508ca00-b35a-11eb-9346-4a605942857b.png)
 
 ## Features
 * [x] High Availability K3s Cluster provisioned on the DigitalOcean platform
 * [x] Managed **PostgreSQL**/**MySQL** database provisioned. Serves as the datastore for the cluster's state (configurable options: size & node count)
-* [x] The number of provisioned Servers (Masters) and Agents (Workers) is configurable
-* [x] Cluster API/Servers are behind a provisioned load balancer for high availability
+* [x] Dedicated VPC provisioned for cluster use (IP Range: `10.10.10.0/24`)
+* [x] Number of provisioned Servers (Masters) and Agents (Workers) is configurable
+* [x] Cluster API/Server(s) are behind a provisioned load balancer for high availability
+* [x] All resources assigned to a dedicated DigitalOcean project (expect Load Balancers auto provisioned by apps)
 * [x] Flannel backend is configurable. Choose from `vxlan`, `host-gw`, `ipsec` (default) or `wireguard`
 * [x] DigitalOcean's CCM ([Cloud Controller Manager](https://github.com/digitalocean/digitalocean-cloud-controller-manager)) and CSI ([Container Storage Interface](https://github.com/digitalocean/csi-digitalocean)) plugins are pre-installed. Enables the cluster to leverage DigitalOcean's load balancer and volume resources
 * [x] Option to make Servers (Masters) schedulable. Default is `false` i.e. `CriticalAddonsOnly=true:NoExecute`
-* [x] Cluster database engine is configurable. Choose from **PostgreSQL** (v11) or **MySQL** (v8)
+* [x] Cluster database engine is configurable. Choose between **PostgreSQL** (v11) or **MySQL** (v8)
 * [x] Pre-install the Kubernetes Dashboard (optional)
 * [x] Pre-install Jetstack's [cert-manager](https://github.com/jetstack/cert-manager) (optional)
+* [x] Firewalled Nodes & Database
 * [ ] Pre-install an ingress controller from **Kong**, **Nginx** or **Traefik v2** (optional)
 * [ ] Generate custom `kubeconfig` file (optional)
 
@@ -44,6 +47,40 @@ module "do-ha-k3s" {
   ssh_key_fingerprints      = ["00:11:22:33:44:55:66:77:88:99:aa:bb:cc:dd:ee:ff"]
 }
 ```
+Example output:
+```
+cluster_summary = {
+  "agents" = [
+    {
+      "id" = "246685594"
+      "ip_private" = "10.10.10.4"
+      "ip_public" = "203.0.113.10"
+      "name" = "k3s-agent-fra1-1a9f-1"
+      "price" = 10
+    },
+  ]
+  "api_server_ip" = "198.51.100.10"
+  "cluster_region" = "fra1"
+  "servers" = [
+    {
+      "id" = "246685751"
+      "ip_private" = "10.10.10.5"
+      "ip_public" = "203.0.113.11"
+      "name" = "k3s-server-fra1-55b4-1"
+      "price" = 10
+    },
+    {
+      "id" = "246685808"
+      "ip_private" = "10.10.10.6"
+      "ip_public" = "203.0.113.12"
+      "name" = "k3s-server-fra1-d6e7-2"
+      "price" = 10
+    },
+  ]
+}
+```
+
+> To manage K3s from outside the cluster, SSH into any Server node and copy the contents of `/etc/rancher/k3s/k3s.yaml` to `~/.kube/config` on an external machine where you have installed `kubectl`, replacing `127.0.0.1` with the API Load Balancer IP address of your K3s Cluster (the `api_server_ip` key from the Terraform `cluster_summary` output).
 
 Functional examples are included in the
 [examples](./examples/) directory.
@@ -55,12 +92,13 @@ Functional examples are included in the
 | do_token | DigitalOcean Personal Access Token | string | N/A | yes |
 | ssh_key_fingerprints | List of SSH Key fingerprints | list(string) | N/A | yes |
 | region | Region in which to deploy cluster | string | `fra1` | no |
+| vpc_network_range | Range of IP addresses for the VPC in CIDR notation | string | `10.10.10.0/24` | no |
 | k3s_channel | K3s release channel. `stable`, `latest`, `testing` or a specific channel or version e.g. `v1.20`, `v1.19.8+k3s1` | string | `"stable"` | no |
 | database_user | Database username | string | `"k3s_default_user"` | no |
 | database_engine | Database engine. `postgres` (v13) or `mysql` (v8) | string | `"postgres"` | no |
 | database_size | Database Droplet size associated with the cluster e.g. `db-s-1vcpu-1gb` | string |`"db-s-1vcpu-1gb"` | no |
 | database_node_count | Number of nodes that comprise the database cluster | number | `1`| no |
-| flannel_backend | Flannel Backend Type. Valid options include `vxlan`, `host-gw`, `ipsec` (default) or `wireguard` | string | `ipsec`| no |
+| flannel_backend | Flannel Backend Type. Valid options include `vxlan`, `ipsec` or `wireguard` | string | `vxlan`| no |
 | server_size | Server droplet size. e.g. `s-1vcpu-2gb` | string | `s-1vcpu-2gb`| no |
 | agent_size | Agent droplet size. e.g. `s-1vcpu-2gb` | string | `s-1vcpu-2gb`| no |
 | server_count | Number of server (master) nodes to provision | number | `2`| no |

agent.tf

@@ -3,7 +3,7 @@ resource "digitalocean_droplet" "k3s_agent" {
   name  = "k3s-agent-${var.region}-${random_id.agent_node_id[count.index].hex}-${count.index + 1}"
 
   image              = "ubuntu-20-04-x64"
-  tags               = ["k3s_agent"]
+  tags               = [local.agent_droplet_tag]
   region             = var.region
   size               = var.agent_size
   monitoring         = true

db_firewall.tf

@@ -0,0 +1,9 @@
+resource "digitalocean_database_firewall" "k3s" {
+
+  cluster_id = digitalocean_database_cluster.k3s.id
+
+  rule {
+    type  = "tag"
+    value = local.server_droplet_tag
+  }
+}

firewall.tf

@@ -0,0 +1,53 @@
+resource "digitalocean_firewall" "ccm_firewall" {
+  name = "ccm-firewall"
+
+  outbound_rule {
+    protocol              = "icmp"
+    destination_addresses = ["0.0.0.0/0", "::/0"]
+  }
+}
+
+resource "digitalocean_firewall" "k3s_firewall" {
+  name = "k3s-firewall"
+
+  tags = [local.server_droplet_tag, local.agent_droplet_tag]
+
+  inbound_rule {
+    # Allow ICMP communication on all ports to defined 'tags' via VPC Network
+    protocol         = "icmp"
+    source_addresses = [var.vpc_network_range]
+  }
+
+  inbound_rule {
+    # Allow TCP communication on all ports to defined 'tags' via VPC Network
+    protocol         = "tcp"
+    port_range       = "1-65535"
+    source_addresses = [var.vpc_network_range]
+  }
+
+  inbound_rule {
+    # Allow UDP communication on all ports to defined 'tags' via VPC Network
+    protocol         = "udp"
+    port_range       = "1-65535"
+    source_addresses = [var.vpc_network_range]
+  }
+
+  inbound_rule {
+    # Allow SSH access from all hosts
+    protocol         = "tcp"
+    port_range       = "22"
+    source_addresses = ["0.0.0.0/0", "::/0"]
+  }
+
+  outbound_rule {
+    protocol              = "udp"
+    port_range            = "1-65535"
+    destination_addresses = ["0.0.0.0/0", "::/0"]
+  }
+
+  outbound_rule {
+    protocol              = "tcp"
+    port_range            = "1-65535"
+    destination_addresses = ["0.0.0.0/0", "::/0"]
+  }
+}

loadbalancer.tf

@@ -17,7 +17,7 @@ resource "digitalocean_loadbalancer" "k3s_lb" {
     protocol = "tcp"
   }
 
-  droplet_tag = "k3s_server"
+  droplet_tag = local.server_droplet_tag
 }
 
 resource "digitalocean_project_resources" "k3s_api_server_lb" {

locals.tf

@@ -1,5 +1,5 @@
 locals {
-  db_host = digitalocean_database_cluster.k3s.host
+  db_host = digitalocean_database_cluster.k3s.private_host
   db_port = digitalocean_database_cluster.k3s.port
   db_user = var.database_user
   db_pass = digitalocean_database_user.dbuser.password
@@ -10,6 +10,10 @@ locals {
 
   db_cluster_uri = var.database_engine == "postgres" ? local.postgres_uri : local.mysql_uri
 
+  server_droplet_tag = "k3s_server"
+  agent_droplet_tag  = "k3s_agent"
+  ccm_fw_tags        = var.server_taint_criticalonly == false ? join(",", [local.server_droplet_tag, local.agent_droplet_tag]) : local.agent_droplet_tag
+
   critical_addons_only_true = "--node-taint \"CriticalAddonsOnly=true:NoExecute\" \\"
 
   taint_critical = var.server_taint_criticalonly == true ? local.critical_addons_only_true : "\\"

manifests/do-ccm.yaml

@@ -48,6 +48,16 @@ spec:
               configMapKeyRef:
                 name: digitalocean
                 key: do-cluster-vpc-id
+          - name: PUBLIC_ACCESS_FIREWALL_NAME
+            valueFrom:
+              configMapKeyRef:
+                name: digitalocean
+                key: public-access-firewall-name
+          - name: PUBLIC_ACCESS_FIREWALL_TAGS
+            valueFrom:
+              configMapKeyRef:
+                name: digitalocean
+                key: public-access-firewall-tags
           - name: DO_ACCESS_TOKEN
             valueFrom:
               secretKeyRef:

server.tf

@@ -3,7 +3,7 @@ resource "digitalocean_droplet" "k3s_server" {
   name  = "k3s-server-${var.region}-${random_id.server_node_id[count.index + 1].hex}-${count.index + 2}"
 
   image              = "ubuntu-20-04-x64"
-  tags               = ["k3s_server"]
+  tags               = [local.server_droplet_tag]
   region             = var.region
   size               = var.server_size
   monitoring         = true

server_init.tf

@@ -3,7 +3,7 @@ resource "digitalocean_droplet" "k3s_server_init" {
   name  = "k3s-server-${var.region}-${random_id.server_node_id[count.index].hex}-1"
 
   image              = "ubuntu-20-04-x64"
-  tags               = ["k3s_server"]
+  tags               = [local.server_droplet_tag]
   region             = var.region
   size               = var.server_size
   monitoring         = true
@@ -15,6 +15,8 @@ resource "digitalocean_droplet" "k3s_server_init" {
     k3s_token           = random_password.k3s_token.result
     do_token            = var.do_token
     do_cluster_vpc_id   = digitalocean_vpc.k3s_vpc.id
+    do_ccm_fw_name      = digitalocean_firewall.ccm_firewall.name
+    do_ccm_fw_tags      = local.ccm_fw_tags
     flannel_backend     = var.flannel_backend
     k3s_lb_ip           = digitalocean_loadbalancer.k3s_lb.ip
     db_cluster_uri      = local.db_cluster_uri

user_data/ks3_server_init.sh

@@ -33,8 +33,8 @@ done
 # create digitalOcean API access token secret
 kubectl -n kube-system create secret generic digitalocean --from-literal=access-token=${do_token}
 
-# create digitalocean env variable configmap
-kubectl -n kube-system create configmap digitalocean --from-literal=do-cluster-vpc-id=${do_cluster_vpc_id}
+# create digitalocean env variables via configmap (for CCM)
+kubectl -n kube-system create configmap digitalocean --from-literal=do-cluster-vpc-id=${do_cluster_vpc_id} --from-literal=public-access-firewall-name=${do_ccm_fw_name} --from-literal=public-access-firewall-tags=${do_ccm_fw_tags}
 
 # install certmanager
 ${cert_manager}

variables.tf

@@ -10,14 +10,20 @@ variable "ssh_key_fingerprints" {
 
 variable "region" {
   type        = string
-  description = "Region in which to deploy the cluster"
+  description = "Region in which to deploy the cluster. Default is fra1 (Frankfurt, Germany)"
   default     = "fra1"
   validation {
     condition     = length(regexall("^nyc1|sfo1|nyc2|ams2|sgp1|lon1|nyc3|ams3|fra1|tor1|sfo2|blr1|sfo3$", var.region)) > 0
     error_message = "Invalid region. Valid regions are nyc1, sfo1, nyc2, ams2, sgp1, lon1, nyc3, ams3, fra1, tor1, sfo2, blr1 or sfo3."
   }
 }
 
+variable "vpc_network_range" {
+  type        = string
+  description = "Range of IP addresses for the VPC in CIDR notation. Cannot be larger than /16 or smaller than /24. Default is 10.10.10.0/24"
+  default     = "10.10.10.0/24"
+}
+
 variable "k3s_channel" {
   type        = string
   description = "K3s release channel. 'stable', 'latest', 'testing' or a specific channel or version e.g. 'v1.20', 'v1.21.0+k3s1'"
@@ -54,11 +60,11 @@ variable "database_node_count" {
 
 variable "flannel_backend" {
   type        = string
-  description = "Flannel Backend Type. Valid options include vxlan, host-gw, ipsec (default) or wireguard"
-  default     = "ipsec"
+  description = "Flannel Backend Type. Valid options include vxlan (default), ipsec or wireguard"
+  default     = "vxlan"
   validation {
-    condition     = length(regexall("^ipsec|vxlan|host-gw|wireguard$", var.flannel_backend)) > 0
-    error_message = "Invalid Flannel backend value. Valid backend types are vxlan, host-gw, ipsec & wireguard."
+    condition     = length(regexall("^ipsec|vxlan|wireguard$", var.flannel_backend)) > 0
+    error_message = "Invalid Flannel backend value. Valid backend types are vxlan, ipsec & wireguard."
   }
 }
 

vpc.tf

@@ -1,4 +1,5 @@
 resource "digitalocean_vpc" "k3s_vpc" {
-  name   = "k3s-vpc-01"
-  region = var.region
+  name     = "k3s-vpc-01"
+  region   = var.region
+  ip_range = var.vpc_network_range
 }