From 1a8dfa1b694b6e58f86213c9ea5da7bf84b98c8d Mon Sep 17 00:00:00 2001 From: Michael Barroco Date: Mon, 7 Nov 2022 10:28:31 +0100 Subject: [PATCH] [terraform] #874: terraform module for gcp --- deploy/infrastructure/.gitignore | 5 + deploy/infrastructure/examples/DNS.md | 40 +++++++ deploy/infrastructure/examples/README.md | 68 ++++++++++++ .../examples/interuss-mini-google/main.tf | 10 ++ .../examples/interuss-mini-google/output.tf | 8 ++ .../interuss-mini-google/terraform.tfvars | 48 ++++++++ .../examples/interuss-prod-google/main.tf | 10 ++ .../examples/interuss-prod-google/output.tf | 8 ++ .../interuss-prod-google/terraform.tfvars | 48 ++++++++ .../terraform-commons-dss/README.md | 13 +++ .../terraform-commons-dss/main.tf | 0 .../terraform-commons-dss/output.tf | 11 ++ .../templates/apply-certs.sh.tmp | 17 +++ .../templates/get-credentials.sh.tmp | 17 +++ .../templates/main.jsonnet.tmp | 46 ++++++++ .../templates/make-certs.sh.tmp | 17 +++ .../templates/spec.json.tmp | 11 ++ .../terraform-commons-dss/variables.tf | 103 ++++++++++++++++++ .../terraform-commons-dss/workspace_files.tf | 61 +++++++++++ .../terraform-google-dss/README.md | 6 + .../terraform-google-dss/cluster.tf | 57 ++++++++++ .../terraform-google-dss/dns.tf | 25 +++++ .../terraform-google-dss/main.tf | 24 ++++ .../terraform-google-dss/output.tf | 7 ++ .../terraform-google-dss/variables.tf | 30 +++++ 25 files changed, 690 insertions(+) create mode 100644 deploy/infrastructure/.gitignore create mode 100644 deploy/infrastructure/examples/DNS.md create mode 100644 deploy/infrastructure/examples/README.md create mode 100644 deploy/infrastructure/examples/interuss-mini-google/main.tf create mode 100644 deploy/infrastructure/examples/interuss-mini-google/output.tf create mode 100644 deploy/infrastructure/examples/interuss-mini-google/terraform.tfvars create mode 100644 deploy/infrastructure/examples/interuss-prod-google/main.tf create mode 100644 deploy/infrastructure/examples/interuss-prod-google/output.tf create mode 100644 deploy/infrastructure/examples/interuss-prod-google/terraform.tfvars create mode 100644 deploy/infrastructure/terraform-commons-dss/README.md create mode 100644 deploy/infrastructure/terraform-commons-dss/main.tf create mode 100644 deploy/infrastructure/terraform-commons-dss/output.tf create mode 100644 deploy/infrastructure/terraform-commons-dss/templates/apply-certs.sh.tmp create mode 100644 deploy/infrastructure/terraform-commons-dss/templates/get-credentials.sh.tmp create mode 100644 deploy/infrastructure/terraform-commons-dss/templates/main.jsonnet.tmp create mode 100644 deploy/infrastructure/terraform-commons-dss/templates/make-certs.sh.tmp create mode 100644 deploy/infrastructure/terraform-commons-dss/templates/spec.json.tmp create mode 100644 deploy/infrastructure/terraform-commons-dss/variables.tf create mode 100644 deploy/infrastructure/terraform-commons-dss/workspace_files.tf create mode 100644 deploy/infrastructure/terraform-google-dss/README.md create mode 100644 deploy/infrastructure/terraform-google-dss/cluster.tf create mode 100644 deploy/infrastructure/terraform-google-dss/dns.tf create mode 100644 deploy/infrastructure/terraform-google-dss/main.tf create mode 100644 deploy/infrastructure/terraform-google-dss/output.tf create mode 100644 deploy/infrastructure/terraform-google-dss/variables.tf diff --git a/deploy/infrastructure/.gitignore b/deploy/infrastructure/.gitignore new file mode 100644 index 000000000..fb5e06e5b --- /dev/null +++ b/deploy/infrastructure/.gitignore @@ -0,0 +1,5 @@ +.terraform/ +.terraform* +terraform.tfstate +terraform.tfstate.backup +personal/ diff --git a/deploy/infrastructure/examples/DNS.md b/deploy/infrastructure/examples/DNS.md new file mode 100644 index 000000000..0280de99a --- /dev/null +++ b/deploy/infrastructure/examples/DNS.md @@ -0,0 +1,40 @@ +# Setup DNS + +This page describes the options and steps required to setup DNS for a DSS deployment. + +## Terraform managed + +If your DNS zone is managed on the same account, it is possible to instruct terraform to create and manage +it with the rest of the infrastructure. + +- **For Google Cloud Engine**, the terraform module provides the `dns_managed_zone_name` in `google_cluster_context`. If the variable + is set with a zone name which can be listed by running `gcloud dns managed-zones list`, entries will be + automatically created. + +## Manual setup + +If DNS entries are managed manually, set them up manually using the following steps: + +1. Retrieve IP addresses and expected hostnames: `terraform output` + Example of expected output: + ``` + crdb_addresses = [ + { + "address" = "34.65.15.23" + "expected_dns" = "0.interuss.example.com" + }, + { + "address" = "34.65.146.56" + "expected_dns" = "1.interuss.example.com" + }, + { + "address" = "34.65.191.145" + "expected_dns" = "2.interuss.example.com" + }, + ] + gateway_address = { + "address" = "35.186.236.146" + "expected_dns" = "dss.interuss.example.com" + } + +2. Create the related DNS A entries to point to the static ips. diff --git a/deploy/infrastructure/examples/README.md b/deploy/infrastructure/examples/README.md new file mode 100644 index 000000000..971aef030 --- /dev/null +++ b/deploy/infrastructure/examples/README.md @@ -0,0 +1,68 @@ +# DSS Infrastructure Examples + +This folder contains deployment examples for various environments: +- Cloud Provider: + - *-google: Google Cloud Engine +- Deployment size: + - mini-*: Low resources for development + - prod-*: Production grade resources + +## Infrastructure + +### Prerequisites +Download & install the following tools to your workstation: + +1. Install [terraform](https://developer.hashicorp.com/terraform/downloads). +2. Install provider specific tools: + 1. [Google Cloud Engine](./README.md#google-cloud-engine) +3. Install tools from [Prerequisites](../../../build/README.md) + +#### Google Cloud Engine + +1. Install and initialize [Google Cloud CLI](https://cloud.google.com/sdk/docs/install-sdk). + 1. Confirm successful installation with `gcloud version`. +2. Check that the DSS project is correctly selected: gcloud config list project + 1. Set another one if needed using: `gcloud config set project $GOOGLE_PROJECT_NAME` +3. Enable the following API using [Google Cloud CLI](https://cloud.google.com/endpoints/docs/openapi/enable-api#gcloud): + 1. `container.googleapis.com` + 2. If you want to manage DNS entries with terraform: `dns.googleapis.com` +4. Install the auth plugin to connect to kubernetes: `gcloud components install gke-gcloud-auth-plugin` + +### Deployment of the Kubernetes cluster + +1. Copy or edit in place an example folder to `/deploy/infrastructure/personal/`. (Note that the modules can be added to existing projects) +2. Edit `terraform.tfvars` and set the variables according to your environment. +3. Initialize terraform: `terraform init`. +4. Run `terraform plan` to check that the configuration is valid. It will display the resources which will be provisioned. +5. Run `terraform apply` to deploy the cluster. (This operation may take up to 15 min.) + +#### Note on DNS + +DNS entries can be either managed manually or handled by terraform depending on the cloud provider. +See [DNS](DNS.md) for details. + +## Deployment of the DSS services + +During the successful run, the terraform job has created a new [workspace](../../../build/workspace/) +for the new cluster. + +It contains scripts to operate the cluster and setup the services. + +1. Go to `/build/workspace/${CLUSTER_CONTEXT}`. +2. Run `./get_credentials.sh` to login to kubernetes. You can now access the cluster with `kubectl`. +3. Generate the certificates `./make-certs.sh`. Follow script instructions if you are not initializing the cluster. +4. Deploy the certificates `./apply-certs.sh`. +5. Run `tk apply .` to deploy the services to kubernetes. (This may take up to 30 min) +6. Wait for services to initialize. Verify that basic services are functioning by navigating to https://your-gateway-domain.com/healthy. + + - On Google Cloud, the highest-latency operation is provisioning of the HTTPS certificate which generally takes 10-45 minutes. To track this progress: + - Go to the "Services & Ingress" left-side tab from the Kubernetes Engine page. + - Click on the https-ingress item (filter by just the cluster of interest if you have multiple clusters in your project). + - Under the "Ingress" section for Details, click on the link corresponding with "Load balancer". + - Under Frontend for Details, the Certificate column for HTTPS protocol will have an icon next to it which will change to a green checkmark when provisioning is complete. + - Click on the certificate link to see provisioning progress. + - If everything indicates OK and you still receive a cipher mismatch error message when attempting to visit /healthy, wait an additional 5 minutes before attempting to troubleshoot further. + +## Clean up + +To delete all resources, run `terraform destroy`. Note that this operation can't be reverted and all data will be lost. \ No newline at end of file diff --git a/deploy/infrastructure/examples/interuss-mini-google/main.tf b/deploy/infrastructure/examples/interuss-mini-google/main.tf new file mode 100644 index 000000000..41834c019 --- /dev/null +++ b/deploy/infrastructure/examples/interuss-mini-google/main.tf @@ -0,0 +1,10 @@ +# See ../../terraform-google-dss/variables.tf for required schema. +variable "google_cluster_context" {} +variable "dss_configuration" {} + +module "terraform-google-dss" { + source = "../../terraform-google-dss" + google_cluster_context = var.google_cluster_context + dss_configuration = var.dss_configuration +} + diff --git a/deploy/infrastructure/examples/interuss-mini-google/output.tf b/deploy/infrastructure/examples/interuss-mini-google/output.tf new file mode 100644 index 000000000..a031cfbab --- /dev/null +++ b/deploy/infrastructure/examples/interuss-mini-google/output.tf @@ -0,0 +1,8 @@ + +output "crdb_addresses" { + value = module.terraform-google-dss.crdb_addresses +} + +output "gateway_address" { + value = module.terraform-google-dss.gateway_address +} \ No newline at end of file diff --git a/deploy/infrastructure/examples/interuss-mini-google/terraform.tfvars b/deploy/infrastructure/examples/interuss-mini-google/terraform.tfvars new file mode 100644 index 000000000..8a7142404 --- /dev/null +++ b/deploy/infrastructure/examples/interuss-mini-google/terraform.tfvars @@ -0,0 +1,48 @@ +google_cluster_context = { + # Name of the new cluster. + name = "interuss-mini-w6a" + + # Name of the GCP project hosting the future cluster. + project = "" + + # GCP Region where to deploy the cluster. + region = "europe-west6" + + # GCP Zone where to deploy the cluster + zone = "europe-west6-a" + + # GCP machine type used for the Kubernetes node pool. + # Example: n2-standard-4 for production, e2-micro for development + machine_type = "e2-micro" + + # GCP DNS zone name to automatically manage DNS entries. Leave it empty to manage it manually. + dns_managed_zone_name = "" +} + +dss_configuration = { + # See build/README.md (Deploying a DSS via Kubernetes, section 11) for variables description. + + namespace = "default" + + # image = "" # Use default. VAR_DOCKER_IMAGE_NAME + + storage_class = "standard" # VAR_STORAGE_CLASS + + enable_scd = true # VAR_ENABLE_SCD + + should_init = true # VAR_SHOULD_INIT + + app_hostname = "" # VAR_APP_HOSTNAME + + public_key_pem_path = "" # VAR_PUBLIC_KEY_PEM_PATH + + jwks_endpoint = "" # VAR_JWKS_ENDPOINT + + jwks_key_id = "" # VAR_JWKS_KEY_ID + + crdb_hostname_suffix = "interuss.example.com" # VAR_CRDB_HOSTNAME_SUFFIX + + crdb_external_nodes = [] # VAR_EXTERNAL_CRDB_NODEn + + crdb_locality = "" # VAR_CRDB_LOCALITY +} diff --git a/deploy/infrastructure/examples/interuss-prod-google/main.tf b/deploy/infrastructure/examples/interuss-prod-google/main.tf new file mode 100644 index 000000000..41834c019 --- /dev/null +++ b/deploy/infrastructure/examples/interuss-prod-google/main.tf @@ -0,0 +1,10 @@ +# See ../../terraform-google-dss/variables.tf for required schema. +variable "google_cluster_context" {} +variable "dss_configuration" {} + +module "terraform-google-dss" { + source = "../../terraform-google-dss" + google_cluster_context = var.google_cluster_context + dss_configuration = var.dss_configuration +} + diff --git a/deploy/infrastructure/examples/interuss-prod-google/output.tf b/deploy/infrastructure/examples/interuss-prod-google/output.tf new file mode 100644 index 000000000..a031cfbab --- /dev/null +++ b/deploy/infrastructure/examples/interuss-prod-google/output.tf @@ -0,0 +1,8 @@ + +output "crdb_addresses" { + value = module.terraform-google-dss.crdb_addresses +} + +output "gateway_address" { + value = module.terraform-google-dss.gateway_address +} \ No newline at end of file diff --git a/deploy/infrastructure/examples/interuss-prod-google/terraform.tfvars b/deploy/infrastructure/examples/interuss-prod-google/terraform.tfvars new file mode 100644 index 000000000..bbf2ac4d1 --- /dev/null +++ b/deploy/infrastructure/examples/interuss-prod-google/terraform.tfvars @@ -0,0 +1,48 @@ +google_cluster_context = { + # Name of the new cluster. + name = "interuss-mini-w6a" + + # Name of the GCP project hosting the future cluster. + project = "" + + # GCP Region where to deploy the cluster. + region = "europe-west6" + + # GCP Zone where to deploy the cluster + zone = "europe-west6-a" + + # GCP machine type used for the Kubernetes node pool. + # Example: n2-standard-4 for production, e2-micro for development + machine_type = "n2-standard-4" + + # GCP DNS zone name to automatically manage DNS entries. Leave it empty to manage it manually. + dns_managed_zone_name = "" +} + +dss_configuration = { + # See build/README.md (Deploying a DSS via Kubernetes, section 11) for variables description. + + namespace = "default" + + # image = "" # Use default. VAR_DOCKER_IMAGE_NAME + + storage_class = "standard" # VAR_STORAGE_CLASS + + enable_scd = true # VAR_ENABLE_SCD + + should_init = true # VAR_SHOULD_INIT + + app_hostname = "" # VAR_APP_HOSTNAME + + public_key_pem_path = "" # VAR_PUBLIC_KEY_PEM_PATH + + jwks_endpoint = "" # VAR_JWKS_ENDPOINT + + jwks_key_id = "" # VAR_JWKS_KEY_ID + + crdb_hostname_suffix = "interuss.example.com" # VAR_CRDB_HOSTNAME_SUFFIX + + crdb_external_nodes = [] # VAR_EXTERNAL_CRDB_NODEn + + crdb_locality = "" # VAR_CRDB_LOCALITY +} diff --git a/deploy/infrastructure/terraform-commons-dss/README.md b/deploy/infrastructure/terraform-commons-dss/README.md new file mode 100644 index 000000000..5a0435432 --- /dev/null +++ b/deploy/infrastructure/terraform-commons-dss/README.md @@ -0,0 +1,13 @@ +# terraform-google-dss + +This folder contains a terraform module which gathers resources used by all cloud providers. + +It includes the automatic generation of the tanka configuration to deploy the Kubernetes resources +as well as the scripts required to generate the certificates and operate the cluster. + +See `examples/` for configuration examples. + + +## Configuration + +See [variables.tf](./variables.tf) to configure the dss services. diff --git a/deploy/infrastructure/terraform-commons-dss/main.tf b/deploy/infrastructure/terraform-commons-dss/main.tf new file mode 100644 index 000000000..e69de29bb diff --git a/deploy/infrastructure/terraform-commons-dss/output.tf b/deploy/infrastructure/terraform-commons-dss/output.tf new file mode 100644 index 000000000..173dd0171 --- /dev/null +++ b/deploy/infrastructure/terraform-commons-dss/output.tf @@ -0,0 +1,11 @@ + +output "generated_files_location" { + value = <<-EOT + Generated files location: + - workspace: ${local.workspace_location} + - main.jsonnet: ${abspath(local_file.tanka_config_main.filename)} + - spec.json: ${abspath(local_file.tanka_config_spec.filename)} + - make-certs.sh: ${abspath(local_file.make_certs.filename)} + - apply-certs.sh: ${abspath(local_file.apply_certs.filename)} + EOT +} \ No newline at end of file diff --git a/deploy/infrastructure/terraform-commons-dss/templates/apply-certs.sh.tmp b/deploy/infrastructure/terraform-commons-dss/templates/apply-certs.sh.tmp new file mode 100644 index 000000000..11a3e56a9 --- /dev/null +++ b/deploy/infrastructure/terraform-commons-dss/templates/apply-certs.sh.tmp @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# This file was automatically generated by terraform-commons-dss. +# Do not edit it directly. + +set -eo pipefail + +OS=$(uname) +if [[ "$OS" == "Darwin" ]]; then + # OSX uses BSD readlink + BASEDIR="$(dirname "$0")" +else + BASEDIR=$(readlink -e "$(dirname "$0")") +fi +cd "$BASEDIR/../.." || exit 1 + +./apply-certs.sh ${cluster_context} ${namespace} \ No newline at end of file diff --git a/deploy/infrastructure/terraform-commons-dss/templates/get-credentials.sh.tmp b/deploy/infrastructure/terraform-commons-dss/templates/get-credentials.sh.tmp new file mode 100644 index 000000000..c280c374b --- /dev/null +++ b/deploy/infrastructure/terraform-commons-dss/templates/get-credentials.sh.tmp @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# This file was automatically generated by terraform-commons-dss. +# Do not edit it directly. + +set -eo pipefail + +OS=$(uname) +if [[ "$OS" == "Darwin" ]]; then + # OSX uses BSD readlink + BASEDIR="$(dirname "$0")" +else + BASEDIR=$(readlink -e "$(dirname "$0")") +fi +cd "$BASEDIR/../.." || exit 1 + +${get_credentials_cmd} \ No newline at end of file diff --git a/deploy/infrastructure/terraform-commons-dss/templates/main.jsonnet.tmp b/deploy/infrastructure/terraform-commons-dss/templates/main.jsonnet.tmp new file mode 100644 index 000000000..6db1af58c --- /dev/null +++ b/deploy/infrastructure/terraform-commons-dss/templates/main.jsonnet.tmp @@ -0,0 +1,46 @@ +// This file was automatically generated by terraform-commons-dss. +// Do not edit it directly. + +local dss = import '../../deploy/dss.libsonnet'; +local metadataBase = import '../../deploy/metadata_base.libsonnet'; + +// All VAR_* values below must be replaced with appropriate values; see +// dss/build/README.md for more information. + +local metadata = metadataBase { + namespace: '${VAR_NAMESPACE}', + clusterName: '${VAR_CLUSTER_CONTEXT}', + enable_istio: false, + single_cluster: false, + enableScd: ${VAR_ENABLE_SCD}, // <-- This boolean value is VAR_ENABLE_SCD + cockroach+: { + hostnameSuffix: '${VAR_CRDB_HOSTNAME_SUFFIX}', + locality: '${VAR_CRDB_LOCALITY}', + nodeIPs: [${VAR_CRDB_NODE_IPS}], + shouldInit: ${VAR_SHOULD_INIT}, + JoinExisting: [${VAR_CRDB_EXTERNAL_NODES}], + storageClass: '${VAR_STORAGE_CLASS}', + }, + gateway+: { + ipName: '${VAR_INGRESS_NAME}', + image: '${VAR_DOCKER_IMAGE_NAME}', + hostname: '${VAR_APP_HOSTNAME}', + traceRequests: true, + }, + backend+: { + image: '${VAR_DOCKER_IMAGE_NAME}', + pubKeys: ['${VAR_PUBLIC_KEY_PEM_PATH}'], + jwksEndpoint: '${VAR_JWKS_ENDPOINT}', + jwksKeyIds: ['${VAR_JWKS_KEY_ID}'], + }, + schema_manager+: { + image: '${VAR_DOCKER_IMAGE_NAME}', + desired_rid_db_version: '${VAR_DESIRED_RID_DB_VERSION}', + desired_scd_db_version: '${VAR_DESIRED_SCD_DB_VERSION}', + }, + prometheus+: { + storageClass: '${VAR_STORAGE_CLASS}', + }, +}; + +dss.all(metadata) \ No newline at end of file diff --git a/deploy/infrastructure/terraform-commons-dss/templates/make-certs.sh.tmp b/deploy/infrastructure/terraform-commons-dss/templates/make-certs.sh.tmp new file mode 100644 index 000000000..78b1575ba --- /dev/null +++ b/deploy/infrastructure/terraform-commons-dss/templates/make-certs.sh.tmp @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# This file was automatically generated by terraform-commons-dss. +# Do not edit it directly. + +set -eo pipefail + +OS=$(uname) +if [[ "$OS" == "Darwin" ]]; then + # OSX uses BSD readlink + BASEDIR="$(dirname "$0")" +else + BASEDIR=$(readlink -e "$(dirname "$0")") +fi +cd "$BASEDIR/../.." || exit 1 + +python ./make-certs.py --cluster-context ${cluster_context} --namespace ${namespace} --node-address ${node_address} diff --git a/deploy/infrastructure/terraform-commons-dss/templates/spec.json.tmp b/deploy/infrastructure/terraform-commons-dss/templates/spec.json.tmp new file mode 100644 index 000000000..07fa4be59 --- /dev/null +++ b/deploy/infrastructure/terraform-commons-dss/templates/spec.json.tmp @@ -0,0 +1,11 @@ +{ + "apiVersion": "tanka.dev/v1alpha1", + "kind": "Environment", + "metadata": { + "name": "${cluster_context}" + }, + "spec": { + "apiServer": "${api_server}", + "namespace": "${VAR_NAMESPACE}" + } +} \ No newline at end of file diff --git a/deploy/infrastructure/terraform-commons-dss/variables.tf b/deploy/infrastructure/terraform-commons-dss/variables.tf new file mode 100644 index 000000000..7a5587d3b --- /dev/null +++ b/deploy/infrastructure/terraform-commons-dss/variables.tf @@ -0,0 +1,103 @@ +# Public schema used as input by provider specific modules. +variable "dss_configuration" { + type = object({ + # Kubernetes + + # Namespace where to deploy Kubernetes resources + # TODO: Adapt current deployment scripts to support default is supported for the moment. + namespace = optional(string, "default") + + # Full name of the docker image built in the section above. build.sh prints this name as + # the last thing it does when run with DOCKER_URL set. It should look something like + # gcr.io/your-project-id/dss:2020-07-01-46cae72cf if you built the image yourself, or docker.io/interuss/dss + # TODO: Test DOCKER_URL usage as documented in /build/README.md + image = optional(string, "docker.io/interuss/dss:v0.4.0") + + # Infrastructure + + # Number of pool nodes. + # Example: 3 for production. 1 for development. + crdb_node_count = number + + # Kubernetes Storage Class to use for CockroachDB and Prometheus volumes. You can + # check your cluster's possible values with kubectl get storageclass. + # Example standard + storage_class = string + + + # DSS Functionalities + + # Set this boolean true to enable ASTM strategic conflict detection functionality. + enable_scd = bool + + # Set to false if joining an existing pool, true if creating the first DSS instance + # for a pool. When set true, this can initialize the data directories on your cluster, + # and prevent you from joining an existing pool. + should_init = optional(bool, false) + + # Fully-qualified domain name of your HTTPS Gateway ingress endpoint. + # Example, dss.example.com. + app_hostname = string + + + # Authorization + + # If providing a .pem file directly as the public key to validate incoming access tokens, specify the name + # of this .pem file here as /public-certs/YOUR-KEY-NAME.pem replacing YOUR-KEY-NAME as appropriate. For instance, + # if using the provided us-demo.pem, use the path /public-certs/us-demo.pem. Note that your .pem file should built + # in the docker image or mounted manually. + # TODO: Add ability to provide the key content + public_key_pem_path = string + + # If providing the access token public key via JWKS, specify the JWKS endpoint here. + # Example: https://auth.example.com/.well-known/jwks.json + jwks_endpoint = string + + # If providing the access token public key via JWKS, specify the kid (key ID) of they appropriate key in the JWKS file referenced above. + # If providing a .pem file directly as the public key to valid incoming access tokens, provide a blank string for this parameter. + jwks_key_id = string + + # Database + + # The domain name suffix shared by all of your CockroachDB nodes. For instance, + # if your CRDB nodes were addressable at 0.db.example.com, 1.db.example.com, and + # 2.db.example.com, then VAR_CRDB_HOSTNAME_SUFFIX would be db.example.com. + # Example: db.example.com + crdb_hostname_suffix = string + + # Unique name for your DSS instance. Currently, we recommend "_", + # and the = character is not allowed. However, any unique (among all other participating + # DSS instances) value is acceptable. + crdb_locality = string # + + # Fully-qualified domain name of your HTTPS Gateway ingress endpoint. For example, dss.example.com. + crdb_external_nodes = list(string) + }) + + validation { + condition = var.dss_configuration.should_init == true && length(var.dss_configuration.crdb_external_nodes) == 0 + error_message = "crdb_external_nodes should be empty when should_init is set to true" + } + + # TODO: Adapt current deployment scripts in /build/deploy to support default is supported for the moment. + validation { + condition = var.dss_configuration.namespace == "default" + error_message = "Only default namespace is supported at the moment" + } +} + +# Internal schema +variable "kubernetes" { + type = object({ + provider_name = string + get_credentials_cmd = string + kubectl_cluster_context_name = string + api_endpoint = string + node_addresses = list(string) + crdb_nodes = list(object({ + dns = string + ip = string + })) + ip_gateway = string + }) +} \ No newline at end of file diff --git a/deploy/infrastructure/terraform-commons-dss/workspace_files.tf b/deploy/infrastructure/terraform-commons-dss/workspace_files.tf new file mode 100644 index 000000000..36aec191e --- /dev/null +++ b/deploy/infrastructure/terraform-commons-dss/workspace_files.tf @@ -0,0 +1,61 @@ +locals { + workspace_location = abspath("${path.module}/../../../build/workspace/${var.kubernetes.kubectl_cluster_context_name}") +} + +resource "local_file" "tanka_config_main" { + content = templatefile("${path.module}/templates/main.jsonnet.tmp", { + root_path = path.module + VAR_NAMESPACE = var.dss_configuration.namespace + VAR_CLUSTER_CONTEXT = var.kubernetes.kubectl_cluster_context_name + VAR_ENABLE_SCD = false + VAR_CRDB_HOSTNAME_SUFFIX = var.dss_configuration.crdb_hostname_suffix + VAR_CRDB_LOCALITY = var.dss_configuration.crdb_locality + VAR_CRDB_NODE_IPS = join(",", [for i in var.kubernetes.crdb_nodes[*].ip : "'${i}'"]) + VAR_INGRESS_NAME = var.kubernetes.ip_gateway + VAR_CRDB_EXTERNAL_NODES = join(",", [for a in var.dss_configuration.crdb_external_nodes : "'${a}'"]) + VAR_STORAGE_CLASS = var.dss_configuration.storage_class + VAR_DOCKER_IMAGE_NAME = var.dss_configuration.image + VAR_APP_HOSTNAME = var.dss_configuration.app_hostname + VAR_PUBLIC_KEY_PEM_PATH = var.dss_configuration.public_key_pem_path + VAR_JWKS_ENDPOINT = var.dss_configuration.jwks_endpoint + VAR_JWKS_KEY_ID = var.dss_configuration.jwks_key_id + VAR_DESIRED_RID_DB_VERSION = "4.0.0" + VAR_DESIRED_SCD_DB_VERSION = "3.1.0" + VAR_SHOULD_INIT = var.dss_configuration.should_init + }) + filename = "${local.workspace_location}/main.jsonnet" +} + +resource "local_file" "tanka_config_spec" { + content = templatefile("${path.module}/templates/spec.json.tmp", { + root_path = path.module + VAR_NAMESPACE = var.dss_configuration.namespace + cluster_context = var.kubernetes.kubectl_cluster_context_name + api_server = var.kubernetes.api_endpoint + }) + filename = "${local.workspace_location}/spec.json" +} + +resource "local_file" "make_certs" { + content = templatefile("${path.module}/templates/make-certs.sh.tmp", { + cluster_context = var.kubernetes.kubectl_cluster_context_name + namespace = var.dss_configuration.namespace + node_address = join(" ", var.kubernetes.node_addresses) + }) + filename = "${local.workspace_location}/make-certs.sh" +} + +resource "local_file" "apply_certs" { + content = templatefile("${path.module}/templates/apply-certs.sh.tmp", { + cluster_context = var.kubernetes.kubectl_cluster_context_name + namespace = var.dss_configuration.namespace + }) + filename = "${local.workspace_location}/apply-certs.sh" +} + +resource "local_file" "get_credentials" { + content = templatefile("${path.module}/templates/get-credentials.sh.tmp", { + get_credentials_cmd = var.kubernetes.get_credentials_cmd + }) + filename = "${local.workspace_location}/get-credentials.sh" +} \ No newline at end of file diff --git a/deploy/infrastructure/terraform-google-dss/README.md b/deploy/infrastructure/terraform-google-dss/README.md new file mode 100644 index 000000000..27d23d4c7 --- /dev/null +++ b/deploy/infrastructure/terraform-google-dss/README.md @@ -0,0 +1,6 @@ +# terraform-google-dss + +This folder contains a terraform module to deploy the infrastructure required to run the DSS on Kubernetes in Google Cloud Engine. +It includes resources common to all cloud providers by using `[terraform-commons-dss](../terraform-commons-dss)`. + +See `examples/` for configuration examples. diff --git a/deploy/infrastructure/terraform-google-dss/cluster.tf b/deploy/infrastructure/terraform-google-dss/cluster.tf new file mode 100644 index 000000000..fde799f38 --- /dev/null +++ b/deploy/infrastructure/terraform-google-dss/cluster.tf @@ -0,0 +1,57 @@ +# Resources related to the kubernetes cluster + +resource "google_container_cluster" "kubernetes_cluster" { + name = var.google_cluster_context.name + location = var.google_cluster_context.zone + + remove_default_node_pool = true + initial_node_count = 1 + + networking_mode = "VPC_NATIVE" + ip_allocation_policy { + # Intentionally left empty. + } +} + +resource "google_container_node_pool" "dss_pool" { + name = "dss-pool" + location = var.google_cluster_context.zone + cluster = google_container_cluster.kubernetes_cluster.name + node_count = var.dss_configuration.crdb_node_count + + node_config { + machine_type = var.google_cluster_context.machine_type + + # TODO: Use non-default service account with IAM roles + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform" + ] + } + + lifecycle { + create_before_destroy = true + } +} + +# Static IP addresses for the gateway +resource "google_compute_global_address" "ip_gateway" { + name = format("%s-ip-gateway", var.google_cluster_context.name) + ip_version = "IPV4" + + # Current google terraform provider doesn't allow tags or labels. Description is used to preserve mapping between ips and hostnames. + description = var.dss_configuration.app_hostname +} + +# Static IP addresses for CRDB instances +resource "google_compute_address" "ip_crdb" { + count = var.dss_configuration.crdb_node_count + name = format("%s-ip-crdb%v", var.google_cluster_context.name, count.index) + region = var.google_cluster_context.region + + # Current google terraform provider doesn't allow tags or labels. Description is used to preserve mapping between ips and hostnames. + description = format("%s.%s", count.index, var.dss_configuration.crdb_hostname_suffix) +} + +locals { + kubectl_cluster_context_name = format("gke_%s_%s_%s", google_container_cluster.kubernetes_cluster.project, google_container_cluster.kubernetes_cluster.location, google_container_cluster.kubernetes_cluster.name) +} \ No newline at end of file diff --git a/deploy/infrastructure/terraform-google-dss/dns.tf b/deploy/infrastructure/terraform-google-dss/dns.tf new file mode 100644 index 000000000..a21435c89 --- /dev/null +++ b/deploy/infrastructure/terraform-google-dss/dns.tf @@ -0,0 +1,25 @@ +# Resources related to the DNS entries if managed. (See google_cluster_context.dns_managed_zone_name configuration field for details) +data "google_dns_managed_zone" "default" { + name = var.google_cluster_context.dns_managed_zone_name +} + +resource "google_dns_record_set" "gateway" { + count = var.google_cluster_context.dns_managed_zone_name == "" ? 0 : 1 + name = "${google_compute_global_address.ip_gateway.description}." # description contains the expected hostname + type = "A" + ttl = 300 + + managed_zone = data.google_dns_managed_zone.default.name + rrdatas = [google_compute_global_address.ip_gateway.address] + +} + +resource "google_dns_record_set" "crdb" { + count = var.google_cluster_context.dns_managed_zone_name == "" ? 0 : var.dss_configuration.crdb_node_count + name = "${google_compute_address.ip_crdb[count.index].description}." # description contains the expected hostname + type = "A" + ttl = 300 + + managed_zone = data.google_dns_managed_zone.default.name + rrdatas = [google_compute_address.ip_crdb[count.index].address] +} \ No newline at end of file diff --git a/deploy/infrastructure/terraform-google-dss/main.tf b/deploy/infrastructure/terraform-google-dss/main.tf new file mode 100644 index 000000000..b3b703e68 --- /dev/null +++ b/deploy/infrastructure/terraform-google-dss/main.tf @@ -0,0 +1,24 @@ +# Providers +provider "google" { + region = var.google_cluster_context.region + project = var.google_cluster_context.project +} + +module "terraform-commons-dss" { + source = "../terraform-commons-dss" + + dss_configuration = var.dss_configuration + + kubernetes = { + provider_name = "google" + get_credentials_cmd = "gcloud container clusters get-credentials --zone ${var.google_cluster_context.zone} ${var.google_cluster_context.name}" + api_endpoint = google_container_cluster.kubernetes_cluster.endpoint + kubectl_cluster_context_name = local.kubectl_cluster_context_name + node_addresses = google_compute_address.ip_crdb[*].description + ip_gateway = google_compute_global_address.ip_gateway.name + crdb_nodes = [for i in google_compute_address.ip_crdb : { + ip = i.address + dns = i.description + }] + } +} \ No newline at end of file diff --git a/deploy/infrastructure/terraform-google-dss/output.tf b/deploy/infrastructure/terraform-google-dss/output.tf new file mode 100644 index 000000000..9aae7841c --- /dev/null +++ b/deploy/infrastructure/terraform-google-dss/output.tf @@ -0,0 +1,7 @@ +output "crdb_addresses" { + value = [for a in google_compute_address.ip_crdb[*] : { expected_dns : a.description, address : a.address }] +} + +output "gateway_address" { + value = { expected_dns : google_compute_global_address.ip_gateway.description, address : google_compute_global_address.ip_gateway.address } +} \ No newline at end of file diff --git a/deploy/infrastructure/terraform-google-dss/variables.tf b/deploy/infrastructure/terraform-google-dss/variables.tf new file mode 100644 index 000000000..ce0f9b8bc --- /dev/null +++ b/deploy/infrastructure/terraform-google-dss/variables.tf @@ -0,0 +1,30 @@ +variable "google_cluster_context" { + type = object({ + # Name of the new cluster. + name = string + + # Name of the GCP project hosting the future cluster. + region = string + + # GCP Region where to deploy the cluster. + zone = string + + # GCP Zone where to deploy the cluster + project = string + + # GCP machine type used for the Kubernetes node pool. + # Example: n2-standard-4 for production, e2-micro for development + dns_managed_zone_name = optional(string, "") + + # GCP DNS zone name to automatically manage DNS entries. Leave it empty to manage it manually. + machine_type = optional(string, "n2-standard-4") + }) + + validation { + condition = startswith(var.google_cluster_context.zone, var.google_cluster_context.region) + error_message = "The zone shall be part of the region" + } +} + +# Intentionally left undefined to reuse schema validation from terraform-commons-dss +variable "dss_configuration" {} \ No newline at end of file