[terraform] #874: terraform module for GCP (#886)

* [terraform] #874: terraform module for gcp

* Add desired db versions as a variable

* Format and simplify commons

* Fix variable name consistency

* Default crdb_node_count to 3

* Add required crdb_node_count to example definitions

* Add /build/workspace to the repository

* Add a note about GCP login

* Reorganize the files to use composition instead of encapsulation

* Move README temporarily to terraform-google-dss

* Refactor variables and use module composition for terraform-google-dss

* Add utility to manage variables of tf modules

* Update variables and example.tfvars

* Format

* Fix examples

* Remove redundant crdb_internal_addresses and adapt make-certs to handle joining cluster

* Fix link in readme

* Fix link in readme

* Update documentation

* Fix link in readme

* Update deploy/infrastructure/modules/terraform-google-dss/README.md

Co-authored-by: Benjamin Pelletier <[email protected]>

* Update deploy/infrastructure/modules/terraform-google-dss/README.md

Co-authored-by: Benjamin Pelletier <[email protected]>

* Update deploy/infrastructure/modules/terraform-google-dss/README.md

Co-authored-by: Benjamin Pelletier <[email protected]>

* Apply suggestions from code review regarding default values

Co-authored-by: Benjamin Pelletier <[email protected]>

* Add missing cd as suggested in PR

* Update deploy/infrastructure/modules/terraform-google-dss/terraform.example.tfvars

Co-authored-by: Benjamin Pelletier <[email protected]>

* Address PR comments

- Update build/deploy/db_schemas/README.md
- Change kubernetes_storage_class.tf to google_kubernetes_storage_class.tf
- Add "latest" value to specify default db schema version
- Move variables descriptions to TFVARS.md instead of the example file
- Improve google_zone documentation and add list of options
- Fix us-demo.pem path
- Include dummy auth option in the authentication variable documentation

* TF format

* Add latest value for image variable

* Add latest value for image and revert default for kubernetes_namespace

* Fail bash script in case of error

* Improve instructions

* Expose cluster context as outpu

* Fix key path

* Propose test key by default in example file

* Split step 3 in instructions as suggested in PR

* Clarify cluster context folder.

Co-authored-by: Benjamin Pelletier <[email protected]>

build/deploy/db_schemas/README.md

@@ -20,3 +20,4 @@ places:
 * [Schema manager main.jsonnet](../examples/schema_manager/main.jsonnet)
 * scd_ or rid_ bootstrapper.sh in [dev/startup](../../dev/startup)
 * /pkg/{rid|scd}/store/cockroach/store.go
+* /deploy/infrastructure/dependencies/terraform-commons-dss/default_latest.tf

deploy/infrastructure/.gitignore

@@ -0,0 +1,5 @@
+.terraform/
+.terraform*
+terraform.tfstate
+terraform.tfstate.backup
+personal/

deploy/infrastructure/dependencies/terraform-commons-dss/README.md

@@ -0,0 +1,11 @@
+# terraform-commons-dss
+
+This folder contains a terraform module which gathers resources required by all cloud providers.
+
+It currently consists of the automatic generation of the tanka configuration to deploy 
+the Kubernetes resources as well as the scripts required to generate the certificates 
+and operate the cluster.
+
+## Configuration
+
+See [variables.tf](variables.tf).

deploy/infrastructure/dependencies/terraform-commons-dss/default_latest.tf

@@ -0,0 +1,5 @@
+locals {
+ rid_db_schema = var.desired_rid_db_version == "latest" ? "4.0.0" : var.desired_rid_db_version
+ scd_db_schema = var.desired_scd_db_version == "latest" ? "3.1.0" : var.desired_scd_db_version
+ image = var.image == "latest" ? "docker.io/interuss/dss:v0.6.0" : var.image
+}

deploy/infrastructure/dependencies/terraform-commons-dss/main.tf

@@ -0,0 +1,62 @@
+locals {
+ workspace_location = abspath("${path.module}/../../../../build/workspace/${var.kubernetes_context_name}")
+}
+
+resource "local_file" "tanka_config_main" {
+ content = templatefile("${path.module}/templates/main.jsonnet.tmp", {
+ root_path = path.module
+ VAR_NAMESPACE = var.kubernetes_namespace
+ VAR_CLUSTER_CONTEXT = var.kubernetes_context_name
+ VAR_ENABLE_SCD = var.enable_scd
+ VAR_CRDB_HOSTNAME_SUFFIX = var.crdb_hostname_suffix
+ VAR_CRDB_LOCALITY = var.crdb_locality
+ VAR_CRDB_NODE_IPS = join(",", [for i in var.crdb_internal_nodes[*].ip : "'${i}'"])
+ VAR_INGRESS_NAME = var.ip_gateway
+ VAR_CRDB_EXTERNAL_NODES = join(",", [for a in var.crdb_external_nodes : "'${a}'"])
+ VAR_STORAGE_CLASS = var.kubernetes_storage_class
+ VAR_DOCKER_IMAGE_NAME = local.image
+ VAR_APP_HOSTNAME = var.app_hostname
+ VAR_PUBLIC_KEY_PEM_PATH = var.authorization.public_key_pem_path != null ? var.authorization.public_key_pem_path : ""
+ VAR_JWKS_ENDPOINT = var.authorization.jwks != null ? var.authorization.jwks.endpoint : ""
+ VAR_JWKS_KEY_ID = var.authorization.jwks != null ? var.authorization.jwks.key_id : ""
+ VAR_DESIRED_RID_DB_VERSION = local.rid_db_schema
+ VAR_DESIRED_SCD_DB_VERSION = local.scd_db_schema
+ VAR_SHOULD_INIT = var.should_init
+ })
+ filename = "${local.workspace_location}/main.jsonnet"
+}
+
+resource "local_file" "tanka_config_spec" {
+ content = templatefile("${path.module}/templates/spec.json.tmp", {
+ root_path = path.module
+ namespace = var.kubernetes_namespace
+ cluster_context = var.kubernetes_context_name
+ api_server = var.kubernetes_api_endpoint
+ })
+ filename = "${local.workspace_location}/spec.json"
+}
+
+resource "local_file" "make_certs" {
+ content = templatefile("${path.module}/templates/make-certs.sh.tmp", {
+ cluster_context = var.kubernetes_context_name
+ namespace = var.kubernetes_namespace
+ node_address = join(" ", var.crdb_internal_nodes[*].dns)
+ joining_pool = length(var.crdb_external_nodes) > 0
+ })
+ filename = "${local.workspace_location}/make-certs.sh"
+}
+
+resource "local_file" "apply_certs" {
+ content = templatefile("${path.module}/templates/apply-certs.sh.tmp", {
+ cluster_context = var.kubernetes_context_name
+ namespace = var.kubernetes_namespace
+ })
+ filename = "${local.workspace_location}/apply-certs.sh"
+}
+
+resource "local_file" "get_credentials" {
+ content = templatefile("${path.module}/templates/get-credentials.sh.tmp", {
+ get_credentials_cmd = var.kubernetes_get_credentials_cmd
+ })
+ filename = "${local.workspace_location}/get-credentials.sh"
+}

deploy/infrastructure/dependencies/terraform-commons-dss/output.tf

@@ -0,0 +1,10 @@
+output "generated_files_location" {
+ value = <<-EOT
+ Generated files location:
+ - workspace: ${local.workspace_location}
+ - main.jsonnet: ${abspath(local_file.tanka_config_main.filename)}
+ - spec.json: ${abspath(local_file.tanka_config_spec.filename)}
+ - make-certs.sh: ${abspath(local_file.make_certs.filename)}
+ - apply-certs.sh: ${abspath(local_file.apply_certs.filename)}
+ EOT
+}

deploy/infrastructure/dependencies/terraform-commons-dss/templates/apply-certs.sh.tmp

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+# This file was automatically generated by terraform-commons-dss.
+# Do not edit it directly.
+
+set -eo pipefail
+
+OS=$(uname)
+if [[ "$OS" == "Darwin" ]]; then
+ # OSX uses BSD readlink
+ BASEDIR="$(dirname "$0")"
+else
+ BASEDIR=$(readlink -e "$(dirname "$0")")
+fi
+cd "$BASEDIR/../.." || exit 1
+
+./apply-certs.sh ${cluster_context} ${namespace}

deploy/infrastructure/dependencies/terraform-commons-dss/templates/get-credentials.sh.tmp

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+# This file was automatically generated by terraform-commons-dss.
+# Do not edit it directly.
+
+set -eo pipefail
+
+OS=$(uname)
+if [[ "$OS" == "Darwin" ]]; then
+ # OSX uses BSD readlink
+ BASEDIR="$(dirname "$0")"
+else
+ BASEDIR=$(readlink -e "$(dirname "$0")")
+fi
+cd "$BASEDIR/../.." || exit 1
+
+${get_credentials_cmd}

deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp

@@ -0,0 +1,43 @@
+// This file was automatically generated by terraform-commons-dss.
+// Do not edit it directly.
+
+local dss = import '../../deploy/dss.libsonnet';
+local metadataBase = import '../../deploy/metadata_base.libsonnet';
+
+// All VAR_* values below must be replaced with appropriate values; see
+// dss/build/README.md for more information.
+
+local metadata = metadataBase {
+ namespace: '${VAR_NAMESPACE}',
+ clusterName: '${VAR_CLUSTER_CONTEXT}',
+ enable_istio: false,
+ single_cluster: false,
+ enableScd: ${VAR_ENABLE_SCD}, // <-- This boolean value is VAR_ENABLE_SCD
+ cockroach+: {
+ hostnameSuffix: '${VAR_CRDB_HOSTNAME_SUFFIX}',
+ locality: '${VAR_CRDB_LOCALITY}',
+ nodeIPs: [${VAR_CRDB_NODE_IPS}],
+ shouldInit: ${VAR_SHOULD_INIT},
+ JoinExisting: [${VAR_CRDB_EXTERNAL_NODES}],
+ storageClass: '${VAR_STORAGE_CLASS}',
+ },
+ backend+: {
+ ipName: '${VAR_INGRESS_NAME}',
+ image: '${VAR_DOCKER_IMAGE_NAME}',
+ pubKeys: ['${VAR_PUBLIC_KEY_PEM_PATH}'],
+ jwksEndpoint: '${VAR_JWKS_ENDPOINT}',
+ jwksKeyIds: ['${VAR_JWKS_KEY_ID}'],
+ hostname: '${VAR_APP_HOSTNAME}',
+ dumpRequests: true,
+ },
+ schema_manager+: {
+ image: '${VAR_DOCKER_IMAGE_NAME}',
+ desired_rid_db_version: '${VAR_DESIRED_RID_DB_VERSION}',
+ desired_scd_db_version: '${VAR_DESIRED_SCD_DB_VERSION}',
+ },
+ prometheus+: {
+ storageClass: '${VAR_STORAGE_CLASS}',
+ },
+};
+
+dss.all(metadata)

deploy/infrastructure/dependencies/terraform-commons-dss/templates/make-certs.sh.tmp

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# This file was automatically generated by terraform-commons-dss.
+# Do not edit it directly.
+
+set -eo pipefail
+
+%{ if joining_pool }
+CA_CERT=$${1:?Please, provide the CA certificate of the pool to join as first argument. See /build/README.md for more details.}
+CA_CERT_ARG=" --ca-cert-to-join $${CA_CERT}"
+%{ else }
+CA_CERT_ARG=""
+%{ endif }
+
+OS=$(uname)
+if [[ "$OS" == "Darwin" ]]; then
+ # OSX uses BSD readlink
+ BASEDIR="$(dirname "$0")"
+else
+ BASEDIR=$(readlink -e "$(dirname "$0")")
+fi
+cd "$BASEDIR/../.." || exit 1
+
+python ./make-certs.py --cluster-context ${cluster_context} --namespace ${namespace} --node-address ${node_address}$${CA_CERT_ARG}

deploy/infrastructure/dependencies/terraform-commons-dss/templates/spec.json.tmp

@@ -0,0 +1,11 @@
+{
+ "apiVersion": "tanka.dev/v1alpha1",
+ "kind": "Environment",
+ "metadata": {
+ "name": "${cluster_context}"
+ },
+ "spec": {
+ "apiServer": "${api_server}",
+ "namespace": "${namespace}"
+ }
+}

deploy/infrastructure/dependencies/terraform-commons-dss/variables.tf

@@ -0,0 +1,169 @@
+
+# This file has been automatically generated by /deploy/infrastructure/utils/generate_terraform_variables.py.
+# Please do not modify manually.
+
+variable "image" {
+ type = string
+ description = <<EOT
+ Full name of the docker image built in the section above. build.sh prints this name as
+ the last thing it does when run with DOCKER_URL set. It should look something like
+ gcr.io/your-project-id/dss:2020-07-01-46cae72cf if you built the image yourself as
+ documented in /build/README.md, or docker.io/interuss/dss.
+
+ `latest` can be used to use the latest official interuss docker image.
+
+ Example: `latest` or `docker.io/interuss/dss:v0.6.0`
+ EOT
+}
+
+variable "authorization" {
+ type = object({
+ public_key_pem_path = optional(string)
+ jwks = optional(object({
+ endpoint = string
+ key_id = string
+ }))
+ })
+ description = <<EOT
+ One of `public_key_pem_path` or `jwks` should be provided but not both.
+
+ - public_key_pem_path
+ If providing the access token public key via JWKS, do not provide this parameter.
+ If providing a .pem file directly as the public key to validate incoming access tokens, specify the name
+ of this .pem file here as /public-certs/YOUR-KEY-NAME.pem replacing YOUR-KEY-NAME as appropriate. For instance,
+ if using the provided us-demo.pem, use the path /public-certs/us-demo.pem. Note that your .pem file should built
+ in the docker image or mounted manually.
+ Example:
+ ```json
+ Example 1 (dummy auth):
+ {
+ public_key_pem_path = "/test-certs/auth2.pem"
+ }
+ Example 2:
+ {
+ public_key_pem_path = "/jwt-public-certs/us-demo.pem"
+ }
+ ```
+
+ - jwks
+ If providing a .pem file directly as the public key to validate incoming access tokens, do not provide this parameter.
+ - endpoint
+ If providing the access token public key via JWKS, specify the JWKS endpoint here.
+ Example: https://auth.example.com/.well-known/jwks.json
+ - key_id:
+ If providing the access token public key via JWKS, specify the kid (key ID) of they appropriate key in the JWKS file referenced above.
+ Example:
+ ```json
+ {
+ jwks = {
+ endpoint = "https://auth.example.com/.well-known/jwks.json"
+ key_id = "9C6DF78B-77A7-4E89-8990-E654841A7826"
+ }
+ }
+ ```
+ EOT
+
+ validation {
+ condition = (var.authorization.jwks == null && var.authorization.public_key_pem_path != null) || (var.authorization.jwks != null && var.authorization.public_key_pem_path == null)
+ error_message = "Public key to validate incoming access tokens shall be provided exclusively either with a .pem file or via JWKS."
+ }
+}
+
+variable "enable_scd" {
+ type = bool
+ description = "Set this boolean true to enable ASTM strategic conflict detection functionality"
+ default = true
+}
+
+variable "should_init" {
+ type = bool
+ description = <<-EOT
+ Set to false if joining an existing pool, true if creating the first DSS instance
+ for a pool. When set true, this can initialize the data directories on your cluster,
+ and prevent you from joining an existing pool.
+
+ Example: `true`
+ EOT
+}
+
+variable "desired_rid_db_version" {
+ type = string
+ description = <<EOT
+ Desired RID DB schema version.
+ Use `latest` to use the latest schema version.
+
+ Example: `4.0.0`
+ EOT
+
+ default = "latest"
+}
+
+variable "desired_scd_db_version" {
+ type = string
+ description = <<EOT
+ Desired SCD DB schema version.
+ Use `latest` to use the latest schema version.
+
+ Example: `3.1.0`
+ EOT
+
+ default = "latest"
+}
+
+variable "crdb_locality" {
+ type = string
+ description = <<-EOT
+ Unique name for your DSS instance. Currently, we recommend "<ORG_NAME>_<CLUSTER_NAME>",
+ and the = character is not allowed. However, any unique (among all other participating
+ DSS instances) value is acceptable.
+
+ Example: <ORGNAME_CLUSTER_NAME>
+ EOT
+}
+
+variable "crdb_external_nodes" {
+ type = list(string)
+ description = <<-EOT
+ Fully-qualified domain name of existing CRDB nodes outside of the cluster if you are joining an existing pool.
+ Example: ["0.db.dss.example.com", "1.db.dss.example.com", "2.db.dss.example.com"]
+ EOT
+ default = []
+}
+
+variable "kubernetes_namespace" {
+ type = string
+ description = <<-EOT
+ Namespace where to deploy Kubernetes resources. Only default is supported at the moment.
+
+ Example: `default`
+ EOT
+
+ default = "default"
+
+ # TODO: Adapt current deployment scripts in /build/deploy to support default is supported for the moment.
+ validation {
+ condition = var.kubernetes_namespace == "default"
+ error_message = "Only default namespace is supported at the moment"
+ }
+}
+
+variable "app_hostname" {
+ type = string
+ description = <<-EOT
+ Fully-qualified domain name of your HTTPS Gateway ingress endpoint.
+
+ Example: `dss.example.com`
+ EOT
+}
+
+variable "crdb_hostname_suffix" {
+ type = string
+ description = <<-EOT
+ The domain name suffix shared by all of your CockroachDB nodes.
+ For instance, if your CRDB nodes were addressable at 0.db.example.com,
+ 1.db.example.com and 2.db.example.com, then the value would be db.example.com.
+
+ Example: db.example.com
+ EOT
+}
+