[terraform] interuss#874: terraform module for gcp

deploy/infrastructure/.gitignore

@@ -0,0 +1,5 @@
+.terraform/
+.terraform*
+terraform.tfstate
+terraform.tfstate.backup
+personal/

deploy/infrastructure/examples/DNS.md

@@ -0,0 +1,41 @@
+# Setup DNS
+
+This page describes the options and steps required to setup DNS hostnames to deploy a DSS.
+This step should be undertaken after the infrastructure has been provisioned and the kubernetes 
+cluster is running.
+
+## Manual setup 
+
+If DNS entries are managed manually, set them up manually using the following steps:
+
+1. Retrieve IP addresses and hostnames: `terraform output`
+ Example of expected output:
+ ```
+ crdb_addresses = [
+ {
+ "address" = "34.65.15.23"
+ "expected_dns" = "0.interuss.example.com"
+ },
+ {
+ "address" = "34.65.146.56"
+ "expected_dns" = "1.interuss.example.com"
+ },
+ {
+ "address" = "34.65.191.145"
+ "expected_dns" = "2.interuss.example.com"
+ },
+ ]
+ gateway_address = {
+ "address" = "35.186.236.146"
+ "expected_dns" = "dss.interuss.example.com"
+ }
+2. Create the related DNS A entries to point to the static ips.
+
+## Terraform managed
+
+If your DNS zone is managed on the same account, it is possible to instruct terraform to create and manage
+it with the rest of the infrastructure.
+
+- **For Google Cloud Engine**, the terraform module provides the `dns_managed_zone_name` in `google_cluster_context`. If the variable 
+is set with a zone name which can be listed by running `gcloud dns managed-zones list`, entries will be 
+automatically created.

deploy/infrastructure/examples/README.md

@@ -0,0 +1,66 @@
+# DSS Infrastructure Examples
+
+This folder contains deployment example for various environments:
+- Google Cloud Engine
+
+## Infrastructure
+
+### Prerequisites
+Download & install the following tools to your workstation:
+
+1. Install [terraform](https://developer.hashicorp.com/terraform/downloads).
+2. Install provider specific tools:
+ 1. [Google Cloud Engine](./README.md#google-cloud-engine)
+3. Install tools from [Prerequisites](../../../build/README.md)
+
+
+#### Google Cloud Engine
+
+1. Install and initialize [Google Cloud CLI](https://cloud.google.com/sdk/docs/install-sdk).
+ 1. Confirm successful installation with `gcloud version`.
+2. Check that the DSS project is correctly selected: gcloud config list project
+ 1. Set another one if needed using: `gcloud config set project $GOOGLE_PROJECT_NAME`
+3. Enable the following API using [Google Cloud CLI](https://cloud.google.com/endpoints/docs/openapi/enable-api#gcloud):
+ 1. `container.googleapis.com`
+ 2. If you want to manage DNS entries with terraform: `dns.googleapis.com`
+4. Install the auth plugin to connect to kubernetes: `gcloud components install gke-gcloud-auth-plugin`
+
+
+### Deployment of the Kubernetes cluster
+
+1. Go to an example folder. 
+2. Edit terraform.tfvars and set the variables according to your environment.
+3. Initialize terraform: `terraform init`.
+4. Run `terraform plan` to check that the configuration is valid.
+5. Run `terraform apply` to deploy the cluster. (This operation may take up to 15 min.)
+6. Run `terraform output` and look for the ip addresses.
+
+
+#### Note on DNS
+
+DNS entries can be either managed manually or handled by terraform depending on the cloud provider.
+See [DNS](DNS.md) for details.
+
+## Deployment of the DSS services
+
+The terraform provisioning has created a new workspace with the cluster name [workspace](../../../build/workspace/). 
+It contains scripts to operate the cluster and setup the services.
+
+1. Go to `/build/workspace/${CLUSTER_CONTEXT}`.
+2. Run `./get_credentials.sh` to login to kubernetes.
+3. Generate the certificates `./make-certs.sh`. Follow script instructions if you are not initializing the cluster. 
+4. Deploy the certificates `./apply-certs.sh`.
+5. Run `tk apply .` to deploy the services to kubernetes.
+6. Wait for services to initialize. Verify that basic services are functioning by navigating to https://your-gateway-domain.com/healthy.
+
+ - On Google Cloud, the highest-latency operation is provisioning of the HTTPS certificate which generally takes 10-45 minutes. To track this progress:
+ - Go to the "Services & Ingress" left-side tab from the Kubernetes Engine page.
+ - Click on the https-ingress item (filter by just the cluster of interest if you have multiple clusters in your project).
+ - Under the "Ingress" section for Details, click on the link corresponding with "Load balancer".
+ - Under Frontend for Details, the Certificate column for HTTPS protocol will have an icon next to it which will change to a green checkmark when provisioning is complete.
+ - Click on the certificate link to see provisioning progress.
+ - If everything indicates OK and you still receive a cipher mismatch error message when attempting to visit /healthy, wait an additional 5 minutes before attempting to troubleshoot further.
+
+## Clean up
+
+To delete all resources, run `terraform destroy`. Note that this operation can't be reverted and all data will be lost.

deploy/infrastructure/examples/interuss-mini-google/main.tf

@@ -0,0 +1,10 @@
+# See ../../terraform-google-dss/variables.tf for required schema.
+variable "google_cluster_context" {}
+variable "dss_configuration" {}
+
+module "terraform-google-dss" {
+ source = "../../terraform-google-dss"
+ google_cluster_context = var.google_cluster_context
+ dss_configuration = var.dss_configuration
+}
+

deploy/infrastructure/examples/interuss-mini-google/output.tf

@@ -0,0 +1,8 @@
+
+output "crdb_addresses" {
+ value = module.terraform-google-dss.crdb_addresses
+}
+
+output "gateway_address" {
+ value = module.terraform-google-dss.gateway_address
+}

deploy/infrastructure/examples/interuss-mini-google/terraform.tfvars

@@ -0,0 +1,48 @@
+google_cluster_context = {
+ # Name of the new cluster.
+ name = "interuss-mini-w6a"
+
+ # Name of the GCP project hosting the future cluster.
+ project = ""
+
+ # GCP Region where to deploy the cluster.
+ region = "europe-west6"
+
+ # GCP Zone where to deploy the cluster
+ zone = "europe-west6-a"
+
+ # GCP machine type used for the Kubernetes node pool.
+ # Example: n2-standard-4 for production, e2-micro for development
+ machine_type = "e2-micro"
+
+ # GCP DNS zone name to automatically manage DNS entries. Leave it empty to manage it manually.
+ dns_managed_zone_name = ""
+}
+
+dss_configuration = {
+ # See build/README.md (Deploying a DSS via Kubernetes, section 11) for variables description.
+
+ namespace = "default"
+
+ # image = "" # Use default. VAR_DOCKER_IMAGE_NAME
+
+ storage_class = "standard" # VAR_STORAGE_CLASS
+
+ enable_scd = true # VAR_ENABLE_SCD
+
+ should_init = true # VAR_SHOULD_INIT
+
+ app_hostname = "" # VAR_APP_HOSTNAME
+
+ public_key_pem_path = "" # VAR_PUBLIC_KEY_PEM_PATH
+
+ jwks_endpoint = "" # VAR_JWKS_ENDPOINT
+
+ jwks_key_id = "" # VAR_JWKS_KEY_ID
+
+ crdb_hostname_suffix = "interuss.example.com" # VAR_CRDB_HOSTNAME_SUFFIX
+
+ crdb_external_nodes = [] # VAR_EXTERNAL_CRDB_NODEn
+
+ crdb_locality = "" # VAR_CRDB_LOCALITY
+}

deploy/infrastructure/examples/interuss-prod-google/main.tf

@@ -0,0 +1,10 @@
+# See ../../terraform-google-dss/variables.tf for required schema.
+variable "google_cluster_context" {}
+variable "dss_configuration" {}
+
+module "terraform-google-dss" {
+ source = "../../terraform-google-dss"
+ google_cluster_context = var.google_cluster_context
+ dss_configuration = var.dss_configuration
+}
+

deploy/infrastructure/examples/interuss-prod-google/output.tf

@@ -0,0 +1,8 @@
+
+output "crdb_addresses" {
+ value = module.terraform-google-dss.crdb_addresses
+}
+
+output "gateway_address" {
+ value = module.terraform-google-dss.gateway_address
+}

deploy/infrastructure/examples/interuss-prod-google/terraform.tfvars

@@ -0,0 +1,48 @@
+google_cluster_context = {
+ # Name of the new cluster.
+ name = "interuss-mini-w6a"
+
+ # Name of the GCP project hosting the future cluster.
+ project = ""
+
+ # GCP Region where to deploy the cluster.
+ region = "europe-west6"
+
+ # GCP Zone where to deploy the cluster
+ zone = "europe-west6-a"
+
+ # GCP machine type used for the Kubernetes node pool.
+ # Example: n2-standard-4 for production, e2-micro for development
+ machine_type = "n2-standard-4"
+
+ # GCP DNS zone name to automatically manage DNS entries. Leave it empty to manage it manually.
+ dns_managed_zone_name = ""
+}
+
+dss_configuration = {
+ # See build/README.md (Deploying a DSS via Kubernetes, section 11) for variables description.
+
+ namespace = "default"
+
+ # image = "" # Use default. VAR_DOCKER_IMAGE_NAME
+
+ storage_class = "standard" # VAR_STORAGE_CLASS
+
+ enable_scd = true # VAR_ENABLE_SCD
+
+ should_init = true # VAR_SHOULD_INIT
+
+ app_hostname = "" # VAR_APP_HOSTNAME
+
+ public_key_pem_path = "" # VAR_PUBLIC_KEY_PEM_PATH
+
+ jwks_endpoint = "" # VAR_JWKS_ENDPOINT
+
+ jwks_key_id = "" # VAR_JWKS_KEY_ID
+
+ crdb_hostname_suffix = "interuss.example.com" # VAR_CRDB_HOSTNAME_SUFFIX
+
+ crdb_external_nodes = [] # VAR_EXTERNAL_CRDB_NODEn
+
+ crdb_locality = "" # VAR_CRDB_LOCALITY
+}

deploy/infrastructure/terraform-commons-dss/README.md

@@ -0,0 +1,13 @@
+# terraform-google-dss
+
+This folder contains a terraform module which gathers resources used by all cloud providers.
+
+It includes the automatic generation of the tanka configuration to deploy the Kubernetes resources
+as well as the scripts required to generate the certificates and operate the cluster. 
+
+See `examples/` for configuration examples.
+
+
+## Configuration
+
+See [variables.tf](./variables.tf) to configure the dss services.

deploy/infrastructure/terraform-commons-dss/output.tf

@@ -0,0 +1,11 @@
+
+output "generated_files_location" {
+ value = <<-EOT
+ Generated files location:
+ - workspace: ${local.workspace_location}
+ - main.jsonnet: ${abspath(local_file.tanka_config_main.filename)}
+ - spec.json: ${abspath(local_file.tanka_config_spec.filename)}
+ - make-certs.sh: ${abspath(local_file.make_certs.filename)}
+ - apply-certs.sh: ${abspath(local_file.apply_certs.filename)}
+ EOT
+}

deploy/infrastructure/terraform-commons-dss/templates/apply-certs.sh.tmp

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# This script builds and executes the uss qualifier.
+
+set -eo pipefail
+
+OS=$(uname)
+if [[ "$OS" == "Darwin" ]]; then
+ # OSX uses BSD readlink
+ BASEDIR="$(dirname "$0")"
+else
+ BASEDIR=$(readlink -e "$(dirname "$0")")
+fi
+cd "$BASEDIR/../.." || exit 1
+
+./apply-certs.sh ${cluster_context} ${namespace}

deploy/infrastructure/terraform-commons-dss/templates/get-credentials.sh.tmp

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# This script builds and executes the uss qualifier.
+
+set -eo pipefail
+
+OS=$(uname)
+if [[ "$OS" == "Darwin" ]]; then
+ # OSX uses BSD readlink
+ BASEDIR="$(dirname "$0")"
+else
+ BASEDIR=$(readlink -e "$(dirname "$0")")
+fi
+cd "$BASEDIR/../.." || exit 1
+
+${get_credentials_cmd}

deploy/infrastructure/terraform-commons-dss/templates/main.jsonnet.tmp

@@ -0,0 +1,43 @@
+local dss = import '../../deploy/dss.libsonnet';
+local metadataBase = import '../../deploy/metadata_base.libsonnet';
+
+// All VAR_* values below must be replaced with appropriate values; see
+// dss/build/README.md for more information.
+
+local metadata = metadataBase {
+ namespace: '${VAR_NAMESPACE}',
+ clusterName: '${VAR_CLUSTER_CONTEXT}',
+ enable_istio: false,
+ single_cluster: false,
+ enableScd: ${VAR_ENABLE_SCD}, // <-- This boolean value is VAR_ENABLE_SCD
+ cockroach+: {
+ hostnameSuffix: '${VAR_CRDB_HOSTNAME_SUFFIX}',
+ locality: '${VAR_CRDB_LOCALITY}',
+ nodeIPs: [${VAR_CRDB_NODE_IPS}],
+ shouldInit: ${VAR_SHOULD_INIT},
+ JoinExisting: [${VAR_CRDB_EXTERNAL_NODES}],
+ storageClass: '${VAR_STORAGE_CLASS}',
+ },
+ gateway+: {
+ ipName: '${VAR_INGRESS_NAME}',
+ image: '${VAR_DOCKER_IMAGE_NAME}',
+ hostname: '${VAR_APP_HOSTNAME}',
+ traceRequests: true,
+ },
+ backend+: {
+ image: '${VAR_DOCKER_IMAGE_NAME}',
+ pubKeys: ['${VAR_PUBLIC_KEY_PEM_PATH}'],
+ jwksEndpoint: '${VAR_JWKS_ENDPOINT}',
+ jwksKeyIds: ['${VAR_JWKS_KEY_ID}'],
+ },
+ schema_manager+: {
+ image: '${VAR_DOCKER_IMAGE_NAME}',
+ desired_rid_db_version: '${VAR_DESIRED_RID_DB_VERSION}',
+ desired_scd_db_version: '${VAR_DESIRED_SCD_DB_VERSION}',
+ },
+ prometheus+: {
+ storageClass: '${VAR_STORAGE_CLASS}',
+ },
+};
+
+dss.all(metadata)

deploy/infrastructure/terraform-commons-dss/templates/make-certs.sh.tmp

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# This script builds and executes the uss qualifier.
+
+set -eo pipefail
+
+OS=$(uname)
+if [[ "$OS" == "Darwin" ]]; then
+ # OSX uses BSD readlink
+ BASEDIR="$(dirname "$0")"
+else
+ BASEDIR=$(readlink -e "$(dirname "$0")")
+fi
+cd "$BASEDIR/../.." || exit 1
+
+python ./make-certs.py --cluster-context ${cluster_context} --namespace ${namespace} --node-address ${node_address}

deploy/infrastructure/terraform-commons-dss/templates/spec.json.tmp

@@ -0,0 +1,11 @@
+{
+ "apiVersion": "tanka.dev/v1alpha1",
+ "kind": "Environment",
+ "metadata": {
+ "name": "${cluster_context}"
+ },
+ "spec": {
+ "apiServer": "${api_server}",
+ "namespace": "${VAR_NAMESPACE}"
+ }
+}