[terraform] #874: terraform module for AWS (#942)

build/README.md

@@ -81,6 +81,11 @@ endpoint.
         like `gcr.io/your-project-id` (do not include the image name;
         it will be appended by the build script)
 
+    -   For Amazon Web Services, `DOCKER_URL` should be set similarly to as described
+        [here](https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-push-ecr-image.html),
+        like `${aws_account_id}.dkr.ecr.${region}.amazonaws.com/` (do not include the image name;
+        it will be appended by the build script)
+
 1. Ensure you are logged into your docker registry service.
 
     -   For Google Cloud,
@@ -89,6 +94,10 @@ endpoint.
         Ensure that
         [appropriate permissions are enabled](https://cloud.google.com/container-registry/docs/access-control).
 
+    -   For Amazon Web Services, create a private repository by following the instructions
+        [here](https://docs.aws.amazon.com/AmazonECR/latest/userguide/repository-create.html), then login
+        as described [here](https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-push-ecr-image.html).
+
 1. Use the [`build.sh` script](./build.sh) in this directory to build and push
    an image tagged with the current date and git commit hash.
 

build/apply-certs.sh

@@ -24,10 +24,13 @@ set -x
 CONTEXT="$1"
 DIR="$(pwd)"
 NAMESPACE="$2"
-CLIENTS_CERTS_DIR="$DIR/workspace/$CONTEXT/client_certs_dir"
-NODE_CERTS_DIR="$DIR/workspace/$CONTEXT/node_certs_dir"
-CA_KEY_DIR="$DIR/workspace/$CONTEXT/ca_key_dir"
-CA_CRT_DIR="$DIR/workspace/$CONTEXT/ca_certs_dir"
+
+# Replace characters breaking folder names
+WORKSPACE=$(echo "${CONTEXT}" | tr ':/' '_')
+CLIENTS_CERTS_DIR="$DIR/workspace/$WORKSPACE/client_certs_dir"
+NODE_CERTS_DIR="$DIR/workspace/$WORKSPACE/node_certs_dir"
+CA_KEY_DIR="$DIR/workspace/$WORKSPACE/ca_key_dir"
+CA_CRT_DIR="$DIR/workspace/$WORKSPACE/ca_certs_dir"
 JWT_PUBLIC_CERTS_DIR="$DIR/jwt-public-certs"
 UPLOAD_CA_KEY=true
 

build/deploy/base.libsonnet

@@ -231,4 +231,31 @@ local util = import 'util.libsonnet';
 
     assert std.length(self.containers) > 0 : 'must have at least one container',
   },
+
+  // Reusable cloud provider specific resources
+  AWSLoadBalancer(metadata, name, ipNames, subnet): $.Service(metadata, name) {
+    type:: 'LoadBalancer',
+    metadata+: {
+      annotations+: {
+        'service.beta.kubernetes.io/aws-load-balancer-type': 'external',
+        'service.beta.kubernetes.io/aws-load-balancer-nlb-target-type': 'ip',
+        'service.beta.kubernetes.io/aws-load-balancer-scheme': 'internet-facing',
+        'service.beta.kubernetes.io/aws-load-balancer-eip-allocations': std.join(',', ipNames),
+        'service.beta.kubernetes.io/aws-load-balancer-name': name,
+        'service.beta.kubernetes.io/aws-load-balancer-subnets': metadata.subnet,
+      },
+    },
+    spec+: {
+      loadBalancerClass: "service.k8s.aws/nlb",
+    },
+  },
+
+  AWSLoadBalancerWithManagedCert(metadata, name, ipNames, subnet, certARN): $.AWSLoadBalancer(metadata, name, ipNames, subnet) {
+    metadata+: {
+      annotations+: {
+          'service.beta.kubernetes.io/aws-load-balancer-ssl-ports': '443',
+          'service.beta.kubernetes.io/aws-load-balancer-ssl-cert': certARN,
+      },
+    },
+  }
 }

build/deploy/cockroachdb-auxiliary.libsonnet

@@ -1,7 +1,7 @@
 local base = import 'base.libsonnet';
 local volumes = import 'volumes.libsonnet';
 
-local cockroachLB(metadata, name, ip) = base.Service(metadata, name) {
+local googleCockroachLB(metadata, name, ip) = base.Service(metadata, name) {
   port:: metadata.cockroach.grpc_port,
   app:: 'cockroachdb',
   spec+: {
@@ -10,6 +10,15 @@ local cockroachLB(metadata, name, ip) = base.Service(metadata, name) {
   },
 };
 
+local awsCockroachLB(metadata, name, ip) = base.AWSLoadBalancer(metadata, name, [ip], metadata.subnet) {
+  port:: metadata.cockroach.grpc_port,
+  app:: 'cockroachdb',
+};
+
+local cockroachLB(metadata, name, ip) =
+    if metadata.cloud_provider == "google" then googleCockroachLB(metadata, name, ip)
+    else if metadata.cloud_provider == "aws" then awsCockroachLB(metadata, name, ip);
+
 {
   all(metadata): {
     CockroachInit: if metadata.cockroach.shouldInit then base.Job(metadata, 'init') {

build/deploy/core-service.libsonnet

@@ -1,28 +1,40 @@
 local base = import 'base.libsonnet';
 local volumes = import 'volumes.libsonnet';
 
-local ingress(metadata) = base.Ingress(metadata, 'https-ingress') {
-  metadata+: {
-    annotations: {
-      'kubernetes.io/ingress.global-static-ip-name': metadata.backend.ipName,
-      'kubernetes.io/ingress.allow-http': 'false',
+local awsLoadBalancer(metadata) = base.AWSLoadBalancerWithManagedCert(metadata, 'gateway', [metadata.backend.ipName], metadata.subnet, metadata.backend.certName) {
+  app:: 'core-service',
+  spec+: {
+    ports: [{
+      port: 443,
+      targetPort: metadata.backend.port,
+      protocol: "TCP",
+      name: "http",
+    }]
+  }
+};
+
+{
+  GoogleIngress(metadata): base.Ingress(metadata, 'https-ingress') {
+    metadata+: {
+      annotations: {
+        'kubernetes.io/ingress.global-static-ip-name': metadata.backend.ipName,
+        'kubernetes.io/ingress.allow-http': 'false',
+      },
     },
-  },
-  spec: {
-    defaultBackend: {
-      service: {
-        name: 'core-service',
-        port: {
-          number: metadata.backend.port,
+    spec: {
+      defaultBackend: {
+        service: {
+          name: 'core-service',
+          port: {
+            number: metadata.backend.port,
+          }
         }
-      }
+      },
     },
   },
-};
 
-{
-  ManagedCertIngress(metadata): {
-    ingress: ingress(metadata) {
+  GoogleManagedCertIngress(metadata): {
+    ingress: $.GoogleIngress(metadata) {
       metadata+: {
         annotations+: {
           'networking.gke.io/managed-certificates': 'https-certificate',
@@ -38,22 +50,31 @@ local ingress(metadata) = base.Ingress(metadata, 'https-ingress') {
     },
   },
 
-  PresharedCertIngress(metadata, certName): ingress(metadata) {
+  GooglePresharedCertIngress(metadata, certName): $.GoogleIngress(metadata) {
     metadata+: {
       annotations+: {
         'ingress.gcp.kubernetes.io/pre-shared-cert': certName,
       },
     },
   },
 
-  all(metadata): {
-    ingress: $.ManagedCertIngress(metadata),
-    service: base.Service(metadata, 'core-service') {
-      app:: 'core-service',
-      port:: metadata.backend.port,
-      type:: 'NodePort',
-      enable_monitoring:: false,
+  GoogleService(metadata): base.Service(metadata, 'core-service') {
+    app:: 'core-service',
+    port:: metadata.backend.port,
+    type:: 'NodePort',
+    enable_monitoring:: false,
+  },
+
+  CloudNetwork(metadata): {
+    google: if metadata.cloud_provider == "google" then {
+      ingress: $.GoogleManagedCertIngress(metadata),
+      service: $.GoogleService(metadata),
     },
+    aws_loadbalancer: if metadata.cloud_provider == "aws" then awsLoadBalancer(metadata)
+  },
+
+  all(metadata): {
+    network: $.CloudNetwork(metadata),
 
     deployment: base.Deployment(metadata, 'core-service') {
       apiVersion: 'apps/v1',

build/deploy/metadata_base.libsonnet

@@ -1,4 +1,5 @@
 {
+  cloud_provider: 'google', // Either google or aws
   namespace: error 'must supply namespace',
   clusterName: error 'must supply cluster name',
   enable_istio: false,
@@ -15,7 +16,7 @@
     grpc_port: 26257,
     http_port: 8080,
     image: 'cockroachdb/cockroach:v21.2.7',
-    nodeIPs: error 'must supply the per-node ip addresses as an array',
+    nodeIPs: error 'must supply the per-node ip addresses as an array', // For AWS, this array should contain the allocation id of the elastic ips.
     JoinExisting: [],
     storageClass: 'standard',
   },
@@ -24,7 +25,7 @@
     roleBinding: false,
   },
   backend: {
-    ipName: error 'must supply ip name',
+    ipName: error 'must supply ip name', // For AWS, use the elastic ip allocation id.
     port: 8080,
     image: error 'must specify image',
     prof_grpc_name: '',
@@ -33,6 +34,7 @@
     jwksKeyIds: [],
     hostname: error 'must specify hostname',
     dumpRequests: false,
+    certName: if $.cloud_provider == "aws" then error 'must specify certName for AWS cloud provider', # Only used by AWS
   },
   alert: {
     enable: false,
@@ -56,4 +58,5 @@
     custom_rules: [],  // An array of Prometheus recording rules, each of which is an object with "record" and "expr" properties.
     custom_args: [], // An array of strings to pass as commandline arguments to Prometheus.
   },
+  subnet: if $.cloud_provider == "aws" then error 'must specify subnet for AWS cloud provider', // For AWS, subnet of the elastic ips
 }

build/make-certs.py

@@ -25,7 +25,12 @@ def namespace(self):
 
     @property
     def directory(self):
-        return os.path.join('workspace', self._cluster_context)
+        # Replace characters breaking folder names
+        def remove_special_chars(s: str):
+            for c in [":", "/"]:
+                s = s.replace(c, "_")
+            return s
+        return os.path.join('workspace', remove_special_chars(self._cluster_context))
 
     @property
     def ca_certs_file(self):

deploy/README.md

@@ -0,0 +1,11 @@
+# DSS Deployment
+
+**Work in progress**
+
+This folder contains the increments toward the new deployment approach as described in [#874](https://github.com/interuss/dss/issues/874).
+
+The infrastructure folder contains the terraform modules to deploy the DSS to kubernetes clusters of various cloud providers:
+
+- Amazon Web Services: [terraform-aws-dss](./infrastructure/modules/terraform-aws-dss/README.md)
+- Google Cloud Engine: [terraform-google-dss](./infrastructure/modules/terraform-google-dss/README.md)
+

deploy/infrastructure/.gitignore

@@ -3,3 +3,4 @@
 terraform.tfstate
 terraform.tfstate.backup
 personal/
+*.pem

deploy/infrastructure/dependencies/terraform-aws-kubernetes/.gitignore

@@ -0,0 +1 @@
+test-app.yml