[ci-helm] Deploy the DSS using Helm

.github/workflows/dss-deploy.yml

@@ -33,7 +33,7 @@ jobs:
           role-to-assume: arn:aws:iam::301042233698:role/InterUSSGithubCI
           aws-region: us-east-1
           mask-aws-account-id: true
-          role-duration-seconds: 1800
+          role-duration-seconds: 5400
 
       - name: Caller Id
         run: |
@@ -44,5 +44,6 @@ jobs:
         working-directory: ./deploy/operations/
         env:
           COMPOSE_PROFILES: aws-1
+          AWS_REGION: us-east-1
         run: |
           docker compose up --exit-code-from ci-aws-1

build/apply-certs.sh

@@ -48,11 +48,13 @@ kubectl create secret generic cockroachdb.client.root --namespace default --from
 if [[ $NAMESPACE != "default" ]]; then
   kubectl create secret generic cockroachdb.client.root --namespace "$NAMESPACE" --from-file "$CLIENTS_CERTS_DIR"  --context "$CONTEXT"
 fi
-kubectl create secret generic cockroachdb.node --namespace "$NAMESPACE" --from-file "$NODE_CERTS_DIR"  --context "$CONTEXT"
+kubectl create secret generic cockroachdb.node --namespace "$NAMESPACE" --from-file "$NODE_CERTS_DIR" --context "$CONTEXT"
 # The ca key is not needed for any typical operations, but might be required to sign new certificates.
 $UPLOAD_CA_KEY && kubectl create secret generic cockroachdb.ca.key --namespace "$NAMESPACE" --from-file "$CA_KEY_DIR"  --context "$CONTEXT"
 # The ca.crt is kept in it's own secret to more easily manage cert rotation and
-# adding other operators' certificates.
+# adding other operators' certificates. Note that, for the purpose of the migration to helm and
+# to comply with cockroach db standard configuration, ca.crt has been kept inside cockroach.* secrets.
+# This secret is kept for backward compatibility.
 kubectl create secret generic cockroachdb.ca.crt --namespace "$NAMESPACE" --from-file "$CA_CRT_DIR"  --context "$CONTEXT"
 kubectl create secret generic dss.public.certs --namespace "$NAMESPACE" --from-file "$JWT_PUBLIC_CERTS_DIR"  --context "$CONTEXT"
 

build/make-certs.py

@@ -138,7 +138,16 @@ def main():
         '*.cockroachdb',
         '*.cockroachdb.%s' % cr.namespace,
         'cockroachdb.%s' % cr.namespace,
-        '*.cockroachdb.%s.svc.cluster.local' % cr.namespace
+        '*.cockroachdb.%s.svc.cluster.local' % cr.namespace,
+        # New helm generated address
+        # Individual nodes
+        '*.dss-cockroachdb',
+        '*.dss-cockroachdb.%s' % cr.namespace,
+        '*.dss-cockroachdb.%s.svc.cluster.local' % cr.namespace,
+        # Internal load balancer
+        'dss-cockroachdb-public',
+        'dss-cockroachdb-public.%s' % cr.namespace,
+        'dss-cockroachdb-public.%s.svc.cluster.local' % cr.namespace,
     ])
 
     subprocess.check_call([
@@ -147,9 +156,6 @@ def main():
         '--ca-key', cr.ca_key_file]
         + node_addresses)
 
-    os.remove(os.path.join(cr.node_certs_dir, 'ca.crt'))
-    os.remove(os.path.join(cr.client_certs_dir, 'ca.crt'))
-
     print('Created new node certificate in {}'.format(cr.node_certs_dir))
 
 

deploy/operations/Dockerfile

@@ -1,7 +1,9 @@
 FROM ubuntu:22.04
 
+ENV COCKROACH_VERSION 21.2.7
+
 RUN  apt-get update \
-&& apt-get install -y unzip curl gnupg lsb-release
+&& apt-get install -y unzip curl gnupg lsb-release apt-transport-https ca-certificates
 
 # Terraform CLI
 RUN curl -s https://apt.releases.hashicorp.com/gpg | gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg \
@@ -16,7 +18,27 @@ RUN  curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv
 && rm awscliv2.zip \
 && ./aws/install
 
+# Kubectl && Helm
+RUN curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | tee /usr/share/keyrings/helm.gpg > /dev/null \
+&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | tee /etc/apt/sources.list.d/helm-stable-debian.list \
+&& curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.28/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg \
+&& echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.28/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list \
+&& apt-get update \
+&& apt-get install -y kubectl helm
+
+# Cockroach
+RUN curl "https://binaries.cockroachdb.com/cockroach-v${COCKROACH_VERSION}.linux-amd64.tgz" -o "cockroach-v${COCKROACH_VERSION}.tgz" \
+&& tar -xvf "cockroach-v${COCKROACH_VERSION}.tgz" \
+&& cp -i cockroach-v${COCKROACH_VERSION}.*/cockroach /usr/local/bin/ \
+&& mkdir -p /usr/local/lib/cockroach \
+&& cp -i cockroach-v${COCKROACH_VERSION}.*/lib/libgeos.so /usr/local/lib/cockroach/ \
+&& cp -i cockroach-v${COCKROACH_VERSION}.*/lib/libgeos_c.so /usr/local/lib/cockroach/
+
+# TODO: Migrate scripts to python3 commands
+RUN ln -s /usr/bin/python3 /usr/bin/python & \
+    ln -s /usr/bin/pip3 /usr/bin/pip
+
 # Clean up apt
-RUN apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN apt-get clean && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
-RUN terraform --version
+RUN terraform --version

deploy/operations/ci/aws-1/README.md

@@ -1,6 +1,7 @@
 # AWS-1 CI deployment
 
-This module deploys a Kubernetes cluster to AWS.
+This module deploys a DSS to a Kubernetes cluster in AWS. It is primarily by our [CI](../../../../.github/workflows/dss-deploy.yml).
+See [test.sh](test.sh) for the complete list of actions.
 
 ## Terraform state
 
@@ -23,7 +24,7 @@ Call the kubernetes cluster using `kubectl`
 #### Add other roles
 
 Access to the cluster is managed using the config map `aws-auth`. 
-Its definition is managed by [`kubernetes_admin_access.tf`](./kubernetes_admin_access.tf).
+Its definition is managed in [`kubernetes_admin_access.tf`](./kubernetes_admin_access.tf).
 Currently only the user who bootstrapped the cluster and the ones assuming 
 the administrator role (see [`local_variables.tf`](./local_variables.tf)) have access.
 

deploy/operations/ci/aws-1/kubernetes_admin_access.tf

@@ -1,38 +1,41 @@
-# This module is expected to be applied by the Github CI user. By default, only the user has permission to
-# connect to the cluster. This file gathers resources to grant access to AWS administrators.
+// This module is expected to be applied by the Github CI user. By default, only the user who created the cluster
+// has permission to connect to the cluster. This file gathers resources to grant access to AWS administrators.
 
-resource "kubernetes_config_map_v1_data" "aws-auth" {
-  metadata {
-    name      = "aws-auth"
-    namespace = "kube-system"
-  }
+resource "local_file" "aws-auth-config-map" {
+  content = yamlencode({
+    apiVersion = "v1"
+    kind = "ConfigMap"
+    metadata = {
+      name      = "aws-auth"
+      namespace = "kube-system"
+    }
+    data = {
+      mapRoles = yamlencode([
+        {
+          groups   = [
+            "system:bootstrappers",
+            "system:nodes"
+          ]
+          rolearn  = module.terraform-aws-dss.iam_role_node_group_arn
+          username = "system:node:{{EC2PrivateDNSName}}"
+        },
+        {
+          groups   = [
+            "system:masters"
+          ]
+          rolearn  = var.aws_iam_administrator_role
+          username = "interuss-aws-administrator"
+        },
+        {
+          groups   = [
+            "system:masters"
+          ]
+          rolearn  = var.aws_iam_ci_role
+          username = "interuss-ci"
+        }
+      ])
+    }
+  })
 
-  force = true # EKS provisions this file by default.
-
-  data = {
-    mapRoles = yamlencode([
-      {
-        groups   = [
-          "system:bootstrappers",
-          "system:nodes"
-        ]
-        rolearn  = module.terraform-aws-kubernetes.iam_role_node_group_arn
-        username = "system:node:{{EC2PrivateDNSName}}"
-      },
-      {
-        groups   = [
-          "system:masters"
-        ]
-        rolearn  = var.aws_iam_administrator_role
-        username = "interuss-aws-administrator"
-      },
-      {
-        groups   = [
-          "system:masters"
-        ]
-        rolearn  = var.aws_iam_ci_role
-        username = "interuss-ci"
-      }
-    ])
-  }
+  filename   = "${module.terraform-aws-dss.workspace_location}/aws_auth_config_map.yml"
 }

deploy/operations/ci/aws-1/main.tf

@@ -6,39 +6,22 @@ terraform {
   }
 }
 
-module "terraform-aws-kubernetes" {
-  # See variables.tf for variables description.
-  cluster_name         = var.cluster_name
-  aws_region           = var.aws_region
-  app_hostname         = var.app_hostname
-  crdb_hostname_suffix = var.crdb_hostname_suffix
-  aws_instance_type    = var.aws_instance_type
-  aws_route53_zone_id  = var.aws_route53_zone_id
-  aws_iam_permissions_boundary = var.aws_iam_permissions_boundary
-  node_count           = var.node_count
+module "terraform-aws-dss" {
+  source = "../../../infrastructure/modules/terraform-aws-dss"
 
-  source = "../../../infrastructure/dependencies/terraform-aws-kubernetes"
+  app_hostname                 = var.app_hostname
+  authorization                = var.authorization
+  aws_iam_permissions_boundary = var.aws_iam_permissions_boundary
+  aws_instance_type            = var.aws_instance_type
+  aws_kubernetes_storage_class = var.aws_kubernetes_storage_class
+  aws_region                   = var.aws_region
+  aws_route53_zone_id          = var.aws_route53_zone_id
+  cluster_name                 = var.cluster_name
+  crdb_hostname_suffix         = var.crdb_hostname_suffix
+  crdb_locality                = var.crdb_locality
+  image                        = var.image
+  node_count                   = 3
+  should_init                  = true
+  enable_scd                   = true
 }
 
-module "terraform-commons-dss" {
-  # See variables.tf for variables description.
-  image                          = var.image
-  image_pull_secret              = var.image_pull_secret
-  kubernetes_namespace           = var.kubernetes_namespace
-  kubernetes_storage_class       = var.aws_kubernetes_storage_class
-  app_hostname                   = var.app_hostname
-  crdb_hostname_suffix           = var.crdb_hostname_suffix
-  should_init                    = var.should_init
-  authorization                  = var.authorization
-  crdb_locality                  = var.crdb_locality
-  crdb_internal_nodes            = module.terraform-aws-kubernetes.crdb_nodes
-  ip_gateway                     = module.terraform-aws-kubernetes.ip_gateway
-  kubernetes_api_endpoint        = module.terraform-aws-kubernetes.kubernetes_api_endpoint
-  kubernetes_cloud_provider_name = module.terraform-aws-kubernetes.kubernetes_cloud_provider_name
-  kubernetes_context_name        = module.terraform-aws-kubernetes.kubernetes_context_name
-  kubernetes_get_credentials_cmd = module.terraform-aws-kubernetes.kubernetes_get_credentials_cmd
-  workload_subnet                = module.terraform-aws-kubernetes.workload_subnet
-  gateway_cert_name              = module.terraform-aws-kubernetes.app_hostname_cert_arn
-
-  source = "../../../infrastructure/dependencies/terraform-commons-dss"
-}

deploy/operations/ci/aws-1/output.tf

@@ -1,4 +1,11 @@
 output "generated_files_location" {
-  value = module.terraform-commons-dss.generated_files_location
+  value = module.terraform-aws-dss.generated_files_location
 }
 
+output "workspace_location" {
+  value = module.terraform-aws-dss.workspace_location
+}
+
+output "cluster_context" {
+  value = module.terraform-aws-dss.cluster_context
+}

deploy/operations/ci/aws-1/providers.tf

@@ -1,19 +1,3 @@
 provider "aws" {
   region = "us-east-1"
 }
-
-data "aws_eks_cluster_auth" "kubernetes_cluster" {
-  name = var.cluster_name
-  depends_on = [module.terraform-aws-kubernetes]
-}
-
-data "aws_eks_cluster" "kubernetes_cluster" {
-  name = var.cluster_name
-  depends_on = [module.terraform-aws-kubernetes]
-}
-
-provider kubernetes {
-  host                   = data.aws_eks_cluster.kubernetes_cluster.endpoint
-  cluster_ca_certificate = base64decode(data.aws_eks_cluster.kubernetes_cluster.certificate_authority[0].data)
-  token                  = data.aws_eks_cluster_auth.kubernetes_cluster.token
-}

deploy/operations/ci/aws-1/test.sh

@@ -12,15 +12,50 @@ else
 fi
 cd "${BASEDIR}" || exit 1
 
-clean () {
-  echo "Cleaning infrastructure"
-  terraform destroy -auto-approve
-}
-
+# Initialize terraform
 terraform init
-clean
+# TODO: Fail if env is not clean
+
+# Deploy the Kubernetes cluster
 terraform apply -auto-approve
-# TODO: Deploy the DSS
+KUBE_CONTEXT="$(terraform output -raw cluster_context)"
+WORKSPACE_LOCATION="$(terraform output -raw workspace_location)"
+
+# Login into the Kubernetes Cluster
+cd "${WORKSPACE_LOCATION}"
+./get-credentials.sh
+aws sts get-caller-identity
+
+# Allow access to the cluster to AWS admins
+kubectl apply -f "aws_auth_config_map.yml"
+
+# Generate cockroachdb certificates
+./make-certs.sh
+./apply-certs.sh
+
+# Install the DSS using the helm chart
+cd "$BASEDIR/../../../services/helm-charts/dss"
+RELEASE_NAME="dss"
+helm dep update --kube-context="$KUBE_CONTEXT"
+helm upgrade --install --kube-context="$KUBE_CONTEXT" -f "${WORKSPACE_LOCATION}/helm_values.yml" "$RELEASE_NAME" .
+
 # TODO: Test the deployment of the DSS
-clean
+
+if [ -n "$DO_NOT_DESTROY" ]; then
+  echo "Destroy disabled. Exit."
+  exit 0
+fi
+
+# Cleanup
+# Delete workloads
+helm uninstall --kube-context="$KUBE_CONTEXT" "$RELEASE_NAME"
+
+# Delete PVC to delete persistant volumes
+kubectl delete pvc --all=true
+# TODO: Check completness
+
+# Delete cluster
+cd "$BASEDIR"
+terraform destroy -auto-approve
+
 

deploy/operations/docker-compose.yaml

@@ -4,12 +4,13 @@ services:
     image: interuss-deploy
     profiles: ["aws-1"]
     command: operations/ci/aws-1/test.sh
-    working_dir: /opt/dss
+    working_dir: /opt/dss/deploy
     environment:
       - AWS_ACCESS_KEY_ID
       - AWS_SECRET_ACCESS_KEY
       - AWS_SESSION_TOKEN
+      - AWS_REGION
     volumes:
       - type: bind
-        source: ../
+        source: ../../
         target: /opt/dss/