diff --git a/.github/workflows/dss-deploy.yml b/.github/workflows/dss-deploy.yml index 787d28fdc..70b4ed3f2 100644 --- a/.github/workflows/dss-deploy.yml +++ b/.github/workflows/dss-deploy.yml @@ -33,7 +33,7 @@ jobs: role-to-assume: arn:aws:iam::301042233698:role/InterUSSGithubCI aws-region: us-east-1 mask-aws-account-id: true - role-duration-seconds: 1800 + role-duration-seconds: 5400 - name: Caller Id run: | @@ -44,5 +44,6 @@ jobs: working-directory: ./deploy/operations/ env: COMPOSE_PROFILES: aws-1 + AWS_REGION: us-east-1 run: | docker compose up --exit-code-from ci-aws-1 diff --git a/build/apply-certs.sh b/build/apply-certs.sh index b2074ab51..32403eecc 100755 --- a/build/apply-certs.sh +++ b/build/apply-certs.sh @@ -48,11 +48,13 @@ kubectl create secret generic cockroachdb.client.root --namespace default --from if [[ $NAMESPACE != "default" ]]; then kubectl create secret generic cockroachdb.client.root --namespace "$NAMESPACE" --from-file "$CLIENTS_CERTS_DIR" --context "$CONTEXT" fi -kubectl create secret generic cockroachdb.node --namespace "$NAMESPACE" --from-file "$NODE_CERTS_DIR" --context "$CONTEXT" +kubectl create secret generic cockroachdb.node --namespace "$NAMESPACE" --from-file "$NODE_CERTS_DIR" --context "$CONTEXT" # The ca key is not needed for any typical operations, but might be required to sign new certificates. $UPLOAD_CA_KEY && kubectl create secret generic cockroachdb.ca.key --namespace "$NAMESPACE" --from-file "$CA_KEY_DIR" --context "$CONTEXT" # The ca.crt is kept in it's own secret to more easily manage cert rotation and -# adding other operators' certificates. +# adding other operators' certificates. Note that, for the purpose of the migration to helm and +# to comply with cockroach db standard configuration, ca.crt has been kept inside cockroach.* secrets. +# This secret is kept for backward compatibility. kubectl create secret generic cockroachdb.ca.crt --namespace "$NAMESPACE" --from-file "$CA_CRT_DIR" --context "$CONTEXT" kubectl create secret generic dss.public.certs --namespace "$NAMESPACE" --from-file "$JWT_PUBLIC_CERTS_DIR" --context "$CONTEXT" diff --git a/build/make-certs.py b/build/make-certs.py index 0d25f4d46..2ea8f1b0e 100755 --- a/build/make-certs.py +++ b/build/make-certs.py @@ -138,7 +138,16 @@ def main(): '*.cockroachdb', '*.cockroachdb.%s' % cr.namespace, 'cockroachdb.%s' % cr.namespace, - '*.cockroachdb.%s.svc.cluster.local' % cr.namespace + '*.cockroachdb.%s.svc.cluster.local' % cr.namespace, + # New helm generated address + # Individual nodes + '*.dss-cockroachdb', + '*.dss-cockroachdb.%s' % cr.namespace, + '*.dss-cockroachdb.%s.svc.cluster.local' % cr.namespace, + # Internal load balancer + 'dss-cockroachdb-public', + 'dss-cockroachdb-public.%s' % cr.namespace, + 'dss-cockroachdb-public.%s.svc.cluster.local' % cr.namespace, ]) subprocess.check_call([ @@ -147,9 +156,6 @@ def main(): '--ca-key', cr.ca_key_file] + node_addresses) - os.remove(os.path.join(cr.node_certs_dir, 'ca.crt')) - os.remove(os.path.join(cr.client_certs_dir, 'ca.crt')) - print('Created new node certificate in {}'.format(cr.node_certs_dir)) diff --git a/deploy/operations/Dockerfile b/deploy/operations/Dockerfile index 2ca5ff552..3d7eb0e2f 100644 --- a/deploy/operations/Dockerfile +++ b/deploy/operations/Dockerfile @@ -1,7 +1,9 @@ FROM ubuntu:22.04 +ENV COCKROACH_VERSION 21.2.7 + RUN apt-get update \ -&& apt-get install -y unzip curl gnupg lsb-release +&& apt-get install -y unzip curl gnupg lsb-release apt-transport-https ca-certificates # Terraform CLI RUN curl -s https://apt.releases.hashicorp.com/gpg | gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg \ @@ -16,7 +18,27 @@ RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv && rm awscliv2.zip \ && ./aws/install +# Kubectl && Helm +RUN curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | tee /usr/share/keyrings/helm.gpg > /dev/null \ +&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | tee /etc/apt/sources.list.d/helm-stable-debian.list \ +&& curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.28/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg \ +&& echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.28/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list \ +&& apt-get update \ +&& apt-get install -y kubectl helm + +# Cockroach +RUN curl "https://binaries.cockroachdb.com/cockroach-v${COCKROACH_VERSION}.linux-amd64.tgz" -o "cockroach-v${COCKROACH_VERSION}.tgz" \ +&& tar -xvf "cockroach-v${COCKROACH_VERSION}.tgz" \ +&& cp -i cockroach-v${COCKROACH_VERSION}.*/cockroach /usr/local/bin/ \ +&& mkdir -p /usr/local/lib/cockroach \ +&& cp -i cockroach-v${COCKROACH_VERSION}.*/lib/libgeos.so /usr/local/lib/cockroach/ \ +&& cp -i cockroach-v${COCKROACH_VERSION}.*/lib/libgeos_c.so /usr/local/lib/cockroach/ + +# TODO: Migrate scripts to python3 commands +RUN ln -s /usr/bin/python3 /usr/bin/python & \ + ln -s /usr/bin/pip3 /usr/bin/pip + # Clean up apt -RUN apt-get clean && rm -rf /var/lib/apt/lists/* +RUN apt-get clean && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* -RUN terraform --version \ No newline at end of file +RUN terraform --version diff --git a/deploy/operations/ci/aws-1/kubernetes_admin_access.tf b/deploy/operations/ci/aws-1/kubernetes_admin_access.tf index 5e808c2f7..8a6c10899 100644 --- a/deploy/operations/ci/aws-1/kubernetes_admin_access.tf +++ b/deploy/operations/ci/aws-1/kubernetes_admin_access.tf @@ -1,38 +1,41 @@ -# This module is expected to be applied by the Github CI user. By default, only the user has permission to -# connect to the cluster. This file gathers resources to grant access to AWS administrators. +// This module is expected to be applied by the Github CI user. By default, only the user who created the cluster +// has permission to connect to the cluster. This file gathers resources to grant access to AWS administrators. -resource "kubernetes_config_map_v1_data" "aws-auth" { - metadata { - name = "aws-auth" - namespace = "kube-system" - } +resource "local_file" "aws-auth-config-map" { + content = yamlencode({ + apiVersion = "v1" + kind = "ConfigMap" + metadata = { + name = "aws-auth" + namespace = "kube-system" + } + data = { + mapRoles = yamlencode([ + { + groups = [ + "system:bootstrappers", + "system:nodes" + ] + rolearn = module.terraform-aws-dss.iam_role_node_group_arn + username = "system:node:{{EC2PrivateDNSName}}" + }, + { + groups = [ + "system:masters" + ] + rolearn = var.aws_iam_administrator_role + username = "interuss-aws-administrator" + }, + { + groups = [ + "system:masters" + ] + rolearn = var.aws_iam_ci_role + username = "interuss-ci" + } + ]) + } + }) - force = true # EKS provisions this file by default. - - data = { - mapRoles = yamlencode([ - { - groups = [ - "system:bootstrappers", - "system:nodes" - ] - rolearn = module.terraform-aws-kubernetes.iam_role_node_group_arn - username = "system:node:{{EC2PrivateDNSName}}" - }, - { - groups = [ - "system:masters" - ] - rolearn = var.aws_iam_administrator_role - username = "interuss-aws-administrator" - }, - { - groups = [ - "system:masters" - ] - rolearn = var.aws_iam_ci_role - username = "interuss-ci" - } - ]) - } + filename = "${module.terraform-aws-dss.workspace_location}/aws_auth_config_map.yml" } diff --git a/deploy/operations/ci/aws-1/main.tf b/deploy/operations/ci/aws-1/main.tf index a9ee3093e..3b0fee971 100644 --- a/deploy/operations/ci/aws-1/main.tf +++ b/deploy/operations/ci/aws-1/main.tf @@ -6,39 +6,22 @@ terraform { } } -module "terraform-aws-kubernetes" { - # See variables.tf for variables description. - cluster_name = var.cluster_name - aws_region = var.aws_region - app_hostname = var.app_hostname - crdb_hostname_suffix = var.crdb_hostname_suffix - aws_instance_type = var.aws_instance_type - aws_route53_zone_id = var.aws_route53_zone_id - aws_iam_permissions_boundary = var.aws_iam_permissions_boundary - node_count = var.node_count +module "terraform-aws-dss" { + source = "../../../infrastructure/modules/terraform-aws-dss" - source = "../../../infrastructure/dependencies/terraform-aws-kubernetes" + app_hostname = var.app_hostname + authorization = var.authorization + aws_iam_permissions_boundary = var.aws_iam_permissions_boundary + aws_instance_type = var.aws_instance_type + aws_kubernetes_storage_class = var.aws_kubernetes_storage_class + aws_region = var.aws_region + aws_route53_zone_id = var.aws_route53_zone_id + cluster_name = var.cluster_name + crdb_hostname_suffix = var.crdb_hostname_suffix + crdb_locality = var.crdb_locality + image = var.image + node_count = 3 + should_init = true + enable_scd = true } -module "terraform-commons-dss" { - # See variables.tf for variables description. - image = var.image - image_pull_secret = var.image_pull_secret - kubernetes_namespace = var.kubernetes_namespace - kubernetes_storage_class = var.aws_kubernetes_storage_class - app_hostname = var.app_hostname - crdb_hostname_suffix = var.crdb_hostname_suffix - should_init = var.should_init - authorization = var.authorization - crdb_locality = var.crdb_locality - crdb_internal_nodes = module.terraform-aws-kubernetes.crdb_nodes - ip_gateway = module.terraform-aws-kubernetes.ip_gateway - kubernetes_api_endpoint = module.terraform-aws-kubernetes.kubernetes_api_endpoint - kubernetes_cloud_provider_name = module.terraform-aws-kubernetes.kubernetes_cloud_provider_name - kubernetes_context_name = module.terraform-aws-kubernetes.kubernetes_context_name - kubernetes_get_credentials_cmd = module.terraform-aws-kubernetes.kubernetes_get_credentials_cmd - workload_subnet = module.terraform-aws-kubernetes.workload_subnet - gateway_cert_name = module.terraform-aws-kubernetes.app_hostname_cert_arn - - source = "../../../infrastructure/dependencies/terraform-commons-dss" -} diff --git a/deploy/operations/ci/aws-1/output.tf b/deploy/operations/ci/aws-1/output.tf index ade9609d1..84de2328a 100644 --- a/deploy/operations/ci/aws-1/output.tf +++ b/deploy/operations/ci/aws-1/output.tf @@ -1,4 +1,11 @@ output "generated_files_location" { - value = module.terraform-commons-dss.generated_files_location + value = module.terraform-aws-dss.generated_files_location } +output "workspace_location" { + value = module.terraform-aws-dss.workspace_location +} + +output "cluster_context" { + value = module.terraform-aws-dss.cluster_context +} diff --git a/deploy/operations/ci/aws-1/providers.tf b/deploy/operations/ci/aws-1/providers.tf index 8daa12fe7..c1259402c 100644 --- a/deploy/operations/ci/aws-1/providers.tf +++ b/deploy/operations/ci/aws-1/providers.tf @@ -1,19 +1,3 @@ provider "aws" { region = "us-east-1" } - -data "aws_eks_cluster_auth" "kubernetes_cluster" { - name = var.cluster_name - depends_on = [module.terraform-aws-kubernetes] -} - -data "aws_eks_cluster" "kubernetes_cluster" { - name = var.cluster_name - depends_on = [module.terraform-aws-kubernetes] -} - -provider kubernetes { - host = data.aws_eks_cluster.kubernetes_cluster.endpoint - cluster_ca_certificate = base64decode(data.aws_eks_cluster.kubernetes_cluster.certificate_authority[0].data) - token = data.aws_eks_cluster_auth.kubernetes_cluster.token -} diff --git a/deploy/operations/ci/aws-1/test.sh b/deploy/operations/ci/aws-1/test.sh index 7caeafa93..875bac878 100755 --- a/deploy/operations/ci/aws-1/test.sh +++ b/deploy/operations/ci/aws-1/test.sh @@ -12,15 +12,46 @@ else fi cd "${BASEDIR}" || exit 1 -clean () { - echo "Cleaning infrastructure" - terraform destroy -auto-approve -} - terraform init -clean +# TODO: Fail if env is not clean + +## Deploy the Kubernetes cluster terraform apply -auto-approve -# TODO: Deploy the DSS +KUBE_CONTEXT="$(terraform output -raw cluster_context)" +WORKSPACE_LOCATION="$(terraform output -raw workspace_location)" + +cd "${WORKSPACE_LOCATION}" +./get-credentials.sh +aws sts get-caller-identity + +# Allow access to the cluster to AWS admins +kubectl apply -f "aws_auth_config_map.yml" + +## Generate cockroachdb certificates +./make-certs.sh +./apply-certs.sh + +cd "$BASEDIR/../../../services/helm-charts/dss" +RELEASE_NAME="dss" +helm dep update --kube-context="$KUBE_CONTEXT" +helm upgrade --install --kube-context="$KUBE_CONTEXT" -f "${WORKSPACE_LOCATION}/helm_values.yml" "$RELEASE_NAME" . + # TODO: Test the deployment of the DSS -clean + +if [ -n "$DO_NOT_DESTROY" ]; then + "Destroy disabled. Exit." + exit 0 +fi + +# Cleanup +# Delete workloads +helm uninstall --kube-context="$KUBE_CONTEXT" "$RELEASE_NAME" + +# Delete PVC to delete persistant volumes +kubectl delete pvc --all=true + +# Delete cluster +cd "$BASEDIR" +terraform destroy -auto-approve + diff --git a/deploy/operations/docker-compose.yaml b/deploy/operations/docker-compose.yaml index 4618089db..49a96b6f3 100644 --- a/deploy/operations/docker-compose.yaml +++ b/deploy/operations/docker-compose.yaml @@ -4,12 +4,13 @@ services: image: interuss-deploy profiles: ["aws-1"] command: operations/ci/aws-1/test.sh - working_dir: /opt/dss + working_dir: /opt/dss/deploy environment: - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY - AWS_SESSION_TOKEN + - AWS_REGION volumes: - type: bind - source: ../ + source: ../../ target: /opt/dss/