func: move infra provisionining to a module and remove providers

.gitignore

@@ -9,6 +9,7 @@ Thumbs.db
 .idea
 .fleet
 
+
 # Folders
 _obj
 _test

e2e/terraform/Makefile

@@ -7,7 +7,7 @@ CONSUL_LICENSE_PATH ?=
 custom.tfvars:
 	echo 'nomad_local_binary = "$(PKG_PATH)"' > custom.tfvars
 	echo 'volumes = false' >> custom.tfvars
-	echo 'client_count_ubuntu_jammy_amd64 = 3' >> custom.tfvars
+	echo 'client_count_linux = 3' >> custom.tfvars
 	echo 'client_count_windows_2016_amd64 = 0' >> custom.tfvars
 	echo 'consul_license = "$(shell cat $(CONSUL_LICENSE_PATH))"' >> custom.tfvars
 	echo 'nomad_license = "$(shell cat $(NOMAD_LICENSE_PATH))"' >> custom.tfvars

e2e/terraform/README.md

@@ -51,7 +51,7 @@ Linux clients or Windows clients.
 region                           = "us-east-1"
 instance_type                    = "t2.medium"
 server_count                     = "3"
-client_count_ubuntu_jammy_amd64  = "4"
+client_count_linux               = "4"
 client_count_windows_2016_amd64  = "1"
 ```
 

e2e/terraform/main.tf

@@ -1,34 +1,12 @@
-# Copyright (c) HashiCorp, Inc.
-# SPDX-License-Identifier: BUSL-1.1
-
 provider "aws" {
   region = var.region
 }
 
-data "aws_caller_identity" "current" {
-}
-
-resource "random_pet" "e2e" {
-}
-
-resource "random_password" "windows_admin_password" {
-  length           = 20
-  special          = true
-  override_special = "_%@"
-}
+module "provision-infra" {
+  source = "./provision-infra"
 
-locals {
-  random_name = "${var.name}-${random_pet.e2e.id}"
-}
-
-# Generates keys to use for provisioning and access
-module "keys" {
-  name    = local.random_name
-  path    = "${path.root}/keys"
-  source  = "mitchellh/dynamic-keys/aws"
-  version = "v2.0.0"
-}
-
-data "aws_kms_alias" "e2e" {
-  name = "alias/${var.aws_kms_alias}"
-}
+  server_count = var.client_count_linux
+  client_count_linux =  var.client_count_linux
+  client_count_windows_2016_amd64 = var.client_count_windows_2016_amd64
+  nomad_local_binary = var.nomad_local_binary
+}

e2e/terraform/outputs.tf

@@ -2,43 +2,19 @@
 # SPDX-License-Identifier: BUSL-1.1
 
 output "servers" {
-  value = aws_instance.server.*.public_ip
+  value = module.provision-infra.servers
 }
 
 output "linux_clients" {
-  value = aws_instance.client_ubuntu_jammy_amd64.*.public_ip
+  value = module.provision-infra.linux_clients
 }
 
 output "windows_clients" {
-  value = aws_instance.client_windows_2016_amd64.*.public_ip
+ value = module.provision-infra.windows_clients
 }
 
 output "message" {
-  value = <<EOM
-Your cluster has been provisioned! To prepare your environment, run:
-
-   $(terraform output --raw environment)
-
-Then you can run tests from the e2e directory with:
-
-   go test -v .
-
-ssh into servers with:
-
-%{for ip in aws_instance.server.*.public_ip~}
-   ssh -i keys/${local.random_name}.pem ubuntu@${ip}
-%{endfor~}
-
-ssh into clients with:
-
-%{for ip in aws_instance.client_ubuntu_jammy_amd64.*.public_ip~}
-    ssh -i keys/${local.random_name}.pem ubuntu@${ip}
-%{endfor~}
-%{for ip in aws_instance.client_windows_2016_amd64.*.public_ip~}
-    ssh -i keys/${local.random_name}.pem Administrator@${ip}
-%{endfor~}
-
-EOM
+  value = module.provision-infra.message
 }
 
 # Note: Consul and Vault environment needs to be set in test
@@ -47,15 +23,5 @@ EOM
 output "environment" {
   description = "get connection config by running: $(terraform output environment)"
   sensitive   = true
-  value       = <<EOM
-export NOMAD_ADDR=https://${aws_instance.server[0].public_ip}:4646
-export NOMAD_CACERT=${abspath(path.module)}/keys/tls_ca.crt
-export NOMAD_CLIENT_CERT=${abspath(path.module)}/keys/tls_api_client.crt
-export NOMAD_CLIENT_KEY=${abspath(path.module)}/keys/tls_api_client.key
-export NOMAD_TOKEN=${data.local_sensitive_file.nomad_token.content}
-export NOMAD_E2E=1
-export CONSUL_HTTP_ADDR=https://${aws_instance.consul_server.public_ip}:8501
-export CONSUL_HTTP_TOKEN=${local_sensitive_file.consul_initial_management_token.content}
-export CONSUL_CACERT=${abspath(path.module)}/keys/tls_ca.crt
-EOM
+  value       = module.provision-infra.environment
 }

e2e/terraform/compute.tf → e2e/terraform/provision-infra/compute.tf

@@ -3,6 +3,7 @@
 
 locals {
   ami_prefix = "nomad-e2e-v3"
+  ubuntu_instance_name = "ubuntu-jammy-${var.instance_architecture}"
 }
 
 resource "aws_instance" "server" {
@@ -22,18 +23,18 @@ resource "aws_instance" "server" {
   }
 }
 
-resource "aws_instance" "client_ubuntu_jammy_amd64" {
-  ami                    = data.aws_ami.ubuntu_jammy_amd64.image_id
+resource "aws_instance" "client_ubuntu_jammy" {
+  ami                    = data.aws_ami.ubuntu_jammy.image_id
   instance_type          = var.instance_type
   key_name               = module.keys.key_name
   vpc_security_group_ids = [aws_security_group.clients.id] # see also the secondary ENI
-  count                  = var.client_count_ubuntu_jammy_amd64
+  count                  = var.client_count_linux
   iam_instance_profile   = data.aws_iam_instance_profile.nomad_e2e_cluster.name
   availability_zone      = var.availability_zone
 
   # Instance tags
   tags = {
-    Name           = "${local.random_name}-client-ubuntu-jammy-amd64-${count.index}"
+    Name           = "${local.random_name}-client-ubuntu-jammy-${count.index}"
     ConsulAutoJoin = "auto-join-${local.random_name}"
     User           = data.aws_caller_identity.current.arn
   }
@@ -100,10 +101,30 @@ data "aws_ami" "ubuntu_jammy_amd64" {
     values = ["Ubuntu"]
   }
 
-  filter {
+/*   filter {
     name   = "tag:BuilderSha"
     values = [data.external.packer_sha.result["sha"]]
+  } */
+}
+
+data "aws_ami" "ubuntu_jammy" {
+  most_recent = true
+  owners      = ["self"]
+
+  filter {
+    name   = "name"
+    values = ["${local.ami_prefix}-${local.ubuntu_instance_name}-*"]
   }
+
+  filter {
+    name   = "tag:OS"
+    values = ["Ubuntu"]
+  }
+
+/*   filter {
+    name   = "tag:BuilderSha"
+    values = [data.external.packer_sha.result["sha"]]
+  } */
 }
 
 data "aws_ami" "windows_2016_amd64" {

e2e/terraform/consul-clients.tf → e2e/terraform/provision-infra/consul-clients.tf

@@ -35,12 +35,12 @@ resource "tls_locally_signed_cert" "consul_agents" {
 
 resource "local_sensitive_file" "consul_agents_key" {
   content  = tls_private_key.consul_agents.private_key_pem
-  filename = "uploads/shared/consul.d/agent_cert.key.pem"
+  filename = "${path.module}/provision-nomad/uploads/shared/consul.d/agent_cert.key.pem"
 }
 
 resource "local_sensitive_file" "consul_agents_cert" {
   content  = tls_locally_signed_cert.consul_agents.cert_pem
-  filename = "uploads/shared/consul.d/agent_cert.pem"
+  filename = "${path.module}/provision-nomad/uploads/shared/consul.d/agent_cert.pem"
 }
 
 # Consul tokens for the Consul agents
@@ -52,7 +52,7 @@ resource "local_sensitive_file" "consul_agent_config_file" {
     token          = "${random_uuid.consul_agent_token.result}"
     autojoin_value = "auto-join-${local.random_name}"
   })
-  filename        = "uploads/shared/consul.d/clients.hcl"
+  filename        = "${path.module}/provision-nomad/uploads/shared/consul.d/clients.hcl"
   file_permission = "0600"
 }
 
@@ -66,7 +66,7 @@ resource "local_sensitive_file" "nomad_client_config_for_consul" {
     client_service_name = "client-${local.random_name}"
     server_service_name = "server-${local.random_name}"
   })
-  filename        = "uploads/shared/nomad.d/client-consul.hcl"
+  filename        = "${path.module}/provision-nomad/uploads/shared/nomad.d/client-consul.hcl"
   file_permission = "0600"
 }
 
@@ -76,6 +76,6 @@ resource "local_sensitive_file" "nomad_server_config_for_consul" {
     client_service_name = "client-${local.random_name}"
     server_service_name = "server-${local.random_name}"
   })
-  filename        = "uploads/shared/nomad.d/server-consul.hcl"
+  filename        = "${path.module}/provision-nomad/uploads/shared/nomad.d/server-consul.hcl"
   file_permission = "0600"
 }

e2e/terraform/consul-servers.tf → e2e/terraform/provision-infra/consul-servers.tf

@@ -21,7 +21,7 @@ resource "local_sensitive_file" "consul_server_config_file" {
     nomad_token      = "${random_uuid.consul_token_for_nomad.result}"
     autojoin_value   = "auto-join-${local.random_name}"
   })
-  filename        = "uploads/shared/consul.d/servers.hcl"
+  filename        = "${path.module}/provision-nomad/uploads/shared/consul.d/servers.hcl"
   file_permission = "0600"
 }
 
@@ -59,20 +59,20 @@ resource "tls_locally_signed_cert" "consul_server" {
 
 resource "local_sensitive_file" "consul_server_key" {
   content  = tls_private_key.consul_server.private_key_pem
-  filename = "uploads/shared/consul.d/server_cert.key.pem"
+  filename = "${path.module}/provision-nomad/uploads/shared/consul.d/server_cert.key.pem"
 }
 
 resource "local_sensitive_file" "consul_server_cert" {
   content  = tls_locally_signed_cert.consul_server.cert_pem
-  filename = "uploads/shared/consul.d/server_cert.pem"
+  filename = "${path.module}/provision-nomad/uploads/shared/consul.d/server_cert.pem"
 }
 
 # if consul_license is unset, it'll be a harmless empty license file
 resource "local_sensitive_file" "consul_environment" {
   content = templatefile("${path.module}/provision-nomad/etc/consul.d/.environment", {
     license = var.consul_license
   })
-  filename        = "uploads/shared/consul.d/.environment"
+  filename        = "${path.module}/provision-nomad/uploads/shared/consul.d/.environment"
   file_permission = "0600"
 }
 
@@ -97,23 +97,23 @@ resource "null_resource" "upload_consul_server_configs" {
   }
 
   provisioner "file" {
-    source      = "keys/tls_ca.crt"
+    source      = "${path.root}/keys/tls_ca.crt"
     destination = "/tmp/consul_ca.pem"
   }
   provisioner "file" {
-    source      = "uploads/shared/consul.d/.environment"
+    source      = "${path.module}/provision-nomad/uploads/shared/consul.d/.environment"
     destination = "/tmp/.consul_environment"
   }
   provisioner "file" {
-    source      = "uploads/shared/consul.d/server_cert.pem"
+    source      = "${path.module}/provision-nomad/uploads/shared/consul.d/server_cert.pem"
     destination = "/tmp/consul_cert.pem"
   }
   provisioner "file" {
-    source      = "uploads/shared/consul.d/server_cert.key.pem"
+    source      = "${path.module}/provision-nomad/uploads/shared/consul.d/server_cert.key.pem"
     destination = "/tmp/consul_cert.key.pem"
   }
   provisioner "file" {
-    source      = "uploads/shared/consul.d/servers.hcl"
+    source      = "${path.module}/provision-nomad/uploads/shared/consul.d/servers.hcl"
     destination = "/tmp/consul_server.hcl"
   }
   provisioner "file" {
@@ -166,10 +166,10 @@ resource "null_resource" "bootstrap_consul_acls" {
   depends_on = [null_resource.install_consul_server_configs]
 
   provisioner "local-exec" {
-    command = "./scripts/bootstrap-consul.sh"
+    command = "${path.module}/scripts/bootstrap-consul.sh"
     environment = {
       CONSUL_HTTP_ADDR           = "https://${aws_instance.consul_server.public_ip}:8501"
-      CONSUL_CACERT              = "keys/tls_ca.crt"
+      CONSUL_CACERT              = "${path.root}/keys/tls_ca.crt"
       CONSUL_HTTP_TOKEN          = "${random_uuid.consul_initial_management_token.result}"
       CONSUL_AGENT_TOKEN         = "${random_uuid.consul_agent_token.result}"
       NOMAD_CLUSTER_CONSUL_TOKEN = "${random_uuid.consul_token_for_nomad.result}"

e2e/terraform/hcp_vault.tf → e2e/terraform/provision-infra/hcp_vault.tf

@@ -48,6 +48,6 @@ resource "local_sensitive_file" "nomad_config_for_vault" {
     namespace = var.hcp_vault_namespace
     role      = "nomad-tasks-${local.random_name}"
   })
-  filename        = "uploads/shared/nomad.d/vault.hcl"
+  filename        = "${path.module}/provision-nomad/uploads/shared/nomad.d/vault.hcl"
   file_permission = "0600"
 }

e2e/terraform/provision-infra/main.tf

@@ -0,0 +1,30 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+data "aws_caller_identity" "current" {
+}
+
+resource "random_pet" "e2e" {
+}
+
+resource "random_password" "windows_admin_password" {
+  length           = 20
+  special          = true
+  override_special = "_%@"
+}
+
+locals {
+  random_name = "${var.name}-${random_pet.e2e.id}"
+}
+
+# Generates keys to use for provisioning and access
+module "keys" {
+  name    = local.random_name
+  path    = "${path.root}/keys"
+  source  = "mitchellh/dynamic-keys/aws"
+  version = "v2.0.0"
+}
+
+data "aws_kms_alias" "e2e" {
+  name = "alias/${var.aws_kms_alias}"
+}

e2e/terraform/network.tf → e2e/terraform/provision-infra/network.tf

@@ -207,9 +207,9 @@ resource "aws_network_interface" "clients_secondary" {
   subnet_id       = data.aws_subnet.secondary.id
   security_groups = [aws_security_group.clients_secondary.id]
 
-  count = var.client_count_ubuntu_jammy_amd64
+  count = var.client_count_linux
   attachment {
-    instance     = aws_instance.client_ubuntu_jammy_amd64[count.index].id
+    instance     = aws_instance.client_ubuntu_jammy[count.index].id
     device_index = 1
   }
 }

e2e/terraform/nomad-acls.tf → e2e/terraform/provision-infra/nomad-acls.tf

@@ -14,16 +14,16 @@ resource "null_resource" "bootstrap_nomad_acls" {
     command = "./scripts/bootstrap-nomad.sh"
     environment = {
       NOMAD_ADDR        = "https://${aws_instance.server.0.public_ip}:4646"
-      NOMAD_CACERT      = "keys/tls_ca.crt"
-      NOMAD_CLIENT_CERT = "keys/tls_api_client.crt"
-      NOMAD_CLIENT_KEY  = "keys/tls_api_client.key"
+      NOMAD_CACERT      = "${path.root}/keys/tls_ca.crt"
+      NOMAD_CLIENT_CERT = "${path.root}/keys/tls_api_client.crt"
+      NOMAD_CLIENT_KEY  = "${path.root}/keys/tls_api_client.key"
     }
   }
 }
 
 data "local_sensitive_file" "nomad_token" {
   depends_on = [null_resource.bootstrap_nomad_acls]
-  filename   = "${path.module}/keys/nomad_root_token"
+  filename   = "${path.root}/keys/nomad_root_token"
 }
 
 # push the token out to the servers for humans to use.