fix(terraform): better cloud-init, hostname setup

terraform/ops-cluster-o11y/cloud-init--userdata.yml

@@ -11,8 +11,11 @@ runcmd:
   - usermod -aG docker freecodecamp
   # This sections should be the last in the list because it involves restarting the ssh service,
   # which may prevent commands from running after it in certain scenarios.
-  - sed -i -e '/^PermitRootLogin/s/^.*$/PermitRootLogin no/' /etc/ssh/sshd_config
-  - sed -i -e '/^PasswordAuthentication/s/^.*$/PasswordAuthentication no/' /etc/ssh/sshd_config
-  - sed -i -e '$aAllowUsers freecodecamp' /etc/ssh/sshd_config
+  - |
+    sed -i -e '/^PermitRootLogin/s/^.*$/PermitRootLogin no/' /etc/ssh/sshd_config
+    sed -i -e '/^PasswordAuthentication/s/^.*$/PasswordAuthentication no/' /etc/ssh/sshd_config
+    sed -i -e '$aAllowUsers freecodecamp' /etc/ssh/sshd_config
+    sed -i '/preserve_hostname: false/c\preserve_hostname: true' /etc/cloud/cloud.cfg
+  # restart ssh service
   - restart ssh"}
 final_message: 'Setup complete'

terraform/ops-cluster-o11y/main.tf

@@ -78,13 +78,22 @@ resource "linode_instance_config" "ops_o11y_leaders_config" {
     host     = linode_instance.ops_o11y_leaders[count.index].ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname ldr-${count.index + 1}.o11y.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"ldr-${count.index + 1}.o11y.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 
@@ -183,13 +192,22 @@ resource "linode_instance_config" "ops_o11y_workers_config" {
     host     = linode_instance.ops_o11y_workers[count.index].ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname wkr-${count.index + 1}.o11y.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"wkr-${count.index + 1}.o11y.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 

terraform/ops-test/cloud-init--userdata.yml.tftpl

@@ -0,0 +1,31 @@
+#cloud-config
+users:
+  - name: freecodecamp
+    groups: sudo
+    shell:  /bin/bash
+    sudo: ['ALL=(ALL) NOPASSWD:ALL']
+    ssh_import_id:
+      - gh:camperbot
+      - raisedadead
+runcmd:
+  - usermod -aG docker freecodecamp
+  # Configure sshd
+  - |
+    sed -i -e '/^PermitRootLogin/s/^.*$/PermitRootLogin no/' /etc/ssh/sshd_config
+    sed -i -e '/^PasswordAuthentication/s/^.*$/PasswordAuthentication no/' /etc/ssh/sshd_config
+    sed -i -e '/^PubkeyAuthentication/s/^.*$/PubkeyAuthentication yes/' /etc/ssh/sshd_config
+    sed -i -e '$aAllowUsers freecodecamp' /etc/ssh/sshd_config
+  # Configure hostname
+  - |
+    sed -i '/preserve_hostname: false/c\preserve_hostname: true' /etc/cloud/cloud.cfg
+    hostnamectl set-hostname ${tf_hostname}
+    echo \"${tf_hostname}\" > /etc/hostname
+  #
+  # :-----------------------: WARNING :-----------------------:
+  #
+  #   This next line should be the last command in the list,
+  #   because it involves restarting the ssh service.
+  #
+  # :-----------------------: WARNING :-----------------------:
+  - systemctl restart sshd
+final_message: 'Setup complete'

terraform/ops-test/main.tf

@@ -39,7 +39,11 @@ resource "linode_instance_disk" "ops_test_disk__boot" {
 
   stackscript_id = data.linode_stackscripts.cloudinit_scripts.stackscripts.0.id
   stackscript_data = {
-    userdata = filebase64("${path.root}/cloud-init--userdata.yml")
+    userdata = base64encode(
+      templatefile("${path.root}/cloud-init--userdata.yml.tftpl", {
+        tf_hostname = "test.${data.linode_domain.ops_dns_domain.domain}"
+      })
+    )
   }
 }
 
@@ -73,13 +77,22 @@ resource "linode_instance_config" "ops_test_config" {
     host     = linode_instance.ops_test.ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname test.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"test.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 

terraform/prd-cluster-publish/cloud-init--userdata.yml

@@ -11,8 +11,11 @@ runcmd:
   - usermod -aG docker freecodecamp
   # This sections should be the last in the list because it involves restarting the ssh service,
   # which may prevent commands from running after it in certain scenarios.
-  - sed -i -e '/^PermitRootLogin/s/^.*$/PermitRootLogin no/' /etc/ssh/sshd_config
-  - sed -i -e '/^PasswordAuthentication/s/^.*$/PasswordAuthentication no/' /etc/ssh/sshd_config
-  - sed -i -e '$aAllowUsers freecodecamp' /etc/ssh/sshd_config
+  - |
+    sed -i -e '/^PermitRootLogin/s/^.*$/PermitRootLogin no/' /etc/ssh/sshd_config
+    sed -i -e '/^PasswordAuthentication/s/^.*$/PasswordAuthentication no/' /etc/ssh/sshd_config
+    sed -i -e '$aAllowUsers freecodecamp' /etc/ssh/sshd_config
+    sed -i '/preserve_hostname: false/c\preserve_hostname: true' /etc/cloud/cloud.cfg
+  # restart ssh service
   - restart ssh"}
 final_message: 'Setup complete'

terraform/prd-cluster-publish/main.tf

@@ -78,13 +78,22 @@ resource "linode_instance_config" "prd_publish_leaders_config" {
     host     = linode_instance.prd_publish_leaders[count.index].ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname ldr-${count.index + 1}.publish.prd.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"ldr-${count.index + 1}.publish.prd.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 
@@ -183,13 +192,22 @@ resource "linode_instance_config" "prd_publish_workers_config" {
     host     = linode_instance.prd_publish_workers[count.index].ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname wkr-${count.index + 1}.publish.prd.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"wkr-${count.index + 1}.publish.prd.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 

terraform/stg-cluster-oldeworld/cloud-init--userdata.yml

@@ -11,8 +11,11 @@ runcmd:
   - usermod -aG docker freecodecamp
   # This sections should be the last in the list because it involves restarting the ssh service,
   # which may prevent commands from running after it in certain scenarios.
-  - sed -i -e '/^PermitRootLogin/s/^.*$/PermitRootLogin no/' /etc/ssh/sshd_config
-  - sed -i -e '/^PasswordAuthentication/s/^.*$/PasswordAuthentication no/' /etc/ssh/sshd_config
-  - sed -i -e '$aAllowUsers freecodecamp' /etc/ssh/sshd_config
+  - |
+    sed -i -e '/^PermitRootLogin/s/^.*$/PermitRootLogin no/' /etc/ssh/sshd_config
+    sed -i -e '/^PasswordAuthentication/s/^.*$/PasswordAuthentication no/' /etc/ssh/sshd_config
+    sed -i -e '$aAllowUsers freecodecamp' /etc/ssh/sshd_config
+    sed -i '/preserve_hostname: false/c\preserve_hostname: true' /etc/cloud/cloud.cfg
+  # restart ssh service
   - restart ssh"}
 final_message: 'Setup complete'

terraform/stg-cluster-oldeworld/next-01-nginx.tf

@@ -56,13 +56,22 @@ resource "linode_instance_config" "stg_oldeworld_pxy_config" {
     host     = linode_instance.stg_oldeworld_pxy[count.index].ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname pxy-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"pxy-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 

terraform/stg-cluster-oldeworld/next-11-client-eng.tf

@@ -56,13 +56,22 @@ resource "linode_instance_config" "stg_oldeworld_clteng_config" {
     host     = linode_instance.stg_oldeworld_clteng[count.index].ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname clteng-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"clteng-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 

terraform/stg-cluster-oldeworld/next-12-client-chn.tf

@@ -56,13 +56,22 @@ resource "linode_instance_config" "stg_oldeworld_cltchn_config" {
     host     = linode_instance.stg_oldeworld_cltchn[count.index].ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname cltchn-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"cltchn-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 

terraform/stg-cluster-oldeworld/next-13-client-cnt.tf

@@ -56,13 +56,22 @@ resource "linode_instance_config" "stg_oldeworld_cltcnt_config" {
     host     = linode_instance.stg_oldeworld_cltcnt[count.index].ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname cltcnt-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"cltcnt-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 

terraform/stg-cluster-oldeworld/next-14-client-esp.tf

@@ -56,13 +56,22 @@ resource "linode_instance_config" "stg_oldeworld_cltesp_config" {
     host     = linode_instance.stg_oldeworld_cltesp[count.index].ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname cltesp-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"cltesp-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 

terraform/stg-cluster-oldeworld/next-15-client-ger.tf

@@ -56,13 +56,22 @@ resource "linode_instance_config" "stg_oldeworld_cltger_config" {
     host     = linode_instance.stg_oldeworld_cltger[count.index].ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname cltger-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"cltger-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }