From adea2f2db4ae7f54fd5288784d146b4ac5ff99b9 Mon Sep 17 00:00:00 2001
From: Mrugesh Mohapatra <noreply@mrugesh.dev>
Date: Sun, 16 Jul 2023 21:39:56 +0530
Subject: [PATCH] fix(terraform): better cloud-init, hostname setup

---
 .../ops-cluster-o11y/cloud-init--userdata.yml |  9 ++++--
 terraform/ops-cluster-o11y/main.tf            | 30 ++++++++++++++----
 terraform/ops-test/cloud-init--userdata.yml   | 18 -----------
 .../ops-test/cloud-init--userdata.yml.tftpl   | 31 +++++++++++++++++++
 terraform/ops-test/main.tf                    | 21 ++++++++++---
 .../cloud-init--userdata.yml                  |  9 ++++--
 terraform/prd-cluster-publish/main.tf         | 30 ++++++++++++++----
 .../cloud-init--userdata.yml                  |  9 ++++--
 .../stg-cluster-oldeworld/next-01-nginx.tf    | 15 +++++++--
 .../next-11-client-eng.tf                     | 15 +++++++--
 .../next-12-client-chn.tf                     | 15 +++++++--
 .../next-13-client-cnt.tf                     | 15 +++++++--
 .../next-14-client-esp.tf                     | 15 +++++++--
 .../next-15-client-ger.tf                     | 15 +++++++--
 .../next-16-client-ita.tf                     | 15 +++++++--
 .../next-17-client-jpn.tf                     | 15 +++++++--
 .../next-18-client-por.tf                     | 15 +++++++--
 .../next-19-client-ukr.tf                     | 15 +++++++--
 .../stg-cluster-oldeworld/next-20-api.tf      | 15 +++++++--
 .../stg-cluster-oldeworld/next-50-ghost.tf    | 15 +++++++--
 20 files changed, 258 insertions(+), 79 deletions(-)
 delete mode 100644 terraform/ops-test/cloud-init--userdata.yml
 create mode 100644 terraform/ops-test/cloud-init--userdata.yml.tftpl

diff --git a/terraform/ops-cluster-o11y/cloud-init--userdata.yml b/terraform/ops-cluster-o11y/cloud-init--userdata.yml
index 436fa4cf1..8e8cd2d90 100644
--- a/terraform/ops-cluster-o11y/cloud-init--userdata.yml
+++ b/terraform/ops-cluster-o11y/cloud-init--userdata.yml
@@ -11,8 +11,11 @@ runcmd:
   - usermod -aG docker freecodecamp
   # This sections should be the last in the list because it involves restarting the ssh service,
   # which may prevent commands from running after it in certain scenarios.
-  - sed -i -e '/^PermitRootLogin/s/^.*$/PermitRootLogin no/' /etc/ssh/sshd_config
-  - sed -i -e '/^PasswordAuthentication/s/^.*$/PasswordAuthentication no/' /etc/ssh/sshd_config
-  - sed -i -e '$aAllowUsers freecodecamp' /etc/ssh/sshd_config
+  - |
+    sed -i -e '/^PermitRootLogin/s/^.*$/PermitRootLogin no/' /etc/ssh/sshd_config
+    sed -i -e '/^PasswordAuthentication/s/^.*$/PasswordAuthentication no/' /etc/ssh/sshd_config
+    sed -i -e '$aAllowUsers freecodecamp' /etc/ssh/sshd_config
+    sed -i '/preserve_hostname: false/c\preserve_hostname: true' /etc/cloud/cloud.cfg
+  # restart ssh service
   - restart ssh"}
 final_message: 'Setup complete'
diff --git a/terraform/ops-cluster-o11y/main.tf b/terraform/ops-cluster-o11y/main.tf
index 9dba73551..990f93e3c 100644
--- a/terraform/ops-cluster-o11y/main.tf
+++ b/terraform/ops-cluster-o11y/main.tf
@@ -78,13 +78,22 @@ resource "linode_instance_config" "ops_o11y_leaders_config" {
     host     = linode_instance.ops_o11y_leaders[count.index].ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname ldr-${count.index + 1}.o11y.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"ldr-${count.index + 1}.o11y.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 
@@ -183,13 +192,22 @@ resource "linode_instance_config" "ops_o11y_workers_config" {
     host     = linode_instance.ops_o11y_workers[count.index].ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname wkr-${count.index + 1}.o11y.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"wkr-${count.index + 1}.o11y.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 
diff --git a/terraform/ops-test/cloud-init--userdata.yml b/terraform/ops-test/cloud-init--userdata.yml
deleted file mode 100644
index 436fa4cf1..000000000
--- a/terraform/ops-test/cloud-init--userdata.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-#cloud-config
-users:
-  - name: freecodecamp
-    groups: sudo
-    shell:  /bin/bash
-    sudo: ['ALL=(ALL) NOPASSWD:ALL']
-    ssh_import_id:
-      - gh:camperbot
-      - raisedadead
-runcmd:
-  - usermod -aG docker freecodecamp
-  # This sections should be the last in the list because it involves restarting the ssh service,
-  # which may prevent commands from running after it in certain scenarios.
-  - sed -i -e '/^PermitRootLogin/s/^.*$/PermitRootLogin no/' /etc/ssh/sshd_config
-  - sed -i -e '/^PasswordAuthentication/s/^.*$/PasswordAuthentication no/' /etc/ssh/sshd_config
-  - sed -i -e '$aAllowUsers freecodecamp' /etc/ssh/sshd_config
-  - restart ssh"}
-final_message: 'Setup complete'
diff --git a/terraform/ops-test/cloud-init--userdata.yml.tftpl b/terraform/ops-test/cloud-init--userdata.yml.tftpl
new file mode 100644
index 000000000..3166bd779
--- /dev/null
+++ b/terraform/ops-test/cloud-init--userdata.yml.tftpl
@@ -0,0 +1,31 @@
+#cloud-config
+users:
+  - name: freecodecamp
+    groups: sudo
+    shell:  /bin/bash
+    sudo: ['ALL=(ALL) NOPASSWD:ALL']
+    ssh_import_id:
+      - gh:camperbot
+      - raisedadead
+runcmd:
+  - usermod -aG docker freecodecamp
+  # Configure sshd
+  - |
+    sed -i -e '/^PermitRootLogin/s/^.*$/PermitRootLogin no/' /etc/ssh/sshd_config
+    sed -i -e '/^PasswordAuthentication/s/^.*$/PasswordAuthentication no/' /etc/ssh/sshd_config
+    sed -i -e '/^PubkeyAuthentication/s/^.*$/PubkeyAuthentication yes/' /etc/ssh/sshd_config
+    sed -i -e '$aAllowUsers freecodecamp' /etc/ssh/sshd_config
+  # Configure hostname
+  - |
+    sed -i '/preserve_hostname: false/c\preserve_hostname: true' /etc/cloud/cloud.cfg
+    hostnamectl set-hostname ${tf_hostname}
+    echo \"${tf_hostname}\" > /etc/hostname
+  #
+  # :-----------------------: WARNING :-----------------------:
+  #
+  #   This next line should be the last command in the list,
+  #   because it involves restarting the ssh service.
+  #
+  # :-----------------------: WARNING :-----------------------:
+  - systemctl restart sshd
+final_message: 'Setup complete'
diff --git a/terraform/ops-test/main.tf b/terraform/ops-test/main.tf
index 80a1e3782..333299059 100644
--- a/terraform/ops-test/main.tf
+++ b/terraform/ops-test/main.tf
@@ -39,7 +39,11 @@ resource "linode_instance_disk" "ops_test_disk__boot" {
 
   stackscript_id = data.linode_stackscripts.cloudinit_scripts.stackscripts.0.id
   stackscript_data = {
-    userdata = filebase64("${path.root}/cloud-init--userdata.yml")
+    userdata = base64encode(
+      templatefile("${path.root}/cloud-init--userdata.yml.tftpl", {
+        tf_hostname = "test.${data.linode_domain.ops_dns_domain.domain}"
+      })
+    )
   }
 }
 
@@ -73,13 +77,22 @@ resource "linode_instance_config" "ops_test_config" {
     host     = linode_instance.ops_test.ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname test.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"test.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 
diff --git a/terraform/prd-cluster-publish/cloud-init--userdata.yml b/terraform/prd-cluster-publish/cloud-init--userdata.yml
index 436fa4cf1..8e8cd2d90 100644
--- a/terraform/prd-cluster-publish/cloud-init--userdata.yml
+++ b/terraform/prd-cluster-publish/cloud-init--userdata.yml
@@ -11,8 +11,11 @@ runcmd:
   - usermod -aG docker freecodecamp
   # This sections should be the last in the list because it involves restarting the ssh service,
   # which may prevent commands from running after it in certain scenarios.
-  - sed -i -e '/^PermitRootLogin/s/^.*$/PermitRootLogin no/' /etc/ssh/sshd_config
-  - sed -i -e '/^PasswordAuthentication/s/^.*$/PasswordAuthentication no/' /etc/ssh/sshd_config
-  - sed -i -e '$aAllowUsers freecodecamp' /etc/ssh/sshd_config
+  - |
+    sed -i -e '/^PermitRootLogin/s/^.*$/PermitRootLogin no/' /etc/ssh/sshd_config
+    sed -i -e '/^PasswordAuthentication/s/^.*$/PasswordAuthentication no/' /etc/ssh/sshd_config
+    sed -i -e '$aAllowUsers freecodecamp' /etc/ssh/sshd_config
+    sed -i '/preserve_hostname: false/c\preserve_hostname: true' /etc/cloud/cloud.cfg
+  # restart ssh service
   - restart ssh"}
 final_message: 'Setup complete'
diff --git a/terraform/prd-cluster-publish/main.tf b/terraform/prd-cluster-publish/main.tf
index 0d1a96d3d..a8eb652eb 100644
--- a/terraform/prd-cluster-publish/main.tf
+++ b/terraform/prd-cluster-publish/main.tf
@@ -78,13 +78,22 @@ resource "linode_instance_config" "prd_publish_leaders_config" {
     host     = linode_instance.prd_publish_leaders[count.index].ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname ldr-${count.index + 1}.publish.prd.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"ldr-${count.index + 1}.publish.prd.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 
@@ -183,13 +192,22 @@ resource "linode_instance_config" "prd_publish_workers_config" {
     host     = linode_instance.prd_publish_workers[count.index].ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname wkr-${count.index + 1}.publish.prd.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"wkr-${count.index + 1}.publish.prd.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 
diff --git a/terraform/stg-cluster-oldeworld/cloud-init--userdata.yml b/terraform/stg-cluster-oldeworld/cloud-init--userdata.yml
index 436fa4cf1..8e8cd2d90 100644
--- a/terraform/stg-cluster-oldeworld/cloud-init--userdata.yml
+++ b/terraform/stg-cluster-oldeworld/cloud-init--userdata.yml
@@ -11,8 +11,11 @@ runcmd:
   - usermod -aG docker freecodecamp
   # This sections should be the last in the list because it involves restarting the ssh service,
   # which may prevent commands from running after it in certain scenarios.
-  - sed -i -e '/^PermitRootLogin/s/^.*$/PermitRootLogin no/' /etc/ssh/sshd_config
-  - sed -i -e '/^PasswordAuthentication/s/^.*$/PasswordAuthentication no/' /etc/ssh/sshd_config
-  - sed -i -e '$aAllowUsers freecodecamp' /etc/ssh/sshd_config
+  - |
+    sed -i -e '/^PermitRootLogin/s/^.*$/PermitRootLogin no/' /etc/ssh/sshd_config
+    sed -i -e '/^PasswordAuthentication/s/^.*$/PasswordAuthentication no/' /etc/ssh/sshd_config
+    sed -i -e '$aAllowUsers freecodecamp' /etc/ssh/sshd_config
+    sed -i '/preserve_hostname: false/c\preserve_hostname: true' /etc/cloud/cloud.cfg
+  # restart ssh service
   - restart ssh"}
 final_message: 'Setup complete'
diff --git a/terraform/stg-cluster-oldeworld/next-01-nginx.tf b/terraform/stg-cluster-oldeworld/next-01-nginx.tf
index c0573ed77..06137ddc3 100644
--- a/terraform/stg-cluster-oldeworld/next-01-nginx.tf
+++ b/terraform/stg-cluster-oldeworld/next-01-nginx.tf
@@ -56,13 +56,22 @@ resource "linode_instance_config" "stg_oldeworld_pxy_config" {
     host     = linode_instance.stg_oldeworld_pxy[count.index].ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname pxy-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"pxy-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 
diff --git a/terraform/stg-cluster-oldeworld/next-11-client-eng.tf b/terraform/stg-cluster-oldeworld/next-11-client-eng.tf
index 2e2297f21..417e87140 100644
--- a/terraform/stg-cluster-oldeworld/next-11-client-eng.tf
+++ b/terraform/stg-cluster-oldeworld/next-11-client-eng.tf
@@ -56,13 +56,22 @@ resource "linode_instance_config" "stg_oldeworld_clteng_config" {
     host     = linode_instance.stg_oldeworld_clteng[count.index].ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname clteng-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"clteng-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 
diff --git a/terraform/stg-cluster-oldeworld/next-12-client-chn.tf b/terraform/stg-cluster-oldeworld/next-12-client-chn.tf
index 47bb14f23..4add90b4f 100644
--- a/terraform/stg-cluster-oldeworld/next-12-client-chn.tf
+++ b/terraform/stg-cluster-oldeworld/next-12-client-chn.tf
@@ -56,13 +56,22 @@ resource "linode_instance_config" "stg_oldeworld_cltchn_config" {
     host     = linode_instance.stg_oldeworld_cltchn[count.index].ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname cltchn-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"cltchn-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 
diff --git a/terraform/stg-cluster-oldeworld/next-13-client-cnt.tf b/terraform/stg-cluster-oldeworld/next-13-client-cnt.tf
index dce059b89..081361fe7 100644
--- a/terraform/stg-cluster-oldeworld/next-13-client-cnt.tf
+++ b/terraform/stg-cluster-oldeworld/next-13-client-cnt.tf
@@ -56,13 +56,22 @@ resource "linode_instance_config" "stg_oldeworld_cltcnt_config" {
     host     = linode_instance.stg_oldeworld_cltcnt[count.index].ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname cltcnt-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"cltcnt-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 
diff --git a/terraform/stg-cluster-oldeworld/next-14-client-esp.tf b/terraform/stg-cluster-oldeworld/next-14-client-esp.tf
index e396da0cd..3cfe73b72 100644
--- a/terraform/stg-cluster-oldeworld/next-14-client-esp.tf
+++ b/terraform/stg-cluster-oldeworld/next-14-client-esp.tf
@@ -56,13 +56,22 @@ resource "linode_instance_config" "stg_oldeworld_cltesp_config" {
     host     = linode_instance.stg_oldeworld_cltesp[count.index].ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname cltesp-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"cltesp-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 
diff --git a/terraform/stg-cluster-oldeworld/next-15-client-ger.tf b/terraform/stg-cluster-oldeworld/next-15-client-ger.tf
index ac4c8c20d..193ed447c 100644
--- a/terraform/stg-cluster-oldeworld/next-15-client-ger.tf
+++ b/terraform/stg-cluster-oldeworld/next-15-client-ger.tf
@@ -56,13 +56,22 @@ resource "linode_instance_config" "stg_oldeworld_cltger_config" {
     host     = linode_instance.stg_oldeworld_cltger[count.index].ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname cltger-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"cltger-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 
diff --git a/terraform/stg-cluster-oldeworld/next-16-client-ita.tf b/terraform/stg-cluster-oldeworld/next-16-client-ita.tf
index b33558262..a67a4fdbb 100644
--- a/terraform/stg-cluster-oldeworld/next-16-client-ita.tf
+++ b/terraform/stg-cluster-oldeworld/next-16-client-ita.tf
@@ -56,13 +56,22 @@ resource "linode_instance_config" "stg_oldeworld_cltita_config" {
     host     = linode_instance.stg_oldeworld_cltita[count.index].ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname cltita-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"cltita-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 
diff --git a/terraform/stg-cluster-oldeworld/next-17-client-jpn.tf b/terraform/stg-cluster-oldeworld/next-17-client-jpn.tf
index 9a807b49f..b6dc651af 100644
--- a/terraform/stg-cluster-oldeworld/next-17-client-jpn.tf
+++ b/terraform/stg-cluster-oldeworld/next-17-client-jpn.tf
@@ -56,13 +56,22 @@ resource "linode_instance_config" "stg_oldeworld_cltjpn_config" {
     host     = linode_instance.stg_oldeworld_cltjpn[count.index].ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname cltjpn-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"cltjpn-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 
diff --git a/terraform/stg-cluster-oldeworld/next-18-client-por.tf b/terraform/stg-cluster-oldeworld/next-18-client-por.tf
index 7346d2dab..6548f4e0b 100644
--- a/terraform/stg-cluster-oldeworld/next-18-client-por.tf
+++ b/terraform/stg-cluster-oldeworld/next-18-client-por.tf
@@ -56,13 +56,22 @@ resource "linode_instance_config" "stg_oldeworld_cltpor_config" {
     host     = linode_instance.stg_oldeworld_cltpor[count.index].ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname cltpor-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"cltpor-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 
diff --git a/terraform/stg-cluster-oldeworld/next-19-client-ukr.tf b/terraform/stg-cluster-oldeworld/next-19-client-ukr.tf
index 8f6081f25..573817193 100644
--- a/terraform/stg-cluster-oldeworld/next-19-client-ukr.tf
+++ b/terraform/stg-cluster-oldeworld/next-19-client-ukr.tf
@@ -56,13 +56,22 @@ resource "linode_instance_config" "stg_oldeworld_cltukr_config" {
     host     = linode_instance.stg_oldeworld_cltukr[count.index].ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname cltukr-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"cltukr-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 
diff --git a/terraform/stg-cluster-oldeworld/next-20-api.tf b/terraform/stg-cluster-oldeworld/next-20-api.tf
index 93a777e78..8bb42ea3b 100644
--- a/terraform/stg-cluster-oldeworld/next-20-api.tf
+++ b/terraform/stg-cluster-oldeworld/next-20-api.tf
@@ -56,13 +56,22 @@ resource "linode_instance_config" "stg_oldeworld_api_config" {
     host     = linode_instance.stg_oldeworld_api[count.index].ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname api-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"api-${count.index + 1}.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }
 
diff --git a/terraform/stg-cluster-oldeworld/next-50-ghost.tf b/terraform/stg-cluster-oldeworld/next-50-ghost.tf
index 72d88c0ae..4b306b081 100644
--- a/terraform/stg-cluster-oldeworld/next-50-ghost.tf
+++ b/terraform/stg-cluster-oldeworld/next-50-ghost.tf
@@ -53,13 +53,22 @@ resource "linode_instance_config" "stg_oldeworld_newstst_config" {
     host     = linode_instance.stg_oldeworld_newstst.ip_address
   }
 
+  # All of the provisioning should be done via cloud-init.
+  # This is just to setup the reboot.
   provisioner "remote-exec" {
     inline = [
       # Wait for cloud-init to finish.
       "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
-      # Set the hostname.
-      "hostnamectl set-hostname newstst-1.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}",
-      "echo \"newstst-1.oldeworld.stg.${data.linode_domain.ops_dns_domain.domain}\" > /etc/hostname",
+      "echo Current hostname...; hostname",
+      "shutdown -r +1 'Terraform: Rebooting to apply hostname change in 1 min.'"
+    ]
+  }
+
+  # This run is a hack to trigger the reboot,
+  # which may fail otherwise in the previous step.
+  provisioner "remote-exec" {
+    inline = [
+      "uptime"
     ]
   }