ci: setup test environment for v2 engine

Signed-off-by: Yang Chiu <[email protected]>

test_framework/terraform/aws/oracle/data.tf

@@ -39,6 +39,7 @@ data "template_file" "provision_k3s_agent" {
     k3s_cluster_secret = random_password.cluster_secret.result
     k3s_version =  var.k8s_distro_version
     custom_ssh_public_key = var.custom_ssh_public_key
+    extra_block_device = var.extra_block_device
   }
 }
 
@@ -61,5 +62,6 @@ data "template_file" "provision_rke2_agent" {
     rke2_cluster_secret = random_password.cluster_secret.result
     rke2_version =  var.k8s_distro_version
     custom_ssh_public_key = var.custom_ssh_public_key
+    extra_block_device = var.extra_block_device
   }
 }

test_framework/terraform/aws/oracle/k3s_instances.tf

@@ -78,6 +78,15 @@ resource "aws_volume_attachment" "lh_aws_hdd_volume_att_k3s" {
   force_detach = true
 }
 
+resource "aws_volume_attachment" "lh_aws_ssd_volume_att_k3s" {
+
+  count = var.extra_block_device && var.k8s_distro_name == "k3s" ? var.lh_aws_instance_count_worker : 0
+
+  device_name  = "/dev/xvdh"
+  volume_id    = aws_ebs_volume.lh_aws_ssd_volume[count.index].id
+  instance_id  = aws_instance.lh_aws_instance_worker_k3s[count.index].id
+  force_detach = true
+}
 
 resource "aws_lb_target_group_attachment" "lh_aws_lb_tg_443_attachment_k3s" {
 

test_framework/terraform/aws/oracle/main.tf

@@ -307,6 +307,20 @@ resource "aws_ebs_volume" "lh_aws_hdd_volume" {
   }
 }
 
+resource "aws_ebs_volume" "lh_aws_ssd_volume" {
+
+  count = var.extra_block_device ? var.lh_aws_instance_count_worker : 0
+
+  availability_zone = var.aws_availability_zone
+  size              = var.lh_aws_instance_root_block_device_size_worker
+  type              = "gp2"
+
+  tags = {
+    Name = "lh-aws-ssd-volume-${count.index}-${random_string.random_suffix.id}"
+    Owner = "longhorn-infra"
+  }
+}
+
 # Create load balancer for rancher
 resource "aws_lb_target_group" "lh_aws_lb_tg_443" {
 

test_framework/terraform/aws/oracle/rke2_instances.tf

@@ -78,6 +78,16 @@ resource "aws_volume_attachment" "lh_aws_hdd_volume_att_rke2" {
   force_detach = true
 }
 
+resource "aws_volume_attachment" "lh_aws_ssd_volume_att_rke2" {
+
+  count = var.extra_block_device && var.k8s_distro_name == "rke2" ? var.lh_aws_instance_count_worker : 0
+
+  device_name  = "/dev/xvdh"
+  volume_id    = aws_ebs_volume.lh_aws_ssd_volume[count.index].id
+  instance_id  = aws_instance.lh_aws_instance_worker_rke2[count.index].id
+  force_detach = true
+}
+
 resource "aws_lb_target_group_attachment" "lh_aws_lb_tg_443_attachment_rke2" {
 
   depends_on = [

test_framework/terraform/aws/oracle/user-data-scripts/provision_k3s_agent.sh.tpl

@@ -9,7 +9,20 @@ sudo yum install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools
 sudo systemctl -q enable iscsid
 sudo systemctl start iscsid
 
-if [ -b "/dev/xvdh" ]; then
+modprobe uio
+modprobe uio_pci_generic
+modprobe nvme-tcp
+touch /etc/modules-load.d/modules.conf
+cat > /etc/modules-load.d/modules.conf <<EOF
+uio
+uio_pci_generic
+nvme-tcp
+EOF
+
+echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
+echo "vm.nr_hugepages=1024" >> /etc/sysctl.conf
+
+if [[ "${extra_block_device}" != true ]] && [[ -b "/dev/xvdh" ]]; then
   mkfs.ext4 -E nodiscard /dev/xvdh
   mkdir /var/lib/longhorn
   mount /dev/xvdh /var/lib/longhorn

test_framework/terraform/aws/oracle/user-data-scripts/provision_rke2_agent.sh.tpl

@@ -9,7 +9,20 @@ sudo yum install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools nc
 sudo systemctl -q enable iscsid
 sudo systemctl start iscsid
 
-if [ -b "/dev/xvdh" ]; then
+modprobe uio
+modprobe uio_pci_generic
+modprobe nvme-tcp
+touch /etc/modules-load.d/modules.conf
+cat > /etc/modules-load.d/modules.conf <<EOF
+uio
+uio_pci_generic
+nvme-tcp
+EOF
+
+echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
+echo "vm.nr_hugepages=1024" >> /etc/sysctl.conf
+
+if [[ "${extra_block_device}" != true ]] && [[ -b "/dev/xvdh" ]]; then
   mkfs.ext4 -E nodiscard /dev/xvdh
   mkdir /var/lib/longhorn
   mount /dev/xvdh /var/lib/longhorn

test_framework/terraform/aws/oracle/variables.tf

@@ -123,3 +123,8 @@ variable "custom_ssh_public_key" {
   default = ""
   sensitive = true
 }
+
+variable "extra_block_device" {
+  type = bool
+  default = false
+}

test_framework/terraform/aws/rhel/data.tf

@@ -45,6 +45,7 @@ data "template_file" "provision_k3s_agent" {
     selinux_mode = var.selinux_mode
     enable_selinux = var.selinux_mode == "permissive" ? "false" : "true"
     custom_ssh_public_key = var.custom_ssh_public_key
+    extra_block_device = var.extra_block_device
   }
 }
 
@@ -69,5 +70,6 @@ data "template_file" "provision_rke2_agent" {
     rke2_version =  var.k8s_distro_version
     selinux_mode = var.selinux_mode
     custom_ssh_public_key = var.custom_ssh_public_key
+    extra_block_device = var.extra_block_device
   }
 }

test_framework/terraform/aws/rhel/k3s_instances.tf

@@ -78,6 +78,16 @@ resource "aws_volume_attachment" "lh_aws_hdd_volume_att_k3s" {
   force_detach = true
 }
 
+resource "aws_volume_attachment" "lh_aws_ssd_volume_att_k3s" {
+
+  count = var.extra_block_device && var.k8s_distro_name == "k3s" ? var.lh_aws_instance_count_worker : 0
+
+  device_name  = "/dev/xvdh"
+  volume_id    = aws_ebs_volume.lh_aws_ssd_volume[count.index].id
+  instance_id  = aws_instance.lh_aws_instance_worker_k3s[count.index].id
+  force_detach = true
+}
+
 resource "aws_lb_target_group_attachment" "lh_aws_lb_tg_443_attachment_k3s" {
 
   depends_on = [

test_framework/terraform/aws/rhel/main.tf

@@ -307,6 +307,20 @@ resource "aws_ebs_volume" "lh_aws_hdd_volume" {
   }
 }
 
+resource "aws_ebs_volume" "lh_aws_ssd_volume" {
+
+  count = var.extra_block_device ? var.lh_aws_instance_count_worker : 0
+
+  availability_zone = var.aws_availability_zone
+  size              = var.lh_aws_instance_root_block_device_size_worker
+  type              = "gp2"
+
+  tags = {
+    Name = "lh-aws-ssd-volume-${count.index}-${random_string.random_suffix.id}"
+    Owner = "longhorn-infra"
+  }
+}
+
 # Create load balancer for rancher
 resource "aws_lb_target_group" "lh_aws_lb_tg_443" {
 

test_framework/terraform/aws/rhel/rke2_instances.tf

@@ -78,6 +78,16 @@ resource "aws_volume_attachment" "lh_aws_hdd_volume_att_rke2" {
   force_detach = true
 }
 
+resource "aws_volume_attachment" "lh_aws_ssd_volume_att_rke2" {
+
+  count = var.extra_block_device && var.k8s_distro_name == "rke2" ? var.lh_aws_instance_count_worker : 0
+
+  device_name  = "/dev/xvdh"
+  volume_id    = aws_ebs_volume.lh_aws_ssd_volume[count.index].id
+  instance_id  = aws_instance.lh_aws_instance_worker_rke2[count.index].id
+  force_detach = true
+}
+
 resource "aws_lb_target_group_attachment" "lh_aws_lb_tg_443_attachment_rke2" {
 
   depends_on = [

test_framework/terraform/aws/rhel/user-data-scripts/provision_k3s_agent.sh.tpl

@@ -15,7 +15,20 @@ sudo systemctl -q enable iscsid
 sudo systemctl start iscsid
 sudo systemctl disable nm-cloud-setup.service nm-cloud-setup.timer
 
-if [ -b "/dev/xvdh" ]; then
+modprobe uio
+modprobe uio_pci_generic
+modprobe nvme-tcp
+touch /etc/modules-load.d/modules.conf
+cat > /etc/modules-load.d/modules.conf <<EOF
+uio
+uio_pci_generic
+nvme-tcp
+EOF
+
+echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
+echo "vm.nr_hugepages=1024" >> /etc/sysctl.conf
+
+if [[ "${extra_block_device}" != true ]] && [[ -b "/dev/xvdh" ]]; then
   sudo mkfs.ext4 -E nodiscard /dev/xvdh
   sudo mkdir /var/lib/longhorn
   sudo mount /dev/xvdh /var/lib/longhorn

test_framework/terraform/aws/rhel/user-data-scripts/provision_rke2_agent.sh.tpl

@@ -15,7 +15,20 @@ sudo systemctl -q enable iscsid
 sudo systemctl start iscsid
 sudo systemctl disable nm-cloud-setup.service nm-cloud-setup.timer
 
-if [ -b "/dev/xvdh" ]; then
+modprobe uio
+modprobe uio_pci_generic
+modprobe nvme-tcp
+touch /etc/modules-load.d/modules.conf
+cat > /etc/modules-load.d/modules.conf <<EOF
+uio
+uio_pci_generic
+nvme-tcp
+EOF
+
+echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
+echo "vm.nr_hugepages=1024" >> /etc/sysctl.conf
+
+if [[ "${extra_block_device}" != true ]] && [[ -b "/dev/xvdh" ]]; then
   sudo mkfs.ext4 -E nodiscard /dev/xvdh
   sudo mkdir /var/lib/longhorn
   sudo mount /dev/xvdh /var/lib/longhorn

test_framework/terraform/aws/rhel/variables.tf

@@ -129,3 +129,8 @@ variable "custom_ssh_public_key" {
   default = ""
   sensitive = true
 }
+
+variable "extra_block_device" {
+  type = bool
+  default = false
+}

test_framework/terraform/aws/rockylinux/data.tf

@@ -50,6 +50,7 @@ data "template_file" "provision_k3s_agent" {
     selinux_mode = var.selinux_mode
     enable_selinux = var.selinux_mode == "permissive" ? "false" : "true"
     custom_ssh_public_key = var.custom_ssh_public_key
+    extra_block_device = var.extra_block_device
   }
 }
 
@@ -74,5 +75,6 @@ data "template_file" "provision_rke2_agent" {
     rke2_version =  var.k8s_distro_version
     selinux_mode = var.selinux_mode
     custom_ssh_public_key = var.custom_ssh_public_key
+    extra_block_device = var.extra_block_device
   }
 }

test_framework/terraform/aws/rockylinux/k3s_instances.tf

@@ -78,6 +78,16 @@ resource "aws_volume_attachment" "lh_aws_hdd_volume_att_k3s" {
   force_detach = true
 }
 
+resource "aws_volume_attachment" "lh_aws_ssd_volume_att_k3s" {
+
+  count = var.extra_block_device && var.k8s_distro_name == "k3s" ? var.lh_aws_instance_count_worker : 0
+
+  device_name  = "/dev/xvdh"
+  volume_id    = aws_ebs_volume.lh_aws_ssd_volume[count.index].id
+  instance_id  = aws_instance.lh_aws_instance_worker_k3s[count.index].id
+  force_detach = true
+}
+
 resource "aws_lb_target_group_attachment" "lh_aws_lb_tg_443_attachment_k3s" {
 
   depends_on = [

test_framework/terraform/aws/rockylinux/main.tf

@@ -308,6 +308,20 @@ resource "aws_ebs_volume" "lh_aws_hdd_volume" {
   }
 }
 
+resource "aws_ebs_volume" "lh_aws_ssd_volume" {
+
+  count = var.extra_block_device ? var.lh_aws_instance_count_worker : 0
+
+  availability_zone = var.aws_availability_zone
+  size              = var.lh_aws_instance_root_block_device_size_worker
+  type              = "gp2"
+
+  tags = {
+    Name = "lh-aws-ssd-volume-${count.index}-${random_string.random_suffix.id}"
+    Owner = "longhorn-infra"
+  }
+}
+
 # Create load balancer for rancher
 resource "aws_lb_target_group" "lh_aws_lb_tg_443" {
 

test_framework/terraform/aws/rockylinux/rke2_instances.tf

@@ -78,6 +78,16 @@ resource "aws_volume_attachment" "lh_aws_hdd_volume_att_rke2" {
   force_detach = true
 }
 
+resource "aws_volume_attachment" "lh_aws_ssd_volume_att_rke2" {
+
+  count = var.extra_block_device && var.k8s_distro_name == "rke2" ? var.lh_aws_instance_count_worker : 0
+
+  device_name  = "/dev/xvdh"
+  volume_id    = aws_ebs_volume.lh_aws_ssd_volume[count.index].id
+  instance_id  = aws_instance.lh_aws_instance_worker_rke2[count.index].id
+  force_detach = true
+}
+
 resource "aws_lb_target_group_attachment" "lh_aws_lb_tg_443_attachment_rke2" {
 
   depends_on = [

test_framework/terraform/aws/rockylinux/user-data-scripts/provision_k3s_agent.sh.tpl

@@ -14,7 +14,20 @@ sudo dnf install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools
 sudo systemctl -q enable iscsid
 sudo systemctl start iscsid
 
-if [ -b "/dev/xvdh" ]; then
+modprobe uio
+modprobe uio_pci_generic
+modprobe nvme-tcp
+touch /etc/modules-load.d/modules.conf
+cat > /etc/modules-load.d/modules.conf <<EOF
+uio
+uio_pci_generic
+nvme-tcp
+EOF
+
+echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
+echo "vm.nr_hugepages=1024" >> /etc/sysctl.conf
+
+if [[ "${extra_block_device}" != true ]] && [[ -b "/dev/xvdh" ]]; then
   sudo mkfs.ext4 -E nodiscard /dev/xvdh
   sudo mkdir /var/lib/longhorn
   sudo mount /dev/xvdh /var/lib/longhorn

test_framework/terraform/aws/rockylinux/user-data-scripts/provision_rke2_agent.sh.tpl

@@ -14,7 +14,20 @@ sudo dnf install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools jq nmap-ncat
 sudo systemctl -q enable iscsid
 sudo systemctl start iscsid
 
-if [ -b "/dev/xvdh" ]; then
+modprobe uio
+modprobe uio_pci_generic
+modprobe nvme-tcp
+touch /etc/modules-load.d/modules.conf
+cat > /etc/modules-load.d/modules.conf <<EOF
+uio
+uio_pci_generic
+nvme-tcp
+EOF
+
+echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
+echo "vm.nr_hugepages=1024" >> /etc/sysctl.conf
+
+if [[ "${extra_block_device}" != true ]] && [[ -b "/dev/xvdh" ]]; then
   sudo mkfs.ext4 -E nodiscard /dev/xvdh
   sudo mkdir /var/lib/longhorn
   sudo mount /dev/xvdh /var/lib/longhorn

test_framework/terraform/aws/rockylinux/variables.tf

@@ -128,4 +128,9 @@ variable "custom_ssh_public_key" {
   type = string
   default = ""
   sensitive = true
-}
+}
+
+variable "extra_block_device" {
+  type = bool
+  default = false
+}

test_framework/terraform/aws/sle-micro/data.tf

@@ -32,6 +32,7 @@ data "template_file" "provision_k3s_agent" {
     k3s_cluster_secret = random_password.cluster_secret.result
     k3s_version =  var.k8s_distro_version
     custom_ssh_public_key = var.custom_ssh_public_key
+    extra_block_device = var.extra_block_device
   }
 }
 
@@ -54,5 +55,6 @@ data "template_file" "provision_rke2_agent" {
     rke2_cluster_secret = random_password.cluster_secret.result
     rke2_version =  var.k8s_distro_version
     custom_ssh_public_key = var.custom_ssh_public_key
+    extra_block_device = var.extra_block_device
   }
 }