diff --git a/test_framework/terraform/aws/oracle/data.tf b/test_framework/terraform/aws/oracle/data.tf index 1072b2ce91..418fed389b 100644 --- a/test_framework/terraform/aws/oracle/data.tf +++ b/test_framework/terraform/aws/oracle/data.tf @@ -39,6 +39,7 @@ data "template_file" "provision_k3s_agent" { k3s_cluster_secret = random_password.cluster_secret.result k3s_version = var.k8s_distro_version custom_ssh_public_key = var.custom_ssh_public_key + extra_block_device = var.extra_block_device } } @@ -61,5 +62,6 @@ data "template_file" "provision_rke2_agent" { rke2_cluster_secret = random_password.cluster_secret.result rke2_version = var.k8s_distro_version custom_ssh_public_key = var.custom_ssh_public_key + extra_block_device = var.extra_block_device } } diff --git a/test_framework/terraform/aws/oracle/k3s_instances.tf b/test_framework/terraform/aws/oracle/k3s_instances.tf index 8c8ff8f821..a192d6f814 100644 --- a/test_framework/terraform/aws/oracle/k3s_instances.tf +++ b/test_framework/terraform/aws/oracle/k3s_instances.tf @@ -78,6 +78,15 @@ resource "aws_volume_attachment" "lh_aws_hdd_volume_att_k3s" { force_detach = true } +resource "aws_volume_attachment" "lh_aws_ssd_volume_att_k3s" { + + count = var.extra_block_device && var.k8s_distro_name == "k3s" ? var.lh_aws_instance_count_worker : 0 + + device_name = "/dev/xvdh" + volume_id = aws_ebs_volume.lh_aws_ssd_volume[count.index].id + instance_id = aws_instance.lh_aws_instance_worker_k3s[count.index].id + force_detach = true +} resource "aws_lb_target_group_attachment" "lh_aws_lb_tg_443_attachment_k3s" { diff --git a/test_framework/terraform/aws/oracle/main.tf b/test_framework/terraform/aws/oracle/main.tf index 3fddf19914..97f5367da0 100644 --- a/test_framework/terraform/aws/oracle/main.tf +++ b/test_framework/terraform/aws/oracle/main.tf @@ -307,6 +307,20 @@ resource "aws_ebs_volume" "lh_aws_hdd_volume" { } } +resource "aws_ebs_volume" "lh_aws_ssd_volume" { + + count = var.extra_block_device ? var.lh_aws_instance_count_worker : 0 + + availability_zone = var.aws_availability_zone + size = var.lh_aws_instance_root_block_device_size_worker + type = "gp2" + + tags = { + Name = "lh-aws-ssd-volume-${count.index}-${random_string.random_suffix.id}" + Owner = "longhorn-infra" + } +} + # Create load balancer for rancher resource "aws_lb_target_group" "lh_aws_lb_tg_443" { diff --git a/test_framework/terraform/aws/oracle/rke2_instances.tf b/test_framework/terraform/aws/oracle/rke2_instances.tf index 618642d4b1..768935c0e6 100644 --- a/test_framework/terraform/aws/oracle/rke2_instances.tf +++ b/test_framework/terraform/aws/oracle/rke2_instances.tf @@ -78,6 +78,16 @@ resource "aws_volume_attachment" "lh_aws_hdd_volume_att_rke2" { force_detach = true } +resource "aws_volume_attachment" "lh_aws_ssd_volume_att_rke2" { + + count = var.extra_block_device && var.k8s_distro_name == "rke2" ? var.lh_aws_instance_count_worker : 0 + + device_name = "/dev/xvdh" + volume_id = aws_ebs_volume.lh_aws_ssd_volume[count.index].id + instance_id = aws_instance.lh_aws_instance_worker_rke2[count.index].id + force_detach = true +} + resource "aws_lb_target_group_attachment" "lh_aws_lb_tg_443_attachment_rke2" { depends_on = [ diff --git a/test_framework/terraform/aws/oracle/user-data-scripts/provision_k3s_agent.sh.tpl b/test_framework/terraform/aws/oracle/user-data-scripts/provision_k3s_agent.sh.tpl index 092350791b..abd2338377 100755 --- a/test_framework/terraform/aws/oracle/user-data-scripts/provision_k3s_agent.sh.tpl +++ b/test_framework/terraform/aws/oracle/user-data-scripts/provision_k3s_agent.sh.tpl @@ -9,7 +9,20 @@ sudo yum install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools sudo systemctl -q enable iscsid sudo systemctl start iscsid -if [ -b "/dev/xvdh" ]; then +modprobe uio +modprobe uio_pci_generic +modprobe nvme-tcp +touch /etc/modules-load.d/modules.conf +cat > /etc/modules-load.d/modules.conf < /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages +echo "vm.nr_hugepages=1024" >> /etc/sysctl.conf + +if [[ "${extra_block_device}" != true ]] && [[ -b "/dev/xvdh" ]]; then mkfs.ext4 -E nodiscard /dev/xvdh mkdir /var/lib/longhorn mount /dev/xvdh /var/lib/longhorn diff --git a/test_framework/terraform/aws/oracle/user-data-scripts/provision_rke2_agent.sh.tpl b/test_framework/terraform/aws/oracle/user-data-scripts/provision_rke2_agent.sh.tpl index f4f8089780..0e0a2b3773 100644 --- a/test_framework/terraform/aws/oracle/user-data-scripts/provision_rke2_agent.sh.tpl +++ b/test_framework/terraform/aws/oracle/user-data-scripts/provision_rke2_agent.sh.tpl @@ -9,7 +9,20 @@ sudo yum install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools nc sudo systemctl -q enable iscsid sudo systemctl start iscsid -if [ -b "/dev/xvdh" ]; then +modprobe uio +modprobe uio_pci_generic +modprobe nvme-tcp +touch /etc/modules-load.d/modules.conf +cat > /etc/modules-load.d/modules.conf < /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages +echo "vm.nr_hugepages=1024" >> /etc/sysctl.conf + +if [[ "${extra_block_device}" != true ]] && [[ -b "/dev/xvdh" ]]; then mkfs.ext4 -E nodiscard /dev/xvdh mkdir /var/lib/longhorn mount /dev/xvdh /var/lib/longhorn diff --git a/test_framework/terraform/aws/oracle/variables.tf b/test_framework/terraform/aws/oracle/variables.tf index 3e7cc4dd70..411ce6dd79 100644 --- a/test_framework/terraform/aws/oracle/variables.tf +++ b/test_framework/terraform/aws/oracle/variables.tf @@ -123,3 +123,8 @@ variable "custom_ssh_public_key" { default = "" sensitive = true } + +variable "extra_block_device" { + type = bool + default = false +} diff --git a/test_framework/terraform/aws/rhel/data.tf b/test_framework/terraform/aws/rhel/data.tf index c80ecb91d5..0d509dc661 100644 --- a/test_framework/terraform/aws/rhel/data.tf +++ b/test_framework/terraform/aws/rhel/data.tf @@ -45,6 +45,7 @@ data "template_file" "provision_k3s_agent" { selinux_mode = var.selinux_mode enable_selinux = var.selinux_mode == "permissive" ? "false" : "true" custom_ssh_public_key = var.custom_ssh_public_key + extra_block_device = var.extra_block_device } } @@ -69,5 +70,6 @@ data "template_file" "provision_rke2_agent" { rke2_version = var.k8s_distro_version selinux_mode = var.selinux_mode custom_ssh_public_key = var.custom_ssh_public_key + extra_block_device = var.extra_block_device } } diff --git a/test_framework/terraform/aws/rhel/k3s_instances.tf b/test_framework/terraform/aws/rhel/k3s_instances.tf index 649f1b0ed9..0938fc62b1 100644 --- a/test_framework/terraform/aws/rhel/k3s_instances.tf +++ b/test_framework/terraform/aws/rhel/k3s_instances.tf @@ -78,6 +78,16 @@ resource "aws_volume_attachment" "lh_aws_hdd_volume_att_k3s" { force_detach = true } +resource "aws_volume_attachment" "lh_aws_ssd_volume_att_k3s" { + + count = var.extra_block_device && var.k8s_distro_name == "k3s" ? var.lh_aws_instance_count_worker : 0 + + device_name = "/dev/xvdh" + volume_id = aws_ebs_volume.lh_aws_ssd_volume[count.index].id + instance_id = aws_instance.lh_aws_instance_worker_k3s[count.index].id + force_detach = true +} + resource "aws_lb_target_group_attachment" "lh_aws_lb_tg_443_attachment_k3s" { depends_on = [ diff --git a/test_framework/terraform/aws/rhel/main.tf b/test_framework/terraform/aws/rhel/main.tf index 3fddf19914..97f5367da0 100644 --- a/test_framework/terraform/aws/rhel/main.tf +++ b/test_framework/terraform/aws/rhel/main.tf @@ -307,6 +307,20 @@ resource "aws_ebs_volume" "lh_aws_hdd_volume" { } } +resource "aws_ebs_volume" "lh_aws_ssd_volume" { + + count = var.extra_block_device ? var.lh_aws_instance_count_worker : 0 + + availability_zone = var.aws_availability_zone + size = var.lh_aws_instance_root_block_device_size_worker + type = "gp2" + + tags = { + Name = "lh-aws-ssd-volume-${count.index}-${random_string.random_suffix.id}" + Owner = "longhorn-infra" + } +} + # Create load balancer for rancher resource "aws_lb_target_group" "lh_aws_lb_tg_443" { diff --git a/test_framework/terraform/aws/rhel/rke2_instances.tf b/test_framework/terraform/aws/rhel/rke2_instances.tf index ec2182cdd2..4c30404479 100644 --- a/test_framework/terraform/aws/rhel/rke2_instances.tf +++ b/test_framework/terraform/aws/rhel/rke2_instances.tf @@ -78,6 +78,16 @@ resource "aws_volume_attachment" "lh_aws_hdd_volume_att_rke2" { force_detach = true } +resource "aws_volume_attachment" "lh_aws_ssd_volume_att_rke2" { + + count = var.extra_block_device && var.k8s_distro_name == "rke2" ? var.lh_aws_instance_count_worker : 0 + + device_name = "/dev/xvdh" + volume_id = aws_ebs_volume.lh_aws_ssd_volume[count.index].id + instance_id = aws_instance.lh_aws_instance_worker_rke2[count.index].id + force_detach = true +} + resource "aws_lb_target_group_attachment" "lh_aws_lb_tg_443_attachment_rke2" { depends_on = [ diff --git a/test_framework/terraform/aws/rhel/user-data-scripts/provision_k3s_agent.sh.tpl b/test_framework/terraform/aws/rhel/user-data-scripts/provision_k3s_agent.sh.tpl index 5e2a5f9d0d..944ff602bc 100755 --- a/test_framework/terraform/aws/rhel/user-data-scripts/provision_k3s_agent.sh.tpl +++ b/test_framework/terraform/aws/rhel/user-data-scripts/provision_k3s_agent.sh.tpl @@ -15,7 +15,20 @@ sudo systemctl -q enable iscsid sudo systemctl start iscsid sudo systemctl disable nm-cloud-setup.service nm-cloud-setup.timer -if [ -b "/dev/xvdh" ]; then +modprobe uio +modprobe uio_pci_generic +modprobe nvme-tcp +touch /etc/modules-load.d/modules.conf +cat > /etc/modules-load.d/modules.conf < /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages +echo "vm.nr_hugepages=1024" >> /etc/sysctl.conf + +if [[ "${extra_block_device}" != true ]] && [[ -b "/dev/xvdh" ]]; then sudo mkfs.ext4 -E nodiscard /dev/xvdh sudo mkdir /var/lib/longhorn sudo mount /dev/xvdh /var/lib/longhorn diff --git a/test_framework/terraform/aws/rhel/user-data-scripts/provision_rke2_agent.sh.tpl b/test_framework/terraform/aws/rhel/user-data-scripts/provision_rke2_agent.sh.tpl index 4884b10f4e..43dd98cce7 100755 --- a/test_framework/terraform/aws/rhel/user-data-scripts/provision_rke2_agent.sh.tpl +++ b/test_framework/terraform/aws/rhel/user-data-scripts/provision_rke2_agent.sh.tpl @@ -15,7 +15,20 @@ sudo systemctl -q enable iscsid sudo systemctl start iscsid sudo systemctl disable nm-cloud-setup.service nm-cloud-setup.timer -if [ -b "/dev/xvdh" ]; then +modprobe uio +modprobe uio_pci_generic +modprobe nvme-tcp +touch /etc/modules-load.d/modules.conf +cat > /etc/modules-load.d/modules.conf < /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages +echo "vm.nr_hugepages=1024" >> /etc/sysctl.conf + +if [[ "${extra_block_device}" != true ]] && [[ -b "/dev/xvdh" ]]; then sudo mkfs.ext4 -E nodiscard /dev/xvdh sudo mkdir /var/lib/longhorn sudo mount /dev/xvdh /var/lib/longhorn diff --git a/test_framework/terraform/aws/rhel/variables.tf b/test_framework/terraform/aws/rhel/variables.tf index 6ef3c39a20..3a5197a842 100644 --- a/test_framework/terraform/aws/rhel/variables.tf +++ b/test_framework/terraform/aws/rhel/variables.tf @@ -129,3 +129,8 @@ variable "custom_ssh_public_key" { default = "" sensitive = true } + +variable "extra_block_device" { + type = bool + default = false +} diff --git a/test_framework/terraform/aws/rockylinux/data.tf b/test_framework/terraform/aws/rockylinux/data.tf index 5ed2a86f44..49c3278a20 100644 --- a/test_framework/terraform/aws/rockylinux/data.tf +++ b/test_framework/terraform/aws/rockylinux/data.tf @@ -50,6 +50,7 @@ data "template_file" "provision_k3s_agent" { selinux_mode = var.selinux_mode enable_selinux = var.selinux_mode == "permissive" ? "false" : "true" custom_ssh_public_key = var.custom_ssh_public_key + extra_block_device = var.extra_block_device } } @@ -74,5 +75,6 @@ data "template_file" "provision_rke2_agent" { rke2_version = var.k8s_distro_version selinux_mode = var.selinux_mode custom_ssh_public_key = var.custom_ssh_public_key + extra_block_device = var.extra_block_device } } diff --git a/test_framework/terraform/aws/rockylinux/k3s_instances.tf b/test_framework/terraform/aws/rockylinux/k3s_instances.tf index 17836056e1..a3b25bad30 100644 --- a/test_framework/terraform/aws/rockylinux/k3s_instances.tf +++ b/test_framework/terraform/aws/rockylinux/k3s_instances.tf @@ -78,6 +78,16 @@ resource "aws_volume_attachment" "lh_aws_hdd_volume_att_k3s" { force_detach = true } +resource "aws_volume_attachment" "lh_aws_ssd_volume_att_k3s" { + + count = var.extra_block_device && var.k8s_distro_name == "k3s" ? var.lh_aws_instance_count_worker : 0 + + device_name = "/dev/xvdh" + volume_id = aws_ebs_volume.lh_aws_ssd_volume[count.index].id + instance_id = aws_instance.lh_aws_instance_worker_k3s[count.index].id + force_detach = true +} + resource "aws_lb_target_group_attachment" "lh_aws_lb_tg_443_attachment_k3s" { depends_on = [ diff --git a/test_framework/terraform/aws/rockylinux/main.tf b/test_framework/terraform/aws/rockylinux/main.tf index e8e7be4756..dbb901d073 100644 --- a/test_framework/terraform/aws/rockylinux/main.tf +++ b/test_framework/terraform/aws/rockylinux/main.tf @@ -308,6 +308,20 @@ resource "aws_ebs_volume" "lh_aws_hdd_volume" { } } +resource "aws_ebs_volume" "lh_aws_ssd_volume" { + + count = var.extra_block_device ? var.lh_aws_instance_count_worker : 0 + + availability_zone = var.aws_availability_zone + size = var.lh_aws_instance_root_block_device_size_worker + type = "gp2" + + tags = { + Name = "lh-aws-ssd-volume-${count.index}-${random_string.random_suffix.id}" + Owner = "longhorn-infra" + } +} + # Create load balancer for rancher resource "aws_lb_target_group" "lh_aws_lb_tg_443" { diff --git a/test_framework/terraform/aws/rockylinux/rke2_instances.tf b/test_framework/terraform/aws/rockylinux/rke2_instances.tf index c910cfb8d7..261d43460a 100644 --- a/test_framework/terraform/aws/rockylinux/rke2_instances.tf +++ b/test_framework/terraform/aws/rockylinux/rke2_instances.tf @@ -78,6 +78,16 @@ resource "aws_volume_attachment" "lh_aws_hdd_volume_att_rke2" { force_detach = true } +resource "aws_volume_attachment" "lh_aws_ssd_volume_att_rke2" { + + count = var.extra_block_device && var.k8s_distro_name == "rke2" ? var.lh_aws_instance_count_worker : 0 + + device_name = "/dev/xvdh" + volume_id = aws_ebs_volume.lh_aws_ssd_volume[count.index].id + instance_id = aws_instance.lh_aws_instance_worker_rke2[count.index].id + force_detach = true +} + resource "aws_lb_target_group_attachment" "lh_aws_lb_tg_443_attachment_rke2" { depends_on = [ diff --git a/test_framework/terraform/aws/rockylinux/user-data-scripts/provision_k3s_agent.sh.tpl b/test_framework/terraform/aws/rockylinux/user-data-scripts/provision_k3s_agent.sh.tpl index 7dab190733..cca678c8d0 100755 --- a/test_framework/terraform/aws/rockylinux/user-data-scripts/provision_k3s_agent.sh.tpl +++ b/test_framework/terraform/aws/rockylinux/user-data-scripts/provision_k3s_agent.sh.tpl @@ -14,7 +14,20 @@ sudo dnf install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools sudo systemctl -q enable iscsid sudo systemctl start iscsid -if [ -b "/dev/xvdh" ]; then +modprobe uio +modprobe uio_pci_generic +modprobe nvme-tcp +touch /etc/modules-load.d/modules.conf +cat > /etc/modules-load.d/modules.conf < /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages +echo "vm.nr_hugepages=1024" >> /etc/sysctl.conf + +if [[ "${extra_block_device}" != true ]] && [[ -b "/dev/xvdh" ]]; then sudo mkfs.ext4 -E nodiscard /dev/xvdh sudo mkdir /var/lib/longhorn sudo mount /dev/xvdh /var/lib/longhorn diff --git a/test_framework/terraform/aws/rockylinux/user-data-scripts/provision_rke2_agent.sh.tpl b/test_framework/terraform/aws/rockylinux/user-data-scripts/provision_rke2_agent.sh.tpl index e2a67e95f4..fd03f60b26 100755 --- a/test_framework/terraform/aws/rockylinux/user-data-scripts/provision_rke2_agent.sh.tpl +++ b/test_framework/terraform/aws/rockylinux/user-data-scripts/provision_rke2_agent.sh.tpl @@ -14,7 +14,20 @@ sudo dnf install -y iscsi-initiator-utils nfs-utils nfs4-acl-tools jq nmap-ncat sudo systemctl -q enable iscsid sudo systemctl start iscsid -if [ -b "/dev/xvdh" ]; then +modprobe uio +modprobe uio_pci_generic +modprobe nvme-tcp +touch /etc/modules-load.d/modules.conf +cat > /etc/modules-load.d/modules.conf < /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages +echo "vm.nr_hugepages=1024" >> /etc/sysctl.conf + +if [[ "${extra_block_device}" != true ]] && [[ -b "/dev/xvdh" ]]; then sudo mkfs.ext4 -E nodiscard /dev/xvdh sudo mkdir /var/lib/longhorn sudo mount /dev/xvdh /var/lib/longhorn diff --git a/test_framework/terraform/aws/rockylinux/variables.tf b/test_framework/terraform/aws/rockylinux/variables.tf index cd60eae666..eebb302667 100644 --- a/test_framework/terraform/aws/rockylinux/variables.tf +++ b/test_framework/terraform/aws/rockylinux/variables.tf @@ -128,4 +128,9 @@ variable "custom_ssh_public_key" { type = string default = "" sensitive = true -} \ No newline at end of file +} + +variable "extra_block_device" { + type = bool + default = false +} diff --git a/test_framework/terraform/aws/sle-micro/data.tf b/test_framework/terraform/aws/sle-micro/data.tf index 472d0ca177..a0ddd6788a 100644 --- a/test_framework/terraform/aws/sle-micro/data.tf +++ b/test_framework/terraform/aws/sle-micro/data.tf @@ -32,6 +32,7 @@ data "template_file" "provision_k3s_agent" { k3s_cluster_secret = random_password.cluster_secret.result k3s_version = var.k8s_distro_version custom_ssh_public_key = var.custom_ssh_public_key + extra_block_device = var.extra_block_device } } @@ -54,5 +55,6 @@ data "template_file" "provision_rke2_agent" { rke2_cluster_secret = random_password.cluster_secret.result rke2_version = var.k8s_distro_version custom_ssh_public_key = var.custom_ssh_public_key + extra_block_device = var.extra_block_device } } diff --git a/test_framework/terraform/aws/sle-micro/k3s_instaces.tf b/test_framework/terraform/aws/sle-micro/k3s_instaces.tf index c233186a15..61592c43b2 100644 --- a/test_framework/terraform/aws/sle-micro/k3s_instaces.tf +++ b/test_framework/terraform/aws/sle-micro/k3s_instaces.tf @@ -75,6 +75,16 @@ resource "aws_volume_attachment" "lh_aws_hdd_volume_att_k3s" { force_detach = true } +resource "aws_volume_attachment" "lh_aws_ssd_volume_att_k3s" { + + count = var.extra_block_device && var.k8s_distro_name == "k3s" ? var.lh_aws_instance_count_worker : 0 + + device_name = "/dev/xvdh" + volume_id = aws_ebs_volume.lh_aws_ssd_volume[count.index].id + instance_id = aws_instance.lh_aws_instance_worker_k3s[count.index].id + force_detach = true +} + resource "aws_lb_target_group_attachment" "lh_aws_lb_tg_443_attachment_k3s" { depends_on = [ diff --git a/test_framework/terraform/aws/sle-micro/main.tf b/test_framework/terraform/aws/sle-micro/main.tf index 7134f62a2c..c9755d91b5 100644 --- a/test_framework/terraform/aws/sle-micro/main.tf +++ b/test_framework/terraform/aws/sle-micro/main.tf @@ -203,6 +203,20 @@ resource "aws_ebs_volume" "lh_aws_hdd_volume" { } } +resource "aws_ebs_volume" "lh_aws_ssd_volume" { + + count = var.extra_block_device ? var.lh_aws_instance_count_worker : 0 + + availability_zone = var.aws_availability_zone + size = var.lh_aws_instance_root_block_device_size_worker + type = "gp2" + + tags = { + Name = "lh-aws-ssd-volume-${count.index}-${random_string.random_suffix.id}" + Owner = "longhorn-infra" + } +} + # Create load balancer for rancher resource "aws_lb_target_group" "lh_aws_lb_tg_443" { diff --git a/test_framework/terraform/aws/sle-micro/rke2_instances.tf b/test_framework/terraform/aws/sle-micro/rke2_instances.tf index 07d2957ee6..27c71e0205 100644 --- a/test_framework/terraform/aws/sle-micro/rke2_instances.tf +++ b/test_framework/terraform/aws/sle-micro/rke2_instances.tf @@ -75,6 +75,16 @@ resource "aws_volume_attachment" "lh_aws_hdd_volume_att_rke2" { force_detach = true } +resource "aws_volume_attachment" "lh_aws_ssd_volume_att_rke2" { + + count = var.extra_block_device && var.k8s_distro_name == "rke2" ? var.lh_aws_instance_count_worker : 0 + + device_name = "/dev/xvdh" + volume_id = aws_ebs_volume.lh_aws_ssd_volume[count.index].id + instance_id = aws_instance.lh_aws_instance_worker_rke2[count.index].id + force_detach = true +} + resource "aws_lb_target_group_attachment" "lh_aws_lb_tg_443_attachment_rke2" { depends_on = [ diff --git a/test_framework/terraform/aws/sle-micro/user-data-scripts/provision_k3s_agent.sh.tpl b/test_framework/terraform/aws/sle-micro/user-data-scripts/provision_k3s_agent.sh.tpl index fb81b0214d..9d9b1f1c3a 100755 --- a/test_framework/terraform/aws/sle-micro/user-data-scripts/provision_k3s_agent.sh.tpl +++ b/test_framework/terraform/aws/sle-micro/user-data-scripts/provision_k3s_agent.sh.tpl @@ -1,21 +1,38 @@ #!/bin/bash -if [ -b "/dev/nvme1n1" ]; then - sudo mkfs.ext4 -E nodiscard /dev/nvme1n1 - sudo mkdir /opt/sda1 - sudo mount /dev/nvme1n1 /opt/sda1 - - sudo mkdir /opt/sda1/local - sudo mkdir /opt/local-path-provisioner - sudo mount --bind /opt/sda1/local /opt/local-path-provisioner - - sudo mkdir /opt/sda1/longhorn - sudo mkdir /var/lib/longhorn - sudo mount --bind /opt/sda1/longhorn /var/lib/longhorn -elif [ -b "/dev/xvdh" ]; then - sudo mkfs.ext4 -E nodiscard /dev/xvdh - sudo mkdir /var/lib/longhorn - sudo mount /dev/xvdh /var/lib/longhorn +set -e + +sudo modprobe uio +sudo modprobe uio_pci_generic +sudo modprobe nvme-tcp +sudo touch /etc/modules-load.d/modules.conf +sudo sh -c "cat > /etc/modules-load.d/modules.conf < /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" +sudo sh -c "echo "vm.nr_hugepages=1024" >> /etc/sysctl.conf" + +if [[ "${extra_block_device}" != true ]]; then + if [[ -b "/dev/nvme1n1" ]]; then + mkfs.ext4 -E nodiscard /dev/nvme1n1 + mkdir /mnt/sda1 + mount /dev/nvme1n1 /mnt/sda1 + + mkdir /mnt/sda1/local + mkdir /opt/local-path-provisioner + mount --bind /mnt/sda1/local /opt/local-path-provisioner + + mkdir /mnt/sda1/longhorn + mkdir /var/lib/longhorn + mount --bind /mnt/sda1/longhorn /var/lib/longhorn + elif [ -b "/dev/xvdh" ]; then + mkfs.ext4 -E nodiscard /dev/xvdh + mkdir /var/lib/longhorn + mount /dev/xvdh /var/lib/longhorn + fi fi curl -sfL https://get.k3s.io | sudo INSTALL_K3S_EXEC="agent --token ${k3s_cluster_secret}" K3S_URL="${k3s_server_url}" INSTALL_K3S_VERSION="${k3s_version}" sh - diff --git a/test_framework/terraform/aws/sle-micro/user-data-scripts/provision_rke2_agent.sh.tpl b/test_framework/terraform/aws/sle-micro/user-data-scripts/provision_rke2_agent.sh.tpl index 6efbaab929..bf1fbe9292 100755 --- a/test_framework/terraform/aws/sle-micro/user-data-scripts/provision_rke2_agent.sh.tpl +++ b/test_framework/terraform/aws/sle-micro/user-data-scripts/provision_rke2_agent.sh.tpl @@ -1,21 +1,36 @@ #!/bin/bash -if [ -b "/dev/nvme1n1" ]; then - sudo mkfs.ext4 -E nodiscard /dev/nvme1n1 - sudo mkdir /opt/sda1 - sudo mount /dev/nvme1n1 /opt/sda1 - - sudo mkdir /opt/sda1/local - sudo mkdir /opt/local-path-provisioner - sudo mount --bind /opt/sda1/local /opt/local-path-provisioner - - sudo mkdir /opt/sda1/longhorn - sudo mkdir /var/lib/longhorn - sudo mount --bind /opt/sda1/longhorn /var/lib/longhorn -elif [ -b "/dev/xvdh" ]; then - sudo mkfs.ext4 -E nodiscard /dev/xvdh - sudo mkdir /var/lib/longhorn - sudo mount /dev/xvdh /var/lib/longhorn +sudo modprobe uio +sudo modprobe uio_pci_generic +sudo modprobe nvme-tcp +sudo touch /etc/modules-load.d/modules.conf +sudo sh -c "cat > /etc/modules-load.d/modules.conf < /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" +sudo sh -c "echo "vm.nr_hugepages=1024" >> /etc/sysctl.conf" + +if [[ "${extra_block_device}" != true ]]; then + if [[ -b "/dev/nvme1n1" ]]; then + mkfs.ext4 -E nodiscard /dev/nvme1n1 + mkdir /mnt/sda1 + mount /dev/nvme1n1 /mnt/sda1 + + mkdir /mnt/sda1/local + mkdir /opt/local-path-provisioner + mount --bind /mnt/sda1/local /opt/local-path-provisioner + + mkdir /mnt/sda1/longhorn + mkdir /var/lib/longhorn + mount --bind /mnt/sda1/longhorn /var/lib/longhorn + elif [ -b "/dev/xvdh" ]; then + mkfs.ext4 -E nodiscard /dev/xvdh + mkdir /var/lib/longhorn + mount /dev/xvdh /var/lib/longhorn + fi fi RKE_SERVER_IP=`echo ${rke2_server_url} | sed 's#https://##' | awk -F ":" '{print $1}'` diff --git a/test_framework/terraform/aws/sle-micro/variables.tf b/test_framework/terraform/aws/sle-micro/variables.tf index 7e1f10f5f9..3cd29a5e61 100644 --- a/test_framework/terraform/aws/sle-micro/variables.tf +++ b/test_framework/terraform/aws/sle-micro/variables.tf @@ -127,4 +127,9 @@ variable "custom_ssh_public_key" { type = string default = "" sensitive = true -} \ No newline at end of file +} + +variable "extra_block_device" { + type = bool + default = false +} diff --git a/test_framework/terraform/aws/sles/data.tf b/test_framework/terraform/aws/sles/data.tf index b8964a56dd..c11395874b 100644 --- a/test_framework/terraform/aws/sles/data.tf +++ b/test_framework/terraform/aws/sles/data.tf @@ -29,6 +29,7 @@ data "template_file" "provision_k3s_agent" { k3s_cluster_secret = random_password.cluster_secret.result k3s_version = var.k8s_distro_version custom_ssh_public_key = var.custom_ssh_public_key + extra_block_device = var.extra_block_device } } @@ -53,5 +54,6 @@ data "template_file" "provision_rke2_agent" { rke2_version = var.k8s_distro_version cis_hardening = var.cis_hardening custom_ssh_public_key = var.custom_ssh_public_key + extra_block_device = var.extra_block_device } } diff --git a/test_framework/terraform/aws/sles/k3s_instances.tf b/test_framework/terraform/aws/sles/k3s_instances.tf index f59f2e65ba..6ed5ef3ca4 100644 --- a/test_framework/terraform/aws/sles/k3s_instances.tf +++ b/test_framework/terraform/aws/sles/k3s_instances.tf @@ -77,6 +77,16 @@ resource "aws_volume_attachment" "lh_aws_hdd_volume_att_k3s" { force_detach = true } +resource "aws_volume_attachment" "lh_aws_ssd_volume_att_k3s" { + + count = var.extra_block_device && var.k8s_distro_name == "k3s" ? var.lh_aws_instance_count_worker : 0 + + device_name = "/dev/xvdh" + volume_id = aws_ebs_volume.lh_aws_ssd_volume[count.index].id + instance_id = aws_instance.lh_aws_instance_worker_k3s[count.index].id + force_detach = true +} + resource "aws_lb_target_group_attachment" "lh_aws_lb_tg_443_attachment_k3s" { depends_on = [ diff --git a/test_framework/terraform/aws/sles/main.tf b/test_framework/terraform/aws/sles/main.tf index 0e78f0a6c6..1267f91df4 100644 --- a/test_framework/terraform/aws/sles/main.tf +++ b/test_framework/terraform/aws/sles/main.tf @@ -315,6 +315,20 @@ resource "aws_ebs_volume" "lh_aws_hdd_volume" { } } +resource "aws_ebs_volume" "lh_aws_ssd_volume" { + + count = var.extra_block_device ? var.lh_aws_instance_count_worker : 0 + + availability_zone = var.aws_availability_zone + size = var.lh_aws_instance_root_block_device_size_worker + type = "gp2" + + tags = { + Name = "lh-aws-ssd-volume-${count.index}-${random_string.random_suffix.id}" + Owner = var.resources_owner + } +} + # Create load balancer for rancher resource "aws_lb_target_group" "lh_aws_lb_tg_443" { diff --git a/test_framework/terraform/aws/sles/rke2_instances.tf b/test_framework/terraform/aws/sles/rke2_instances.tf index 3ed6cade16..fa64c2bf06 100644 --- a/test_framework/terraform/aws/sles/rke2_instances.tf +++ b/test_framework/terraform/aws/sles/rke2_instances.tf @@ -77,6 +77,16 @@ resource "aws_volume_attachment" "lh_aws_hdd_volume_att_rke2" { force_detach = true } +resource "aws_volume_attachment" "lh_aws_ssd_volume_att_rke2" { + + count = var.extra_block_device && var.k8s_distro_name == "rke2" ? var.lh_aws_instance_count_worker : 0 + + device_name = "/dev/xvdh" + volume_id = aws_ebs_volume.lh_aws_ssd_volume[count.index].id + instance_id = aws_instance.lh_aws_instance_worker_rke2[count.index].id + force_detach = true +} + resource "aws_lb_target_group_attachment" "lh_aws_lb_tg_443_attachment_rke2" { depends_on = [ diff --git a/test_framework/terraform/aws/sles/user-data-scripts/provision_k3s_agent.sh.tpl b/test_framework/terraform/aws/sles/user-data-scripts/provision_k3s_agent.sh.tpl index c799b47a07..4ea219dd30 100755 --- a/test_framework/terraform/aws/sles/user-data-scripts/provision_k3s_agent.sh.tpl +++ b/test_framework/terraform/aws/sles/user-data-scripts/provision_k3s_agent.sh.tpl @@ -9,22 +9,37 @@ sudo zypper install -y open-iscsi nfs-client sudo systemctl -q enable iscsid sudo systemctl start iscsid -if [ -b "/dev/nvme1n1" ]; then - mkfs.ext4 -E nodiscard /dev/nvme1n1 - mkdir /mnt/sda1 - mount /dev/nvme1n1 /mnt/sda1 - - mkdir /mnt/sda1/local - mkdir /opt/local-path-provisioner - mount --bind /mnt/sda1/local /opt/local-path-provisioner - - mkdir /mnt/sda1/longhorn - mkdir /var/lib/longhorn - mount --bind /mnt/sda1/longhorn /var/lib/longhorn -elif [ -b "/dev/xvdh" ]; then - mkfs.ext4 -E nodiscard /dev/xvdh - mkdir /var/lib/longhorn - mount /dev/xvdh /var/lib/longhorn +modprobe uio +modprobe uio_pci_generic +modprobe nvme-tcp +touch /etc/modules-load.d/modules.conf +cat > /etc/modules-load.d/modules.conf < /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages +echo "vm.nr_hugepages=1024" >> /etc/sysctl.conf + +if [[ "${extra_block_device}" != true ]]; then + if [[ -b "/dev/nvme1n1" ]]; then + mkfs.ext4 -E nodiscard /dev/nvme1n1 + mkdir /mnt/sda1 + mount /dev/nvme1n1 /mnt/sda1 + + mkdir /mnt/sda1/local + mkdir /opt/local-path-provisioner + mount --bind /mnt/sda1/local /opt/local-path-provisioner + + mkdir /mnt/sda1/longhorn + mkdir /var/lib/longhorn + mount --bind /mnt/sda1/longhorn /var/lib/longhorn + elif [ -b "/dev/xvdh" ]; then + mkfs.ext4 -E nodiscard /dev/xvdh + mkdir /var/lib/longhorn + mount /dev/xvdh /var/lib/longhorn + fi fi # TODO: It looks like "set -e" will break the intended functionality of the remaining code. Consider a refactor. diff --git a/test_framework/terraform/aws/sles/user-data-scripts/provision_rke2_agent.sh.tpl b/test_framework/terraform/aws/sles/user-data-scripts/provision_rke2_agent.sh.tpl index 04a9e1959c..b9bf110c57 100755 --- a/test_framework/terraform/aws/sles/user-data-scripts/provision_rke2_agent.sh.tpl +++ b/test_framework/terraform/aws/sles/user-data-scripts/provision_rke2_agent.sh.tpl @@ -9,22 +9,37 @@ sudo zypper install -y open-iscsi nfs-client sudo systemctl -q enable iscsid sudo systemctl start iscsid -if [ -b "/dev/nvme1n1" ]; then - mkfs.ext4 -E nodiscard /dev/nvme1n1 - mkdir /mnt/sda1 - mount /dev/nvme1n1 /mnt/sda1 - - mkdir /mnt/sda1/local - mkdir /opt/local-path-provisioner - mount --bind /mnt/sda1/local /opt/local-path-provisioner - - mkdir /mnt/sda1/longhorn - mkdir /var/lib/longhorn - mount --bind /mnt/sda1/longhorn /var/lib/longhorn -elif [ -b "/dev/xvdh" ]; then - mkfs.ext4 -E nodiscard /dev/xvdh - mkdir /var/lib/longhorn - mount /dev/xvdh /var/lib/longhorn +modprobe uio +modprobe uio_pci_generic +modprobe nvme-tcp +touch /etc/modules-load.d/modules.conf +cat > /etc/modules-load.d/modules.conf < /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages +echo "vm.nr_hugepages=1024" >> /etc/sysctl.conf + +if [[ "${extra_block_device}" != true ]]; then + if [[ -b "/dev/nvme1n1" ]]; then + mkfs.ext4 -E nodiscard /dev/nvme1n1 + mkdir /mnt/sda1 + mount /dev/nvme1n1 /mnt/sda1 + + mkdir /mnt/sda1/local + mkdir /opt/local-path-provisioner + mount --bind /mnt/sda1/local /opt/local-path-provisioner + + mkdir /mnt/sda1/longhorn + mkdir /var/lib/longhorn + mount --bind /mnt/sda1/longhorn /var/lib/longhorn + elif [ -b "/dev/xvdh" ]; then + mkfs.ext4 -E nodiscard /dev/xvdh + mkdir /var/lib/longhorn + mount /dev/xvdh /var/lib/longhorn + fi fi RKE_SERVER_IP=`echo ${rke2_server_url} | sed 's#https://##' | awk -F ":" '{print $1}'` diff --git a/test_framework/terraform/aws/sles/variables.tf b/test_framework/terraform/aws/sles/variables.tf index 64b8bebdf4..25380c24ea 100644 --- a/test_framework/terraform/aws/sles/variables.tf +++ b/test_framework/terraform/aws/sles/variables.tf @@ -99,7 +99,7 @@ variable "k8s_distro_name" { variable "k8s_distro_version" { type = string - default = "v1.27.1+k3s1" + default = "v1.28.4+k3s1" description = <<-EOT kubernetes version that will be deployed rke: (default: v1.22.5-rancher1-1) @@ -129,7 +129,12 @@ variable "resources_owner" { } variable "custom_ssh_public_key" { - type = string - default = "" + type = string + default = "" sensitive = true -} \ No newline at end of file +} + +variable "extra_block_device" { + type = bool + default = false +} diff --git a/test_framework/terraform/aws/ubuntu/data.tf b/test_framework/terraform/aws/ubuntu/data.tf index 9edd0495f2..a122b1ffac 100644 --- a/test_framework/terraform/aws/ubuntu/data.tf +++ b/test_framework/terraform/aws/ubuntu/data.tf @@ -28,6 +28,7 @@ data "template_file" "provision_k3s_agent" { k3s_cluster_secret = random_password.cluster_secret.result k3s_version = var.k8s_distro_version custom_ssh_public_key = var.custom_ssh_public_key + extra_block_device = var.extra_block_device } } @@ -51,6 +52,7 @@ data "template_file" "provision_rke2_agent" { rke2_cluster_secret = random_password.cluster_secret.result rke2_version = var.k8s_distro_version custom_ssh_public_key = var.custom_ssh_public_key + extra_block_device = var.extra_block_device } } diff --git a/test_framework/terraform/aws/ubuntu/k3s_instaces.tf b/test_framework/terraform/aws/ubuntu/k3s_instaces.tf index b511f66173..d38c799794 100644 --- a/test_framework/terraform/aws/ubuntu/k3s_instaces.tf +++ b/test_framework/terraform/aws/ubuntu/k3s_instaces.tf @@ -41,12 +41,12 @@ resource "aws_instance" "lh_aws_instance_worker_k3s" { ] count = var.k8s_distro_name == "k3s" ? var.lh_aws_instance_count_worker : 0 - + availability_zone = var.aws_availability_zone - + ami = data.aws_ami.aws_ami_ubuntu.id instance_type = var.lh_aws_instance_type_worker - + subnet_id = aws_subnet.lh_aws_private_subnet.id vpc_security_group_ids = [ aws_security_group.lh_aws_secgrp_worker.id @@ -78,6 +78,16 @@ resource "aws_volume_attachment" "lh_aws_hdd_volume_att_k3s" { force_detach = true } +resource "aws_volume_attachment" "lh_aws_ssd_volume_att_k3s" { + + count = var.extra_block_device && var.k8s_distro_name == "k3s" ? var.lh_aws_instance_count_worker : 0 + + device_name = "/dev/xvdh" + volume_id = aws_ebs_volume.lh_aws_ssd_volume[count.index].id + instance_id = aws_instance.lh_aws_instance_worker_k3s[count.index].id + force_detach = true +} + resource "aws_lb_target_group_attachment" "lh_aws_lb_tg_443_attachment_k3s" { depends_on = [ diff --git a/test_framework/terraform/aws/ubuntu/main.tf b/test_framework/terraform/aws/ubuntu/main.tf index 956411de0f..1d38dc22b1 100644 --- a/test_framework/terraform/aws/ubuntu/main.tf +++ b/test_framework/terraform/aws/ubuntu/main.tf @@ -309,6 +309,20 @@ resource "aws_ebs_volume" "lh_aws_hdd_volume" { } } +resource "aws_ebs_volume" "lh_aws_ssd_volume" { + + count = var.extra_block_device ? var.lh_aws_instance_count_worker : 0 + + availability_zone = var.aws_availability_zone + size = var.lh_aws_instance_root_block_device_size_worker + type = "gp2" + + tags = { + Name = "lh-aws-ssd-volume-${count.index}-${random_string.random_suffix.id}" + Owner = "longhorn-infra" + } +} + # Create load balancer for rancher resource "aws_lb_target_group" "lh_aws_lb_tg_443" { diff --git a/test_framework/terraform/aws/ubuntu/rke2_instances.tf b/test_framework/terraform/aws/ubuntu/rke2_instances.tf index 78db0b0844..1844a73ed9 100644 --- a/test_framework/terraform/aws/ubuntu/rke2_instances.tf +++ b/test_framework/terraform/aws/ubuntu/rke2_instances.tf @@ -41,12 +41,12 @@ resource "aws_instance" "lh_aws_instance_worker_rke2" { ] count = var.k8s_distro_name == "rke2" ? var.lh_aws_instance_count_worker : 0 - + availability_zone = var.aws_availability_zone - + ami = data.aws_ami.aws_ami_ubuntu.id instance_type = var.lh_aws_instance_type_worker - + subnet_id = aws_subnet.lh_aws_private_subnet.id vpc_security_group_ids = [ aws_security_group.lh_aws_secgrp_worker.id @@ -78,6 +78,16 @@ resource "aws_volume_attachment" "lh_aws_hdd_volume_att_rke2" { force_detach = true } +resource "aws_volume_attachment" "lh_aws_ssd_volume_att_rke2" { + + count = var.extra_block_device && var.k8s_distro_name == "rke2" ? var.lh_aws_instance_count_worker : 0 + + device_name = "/dev/xvdh" + volume_id = aws_ebs_volume.lh_aws_ssd_volume[count.index].id + instance_id = aws_instance.lh_aws_instance_worker_rke2[count.index].id + force_detach = true +} + resource "aws_lb_target_group_attachment" "lh_aws_lb_tg_443_attachment_rke2" { depends_on = [ @@ -114,7 +124,7 @@ resource "null_resource" "rsync_kubeconfig_file_rke2" { ] provisioner "remote-exec" { - inline = ["until([ -f /etc/rancher/rke2/rke2.yaml ] && [ `sudo KUBECONFIG=/etc/rancher/rke2/rke2.yaml /var/lib/rancher/rke2/bin/kubectl get node -o jsonpath='{.items[*].status.conditions}' | jq '.[] | select(.type == \"Ready\").status' | grep -ci true` -eq $((${var.lh_aws_instance_count_controlplane} + ${var.lh_aws_instance_count_worker})) ]); do echo \"waiting for rke2 cluster nodes to be running\"; sleep 2; done"] + inline = ["until([ -f /etc/rancher/rke2/rke2.yaml ] && [ `KUBECONFIG=/etc/rancher/rke2/rke2.yaml /var/lib/rancher/rke2/bin/kubectl get node -o jsonpath='{.items[*].status.conditions}' | jq '.[] | select(.type == \"Ready\").status' | grep -ci true` -eq $((${var.lh_aws_instance_count_controlplane} + ${var.lh_aws_instance_count_worker})) ]); do echo \"waiting for rke2 cluster nodes to be running\"; sleep 2; done"] connection { diff --git a/test_framework/terraform/aws/ubuntu/user-data-scripts/provision_k3s_agent.sh.tpl b/test_framework/terraform/aws/ubuntu/user-data-scripts/provision_k3s_agent.sh.tpl index c7b825fc63..78ddb6d7d9 100755 --- a/test_framework/terraform/aws/ubuntu/user-data-scripts/provision_k3s_agent.sh.tpl +++ b/test_framework/terraform/aws/ubuntu/user-data-scripts/provision_k3s_agent.sh.tpl @@ -1,9 +1,22 @@ #!/bin/bash apt-get update -apt-get install -y nfs-common +apt-get install -y nfs-common linux-modules-extra-`uname -r` -if [ -b "/dev/xvdh" ]; then +modprobe uio +modprobe uio_pci_generic +modprobe nvme-tcp +touch /etc/modules-load.d/modules.conf +cat > /etc/modules-load.d/modules.conf < /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages +echo "vm.nr_hugepages=1024" >> /etc/sysctl.conf + +if [[ "${extra_block_device}" != true ]] && [[ -b "/dev/xvdh" ]]; then mkfs.ext4 -E nodiscard /dev/xvdh mkdir /var/lib/longhorn mount /dev/xvdh /var/lib/longhorn diff --git a/test_framework/terraform/aws/ubuntu/user-data-scripts/provision_rke2_agent.sh.tpl b/test_framework/terraform/aws/ubuntu/user-data-scripts/provision_rke2_agent.sh.tpl index b2d58b4ed9..c72851123f 100755 --- a/test_framework/terraform/aws/ubuntu/user-data-scripts/provision_rke2_agent.sh.tpl +++ b/test_framework/terraform/aws/ubuntu/user-data-scripts/provision_rke2_agent.sh.tpl @@ -1,9 +1,22 @@ #!/bin/bash apt-get update -apt-get install -y nfs-common +apt-get install -y nfs-common linux-modules-extra-`uname -r` + +modprobe uio +modprobe uio_pci_generic +modprobe nvme-tcp +touch /etc/modules-load.d/modules.conf +cat > /etc/modules-load.d/modules.conf < /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages +echo "vm.nr_hugepages=1024" >> /etc/sysctl.conf -if [ -b "/dev/xvdh" ]; then +if [[ "${extra_block_device}" != true ]] && [[ -b "/dev/xvdh" ]]; then mkfs.ext4 -E nodiscard /dev/xvdh mkdir /var/lib/longhorn mount /dev/xvdh /var/lib/longhorn diff --git a/test_framework/terraform/aws/ubuntu/variables.tf b/test_framework/terraform/aws/ubuntu/variables.tf index e63b7afb9c..ee105e0d6f 100644 --- a/test_framework/terraform/aws/ubuntu/variables.tf +++ b/test_framework/terraform/aws/ubuntu/variables.tf @@ -122,4 +122,9 @@ variable "custom_ssh_public_key" { type = string default = "" sensitive = true -} \ No newline at end of file +} + +variable "extra_block_device" { + type = bool + default = false +}