test: setup out-of-cluster test execution in pipeline

Signed-off-by: Yang Chiu <[email protected]>

e2e/Dockerfile

@@ -24,6 +24,4 @@ WORKDIR /e2e
 
 RUN pip install -r requirements.txt
 
-ADD pipelines/utilities/litmus/ ./litmus
-
 ENTRYPOINT ["./run.sh"]

e2e/libs/node/node.py

@@ -38,7 +38,7 @@ def reboot_node(self, running_on_node_name, reboot_node_name, shut_down_time_in_
             mapping = yaml.safe_load(f)
             reboot_node_id = mapping[reboot_node_name]
 
-        filepath = './litmus/reboot-node.yaml'
+        filepath = './templates/litmus/reboot-node.yaml'
         with open(filepath, 'r') as f:
             data = yaml.safe_load(f)
             data['spec']['components']['runner']['nodeSelector']['kubernetes.io/hostname'] = running_on_node_name

e2e/requirements.txt

@@ -3,10 +3,6 @@ argcomplete==1.10.0
 directio==1.2
 flake8
 kubernetes==27.2.0
-pytest==5.3.1
-pytest-repeat==0.9.1
-pytest-order==1.0.1
 requests==2.31.0
-six==1.12.0
 boto3==1.26.86
-pyyaml==5.4.1
+pyyaml==6.0.1

pipelines/e2e/Dockerfile.setup

@@ -25,7 +25,7 @@ RUN wget -q https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_V
     wget -q "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" && \
     mv yq_linux_amd64 /usr/local/bin/yq && \
     chmod +x /usr/local/bin/yq && \
-    apk add openssl openssh-client ca-certificates git rsync bash curl jq python3 py3-pip && \
+    apk add openssl openssh-client ca-certificates git rsync bash curl jq python3 py3-pip gcc python3-dev libc-dev && \
     ssh-keygen -t rsa -b 4096 -N "" -f ~/.ssh/id_rsa && \
     curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 && \
     chmod 700 get_helm.sh && \

pipelines/e2e/Jenkinsfile

@@ -102,14 +102,18 @@ node {
                                        --env TF_VAR_os_distro_version=${DISTRO_VERSION} \
                                        --env TF_VAR_do_token=${env.TF_VAR_do_token} \
                                        --env TF_VAR_lh_aws_access_key=${AWS_ACCESS_KEY} \
+                                       --env TF_VAR_lh_aws_secret_key=${AWS_SECRET_KEY} \
+                                       --env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY} \
+                                       --env AWS_SECRET_ACCESS_KEY=${AWS_SECRET_KEY} \
+                                       --env AWS_DEFAULT_REGION=${AWS_REGION} \
                                        --env TF_VAR_lh_aws_instance_name_controlplane="${JOB_BASE_NAME}-ctrl" \
                                        --env TF_VAR_lh_aws_instance_name_worker="${JOB_BASE_NAME}-wrk" \
                                        --env TF_VAR_lh_aws_instance_type_controlplane=${CONTROLPLANE_INSTANCE_TYPE} \
                                        --env TF_VAR_lh_aws_instance_type_worker=${WORKER_INSTANCE_TYPE}\
-                                       --env TF_VAR_lh_aws_secret_key=${AWS_SECRET_KEY} \
                                        --env TF_VAR_selinux_mode=${SELINUX_MODE} \
                                        --env TF_VAR_registration_code=${REGISTRATION_CODE} \
                                        --env TF_VAR_cis_hardening=${CIS_HARDENING} \
+                                       --env TF_VAR_resources_owner=longhorn-long-running \
                                        ${imageName}
                 """
             }

pipelines/e2e/scripts/longhorn-setup.sh

@@ -9,6 +9,7 @@ source pipelines/utilities/create_aws_secret.sh
 source pipelines/utilities/install_backupstores.sh
 source pipelines/utilities/create_longhorn_namespace.sh
 source pipelines/utilities/longhorn_manifest.sh
+source pipelines/utilities/longhorn_ui.sh
 source pipelines/utilities/install_litmus.sh
 source pipelines/utilities/run_longhorn_e2e_test.sh
 
@@ -48,7 +49,15 @@ main(){
 
   generate_longhorn_yaml_manifest
   install_longhorn_by_manifest
-  run_longhorn_e2e_test
+
+  setup_longhorn_ui_nodeport
+  export_longhorn_ui_url
+
+  if [[ -n "${LONGHORN_TESTS_CUSTOM_IMAGE}" ]]; then
+    run_longhorn_e2e_test
+  else
+    run_longhorn_e2e_test_out_of_cluster
+  fi
 }
 
 main

pipelines/utilities/kubeconfig.sh

@@ -3,13 +3,13 @@ set_kubeconfig(){
   # but only k3s supports arm64
   if [[ "${TF_VAR_arch}" == "amd64" ]] ; then
     if [[ "${TF_VAR_k8s_distro_name}" == "rke" ]]; then
-        export KUBECONFIG="test_framework/kube_config_rke.yml"
+        export KUBECONFIG="${PWD}/test_framework/kube_config_rke.yml"
     elif [[ "${TF_VAR_k8s_distro_name}" == "rke2" ]]; then
-        export KUBECONFIG="test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO}/rke2.yaml"
+        export KUBECONFIG="${PWD}/test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO}/rke2.yaml"
     else
-        export KUBECONFIG="test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO}/k3s.yaml"
+        export KUBECONFIG="${PWD}/test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO}/k3s.yaml"
     fi
   elif [[ "${TF_VAR_arch}" == "arm64"  ]]; then
-    export KUBECONFIG="test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO}/k3s.yaml"
+    export KUBECONFIG="${PWD}/test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO}/k3s.yaml"
   fi
 }

pipelines/utilities/longhorn_ui.sh

@@ -0,0 +1,7 @@
+setup_longhorn_ui_nodeport(){
+  kubectl expose --type=NodePort deployment longhorn-ui -n longhorn-system --port 8000 --name longhorn-ui-nodeport --overrides '{ "apiVersion": "v1","spec":{"ports": [{"port":8000,"protocol":"TCP","targetPort":8000,"nodePort":30000}]}}'
+}
+
+export_longhorn_ui_url(){
+  export LONGHORN_CLIENT_URL="http://$(cat /tmp/controlplane_public_ip):30000"
+}

pipelines/utilities/run_longhorn_e2e_test.sh

@@ -63,3 +63,16 @@ run_longhorn_e2e_test(){
   kubectl cp ${LONGHORN_TEST_POD_NAME}:/tmp/test-report/output.xml "output.xml" -c longhorn-test-report
   kubectl cp ${LONGHORN_TEST_POD_NAME}:/tmp/test-report/report.html "report.html" -c longhorn-test-report
 }
+
+run_longhorn_e2e_test_out_of_cluster(){
+  cd e2e
+  pip install -r requirements.txt
+
+  eval "ROBOT_COMMAND_ARGS=($PYTEST_CUSTOM_OPTIONS)"
+
+  ./run.sh "${ROBOT_COMMAND_ARGS[@]}"
+
+  cp /tmp/test-report/log.html "${WORKSPACE}/log.html"
+  cp /tmp/test-report/output.xml "${WORKSPACE}/output.xml"
+  cp /tmp/test-report/report.html "${WORKSPACE}/report.html"
+}

pipelines/utilities/terraform_setup.sh

@@ -24,4 +24,6 @@ if [[ "${TF_VAR_k8s_distro_name}" == "k3s" ]]; then
   terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} output -raw instance_mapping | jq 'map({(.name | split(".")[0]): .id}) | add' | jq -s add > /tmp/instance_mapping
 fi
 
+terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} output -raw controlplane_public_ip > /tmp/controlplane_public_ip
+
 exit $?

test_framework/terraform/aws/sles/k3s_instances.tf

@@ -28,7 +28,7 @@ resource "aws_instance" "lh_aws_instance_controlplane_k3s" {
   tags = {
     Name = "${var.lh_aws_instance_name_controlplane}-${count.index}-${random_string.random_suffix.id}"
     DoNotDelete = "true"
-    Owner = "longhorn-infra"
+    Owner = var.resources_owner
   }
 }
 
@@ -63,7 +63,7 @@ resource "aws_instance" "lh_aws_instance_worker_k3s" {
   tags = {
     Name = "${var.lh_aws_instance_name_worker}-${count.index}-${random_string.random_suffix.id}"
     DoNotDelete = "true"
-    Owner = "longhorn-infra"
+    Owner = var.resources_owner
   }
 }
 

test_framework/terraform/aws/sles/main.tf

@@ -33,7 +33,7 @@ resource "aws_vpc" "lh_aws_vpc" {
 
   tags = {
     Name = "${var.lh_aws_vpc_name}-${random_string.random_suffix.id}"
-    Owner = "longhorn-infra"
+    Owner = var.resources_owner
   }
 }
 
@@ -43,7 +43,7 @@ resource "aws_internet_gateway" "lh_aws_igw" {
 
   tags = {
     Name = "lh_igw-${random_string.random_suffix.id}"
-    Owner = "longhorn-infra"
+    Owner = var.resources_owner
   }
 }
 
@@ -85,6 +85,14 @@ resource "aws_security_group" "lh_aws_secgrp_controlplane" {
     cidr_blocks = ["0.0.0.0/0"]
   }
 
+  ingress {
+    description = "Allow longhorn-ui nodeport"
+    from_port   = 30000
+    to_port     = 30000
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
   ingress {
     description = "Allow UDP connection for longhorn-webhooks"
     from_port   = 0
@@ -110,7 +118,7 @@ resource "aws_security_group" "lh_aws_secgrp_controlplane" {
 
   tags = {
     Name = "lh_aws_sec_grp_controlplane-${random_string.random_suffix.id}"
-    Owner = "longhorn-infra"
+    Owner = var.resources_owner
   }
 }
 
@@ -154,7 +162,7 @@ resource "aws_security_group" "lh_aws_secgrp_worker" {
 
   tags = {
     Name = "lh_aws_sec_grp_worker-${random_string.random_suffix.id}"
-    Owner = "longhorn-infra"
+    Owner = var.resources_owner
   }
 }
 
@@ -167,7 +175,7 @@ resource "aws_subnet" "lh_aws_public_subnet" {
 
   tags = {
     Name = "lh_public_subnet-${random_string.random_suffix.id}"
-    Owner = "longhorn-infra"
+    Owner = var.resources_owner
   }
 }
 
@@ -179,7 +187,7 @@ resource "aws_subnet" "lh_aws_private_subnet" {
 
   tags = {
     Name = "lh_private_subnet-${random_string.random_suffix.id}"
-    Owner = "longhorn-infra"
+    Owner = var.resources_owner
   }
 }
 
@@ -189,7 +197,7 @@ resource "aws_eip" "lh_aws_eip_nat_gw" {
 
   tags = {
     Name = "lh_eip_nat_gw-${random_string.random_suffix.id}"
-    Owner = "longhorn-infra"
+    Owner = var.resources_owner
   }
 }
 
@@ -207,7 +215,7 @@ resource "aws_nat_gateway" "lh_aws_nat_gw" {
 
   tags = {
     Name = "lh_eip_nat_gw-${random_string.random_suffix.id}"
-    Owner = "longhorn-infra"
+    Owner = var.resources_owner
   }
 }
 
@@ -227,7 +235,7 @@ resource "aws_route_table" "lh_aws_public_rt" {
 
   tags = {
     Name = "lh_aws_public_rt-${random_string.random_suffix.id}"
-    Owner = "longhorn-infra"
+    Owner = var.resources_owner
   }
 }
 
@@ -246,7 +254,7 @@ resource "aws_route_table" "lh_aws_private_rt" {
 
   tags = {
     Name = "lh_aws_private_rt-${random_string.random_suffix.id}"
-    Owner = "longhorn-infra"
+    Owner = var.resources_owner
   }
 }
 
@@ -279,7 +287,7 @@ resource "aws_key_pair" "lh_aws_pair_key" {
 
   tags = {
     Name = "lh_aws_key_pair-${random_string.random_suffix.id}"
-    Owner = "longhorn-infra"
+    Owner = var.resources_owner
   }
 }
 
@@ -289,7 +297,7 @@ resource "aws_eip" "lh_aws_eip_controlplane" {
 
   tags = {
     Name = "lh_aws_eip_controlplane-${count.index}-${random_string.random_suffix.id}"
-    Owner = "longhorn-infra"
+    Owner = var.resources_owner
   }
 }
 
@@ -303,7 +311,7 @@ resource "aws_ebs_volume" "lh_aws_hdd_volume" {
 
   tags = {
     Name = "lh-aws-hdd-volume-${count.index}-${random_string.random_suffix.id}"
-    Owner = "longhorn-infra"
+    Owner = var.resources_owner
   }
 }
 
@@ -326,7 +334,7 @@ resource "aws_lb_target_group" "lh_aws_lb_tg_443" {
 
   tags = {
     Name = "lh-aws-lb-tg-443-${random_string.random_suffix.id}"
-    Owner = "longhorn-infra"
+    Owner = var.resources_owner
   }
 }
 
@@ -347,7 +355,7 @@ resource "aws_lb" "lh_aws_lb" {
 
   tags = {
     Name = "lh-aws-lb-${random_string.random_suffix.id}"
-    Owner = "longhorn-infra"
+    Owner = var.resources_owner
   }
 }
 
@@ -371,7 +379,7 @@ resource "aws_lb_listener" "lh_aws_lb_listener_443" {
 
   tags = {
     Name = "lh-aws-lb-listener-443-${random_string.random_suffix.id}"
-    Owner = "longhorn-infra"
+    Owner = var.resources_owner
   }
 
 }

test_framework/terraform/aws/sles/output.tf

@@ -71,3 +71,10 @@ output "instance_mapping" {
     )
   )
 }
+
+output "controlplane_public_ip" {
+  depends_on = [
+    aws_eip.lh_aws_eip_controlplane
+  ]
+  value = aws_eip.lh_aws_eip_controlplane[0].public_ip
+}

test_framework/terraform/aws/sles/rke2_instances.tf

@@ -28,7 +28,7 @@ resource "aws_instance" "lh_aws_instance_controlplane_rke2" {
   tags = {
     Name = "${var.lh_aws_instance_name_controlplane}-${count.index}-${random_string.random_suffix.id}"
     DoNotDelete = "true"
-    Owner = "longhorn-infra"
+    Owner = var.resources_owner
   }
 }
 
@@ -63,7 +63,7 @@ resource "aws_instance" "lh_aws_instance_worker_rke2" {
   tags = {
     Name = "${var.lh_aws_instance_name_worker}-${count.index}-${random_string.random_suffix.id}"
     DoNotDelete = "true"
-    Owner = "longhorn-infra"
+    Owner = var.resources_owner
   }
 }
 

test_framework/terraform/aws/sles/rke_instances.tf

@@ -28,7 +28,7 @@ resource "aws_instance" "lh_aws_instance_controlplane_rke" {
   tags = {
     Name = "${var.lh_aws_instance_name_controlplane}-${count.index}-${random_string.random_suffix.id}"
     DoNotDelete = "true"
-    Owner = "longhorn-infra"
+    Owner = var.resources_owner
   }
 }
 
@@ -87,7 +87,7 @@ resource "aws_instance" "lh_aws_instance_worker_rke" {
   tags = {
     Name = "${var.lh_aws_instance_name_worker}-${count.index}-${random_string.random_suffix.id}"
     DoNotDelete = "true"
-    Owner = "longhorn-infra"
+    Owner = var.resources_owner
   }
 }