From 4d26fab5dbd79d04ffd1087e7d024b68d5a44427 Mon Sep 17 00:00:00 2001 From: Yang Chiu Date: Fri, 15 Sep 2023 11:02:59 +0800 Subject: [PATCH] test: setup out-of-cluster test execution in pipeline Signed-off-by: Yang Chiu --- e2e/Dockerfile | 2 - e2e/libs/node/node.py | 2 +- e2e/requirements.txt | 6 +-- pipelines/e2e/Dockerfile.setup | 2 +- pipelines/e2e/Jenkinsfile | 6 ++- pipelines/e2e/scripts/longhorn-setup.sh | 11 ++++- pipelines/utilities/kubeconfig.sh | 8 ++-- pipelines/utilities/litmus/reboot-engine.yaml | 37 ----------------- pipelines/utilities/longhorn_ui.sh | 7 ++++ pipelines/utilities/run_longhorn_e2e_test.sh | 13 ++++++ pipelines/utilities/terraform_setup.sh | 2 + .../terraform/aws/sles/k3s_instances.tf | 4 +- test_framework/terraform/aws/sles/main.tf | 40 +++++++++++-------- test_framework/terraform/aws/sles/output.tf | 7 ++++ .../terraform/aws/sles/rke2_instances.tf | 4 +- .../terraform/aws/sles/rke_instances.tf | 4 +- .../terraform/aws/sles/variables.tf | 5 +++ 17 files changed, 86 insertions(+), 74 deletions(-) delete mode 100644 pipelines/utilities/litmus/reboot-engine.yaml create mode 100755 pipelines/utilities/longhorn_ui.sh diff --git a/e2e/Dockerfile b/e2e/Dockerfile index 4ff45b424b..f2f69efa65 100644 --- a/e2e/Dockerfile +++ b/e2e/Dockerfile @@ -24,6 +24,4 @@ WORKDIR /e2e RUN pip install -r requirements.txt -ADD pipelines/utilities/litmus/ ./litmus - ENTRYPOINT ["./run.sh"] \ No newline at end of file diff --git a/e2e/libs/node/node.py b/e2e/libs/node/node.py index 43b018b62f..cdd4528095 100644 --- a/e2e/libs/node/node.py +++ b/e2e/libs/node/node.py @@ -38,7 +38,7 @@ def reboot_node(self, running_on_node_name, reboot_node_name, shut_down_time_in_ mapping = yaml.safe_load(f) reboot_node_id = mapping[reboot_node_name] - filepath = './litmus/reboot-node.yaml' + filepath = './templates/litmus/reboot-node.yaml' with open(filepath, 'r') as f: data = yaml.safe_load(f) data['spec']['components']['runner']['nodeSelector']['kubernetes.io/hostname'] = running_on_node_name diff --git a/e2e/requirements.txt b/e2e/requirements.txt index c0ef5f592e..81eac4a39a 100644 --- a/e2e/requirements.txt +++ b/e2e/requirements.txt @@ -3,10 +3,6 @@ argcomplete==1.10.0 directio==1.2 flake8 kubernetes==27.2.0 -pytest==5.3.1 -pytest-repeat==0.9.1 -pytest-order==1.0.1 requests==2.31.0 -six==1.12.0 boto3==1.26.86 -pyyaml==5.4.1 +pyyaml==6.0.1 diff --git a/pipelines/e2e/Dockerfile.setup b/pipelines/e2e/Dockerfile.setup index 18deaed96b..4b79eafa60 100644 --- a/pipelines/e2e/Dockerfile.setup +++ b/pipelines/e2e/Dockerfile.setup @@ -25,7 +25,7 @@ RUN wget -q https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_V wget -q "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" && \ mv yq_linux_amd64 /usr/local/bin/yq && \ chmod +x /usr/local/bin/yq && \ - apk add openssl openssh-client ca-certificates git rsync bash curl jq python3 py3-pip && \ + apk add openssl openssh-client ca-certificates git rsync bash curl jq python3 py3-pip gcc python3-dev libc-dev && \ ssh-keygen -t rsa -b 4096 -N "" -f ~/.ssh/id_rsa && \ curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 && \ chmod 700 get_helm.sh && \ diff --git a/pipelines/e2e/Jenkinsfile b/pipelines/e2e/Jenkinsfile index d7bcef1876..9335197801 100644 --- a/pipelines/e2e/Jenkinsfile +++ b/pipelines/e2e/Jenkinsfile @@ -102,14 +102,18 @@ node { --env TF_VAR_os_distro_version=${DISTRO_VERSION} \ --env TF_VAR_do_token=${env.TF_VAR_do_token} \ --env TF_VAR_lh_aws_access_key=${AWS_ACCESS_KEY} \ + --env TF_VAR_lh_aws_secret_key=${AWS_SECRET_KEY} \ + --env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY} \ + --env AWS_SECRET_ACCESS_KEY=${AWS_SECRET_KEY} \ + --env AWS_DEFAULT_REGION=${AWS_REGION} \ --env TF_VAR_lh_aws_instance_name_controlplane="${JOB_BASE_NAME}-ctrl" \ --env TF_VAR_lh_aws_instance_name_worker="${JOB_BASE_NAME}-wrk" \ --env TF_VAR_lh_aws_instance_type_controlplane=${CONTROLPLANE_INSTANCE_TYPE} \ --env TF_VAR_lh_aws_instance_type_worker=${WORKER_INSTANCE_TYPE}\ - --env TF_VAR_lh_aws_secret_key=${AWS_SECRET_KEY} \ --env TF_VAR_selinux_mode=${SELINUX_MODE} \ --env TF_VAR_registration_code=${REGISTRATION_CODE} \ --env TF_VAR_cis_hardening=${CIS_HARDENING} \ + --env TF_VAR_resources_owner=longhorn-long-running \ ${imageName} """ } diff --git a/pipelines/e2e/scripts/longhorn-setup.sh b/pipelines/e2e/scripts/longhorn-setup.sh index e632d481da..bc0066b5a8 100755 --- a/pipelines/e2e/scripts/longhorn-setup.sh +++ b/pipelines/e2e/scripts/longhorn-setup.sh @@ -9,6 +9,7 @@ source pipelines/utilities/create_aws_secret.sh source pipelines/utilities/install_backupstores.sh source pipelines/utilities/create_longhorn_namespace.sh source pipelines/utilities/longhorn_manifest.sh +source pipelines/utilities/longhorn_ui.sh source pipelines/utilities/install_litmus.sh source pipelines/utilities/run_longhorn_e2e_test.sh @@ -48,7 +49,15 @@ main(){ generate_longhorn_yaml_manifest install_longhorn_by_manifest - run_longhorn_e2e_test + + setup_longhorn_ui_nodeport + export_longhorn_ui_url + + if [[ -n "${LONGHORN_TESTS_CUSTOM_IMAGE}" ]]; then + run_longhorn_e2e_test + else + run_longhorn_e2e_test_out_of_cluster + fi } main diff --git a/pipelines/utilities/kubeconfig.sh b/pipelines/utilities/kubeconfig.sh index 90eb74d7b5..97154be4a2 100755 --- a/pipelines/utilities/kubeconfig.sh +++ b/pipelines/utilities/kubeconfig.sh @@ -3,13 +3,13 @@ set_kubeconfig(){ # but only k3s supports arm64 if [[ "${TF_VAR_arch}" == "amd64" ]] ; then if [[ "${TF_VAR_k8s_distro_name}" == "rke" ]]; then - export KUBECONFIG="test_framework/kube_config_rke.yml" + export KUBECONFIG="${PWD}/test_framework/kube_config_rke.yml" elif [[ "${TF_VAR_k8s_distro_name}" == "rke2" ]]; then - export KUBECONFIG="test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO}/rke2.yaml" + export KUBECONFIG="${PWD}/test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO}/rke2.yaml" else - export KUBECONFIG="test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO}/k3s.yaml" + export KUBECONFIG="${PWD}/test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO}/k3s.yaml" fi elif [[ "${TF_VAR_arch}" == "arm64" ]]; then - export KUBECONFIG="test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO}/k3s.yaml" + export KUBECONFIG="${PWD}/test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO}/k3s.yaml" fi } \ No newline at end of file diff --git a/pipelines/utilities/litmus/reboot-engine.yaml b/pipelines/utilities/litmus/reboot-engine.yaml deleted file mode 100644 index 5ae4fd0d8c..0000000000 --- a/pipelines/utilities/litmus/reboot-engine.yaml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: litmuschaos.io/v1alpha1 -kind: ChaosEngine -metadata: - name: reboot-engine - namespace: default -spec: - # It can be delete/retain - jobCleanUpPolicy: 'retain' - engineState: 'active' - chaosServiceAccount: ec2-terminate-by-id-sa - components: - runner: - nodeSelector: - kubernetes.io/hostname: engine-running-node - experiments: - - name: ec2-terminate-by-id - spec: - components: - nodeSelector: - kubernetes.io/hostname: engine-running-node - env: - # set chaos duration (in sec) as desired - - name: TOTAL_CHAOS_DURATION - value: '30' - # set interval duration (in sec) as desired - - name: CHAOS_INTERVAL - value: '180' - # Instance ID of the target ec2 instance - # Multiple IDs can also be provided as comma separated values ex: id1,id2 - - name: EC2_INSTANCE_ID - value: instance-to-be-terminated - # provide the region name of the instance - - name: REGION - value: 'us-east-1' - # enable it if the target instance is a part of self-managed nodegroup. - - name: MANAGED_NODEGROUP - value: 'disable' \ No newline at end of file diff --git a/pipelines/utilities/longhorn_ui.sh b/pipelines/utilities/longhorn_ui.sh new file mode 100755 index 0000000000..528d2135a3 --- /dev/null +++ b/pipelines/utilities/longhorn_ui.sh @@ -0,0 +1,7 @@ +setup_longhorn_ui_nodeport(){ + kubectl expose --type=NodePort deployment longhorn-ui -n longhorn-system --port 8000 --name longhorn-ui-nodeport --overrides '{ "apiVersion": "v1","spec":{"ports": [{"port":8000,"protocol":"TCP","targetPort":8000,"nodePort":30000}]}}' +} + +export_longhorn_ui_url(){ + export LONGHORN_CLIENT_URL="http://$(cat /tmp/controlplane_public_ip):30000" +} \ No newline at end of file diff --git a/pipelines/utilities/run_longhorn_e2e_test.sh b/pipelines/utilities/run_longhorn_e2e_test.sh index 5e02d59879..e797438e63 100755 --- a/pipelines/utilities/run_longhorn_e2e_test.sh +++ b/pipelines/utilities/run_longhorn_e2e_test.sh @@ -63,3 +63,16 @@ run_longhorn_e2e_test(){ kubectl cp ${LONGHORN_TEST_POD_NAME}:/tmp/test-report/output.xml "output.xml" -c longhorn-test-report kubectl cp ${LONGHORN_TEST_POD_NAME}:/tmp/test-report/report.html "report.html" -c longhorn-test-report } + +run_longhorn_e2e_test_out_of_cluster(){ + cd e2e + pip install -r requirements.txt + + eval "ROBOT_COMMAND_ARGS=($PYTEST_CUSTOM_OPTIONS)" + + ./run.sh "${ROBOT_COMMAND_ARGS[@]}" + + cp /tmp/test-report/log.html "${WORKSPACE}/log.html" + cp /tmp/test-report/output.xml "${WORKSPACE}/output.xml" + cp /tmp/test-report/report.html "${WORKSPACE}/report.html" +} diff --git a/pipelines/utilities/terraform_setup.sh b/pipelines/utilities/terraform_setup.sh index 70c9c06e42..75a01accc3 100755 --- a/pipelines/utilities/terraform_setup.sh +++ b/pipelines/utilities/terraform_setup.sh @@ -24,4 +24,6 @@ if [[ "${TF_VAR_k8s_distro_name}" == "k3s" ]]; then terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} output -raw instance_mapping | jq 'map({(.name | split(".")[0]): .id}) | add' | jq -s add > /tmp/instance_mapping fi +terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} output -raw controlplane_public_ip > /tmp/controlplane_public_ip + exit $? diff --git a/test_framework/terraform/aws/sles/k3s_instances.tf b/test_framework/terraform/aws/sles/k3s_instances.tf index f17afeec77..f59f2e65ba 100644 --- a/test_framework/terraform/aws/sles/k3s_instances.tf +++ b/test_framework/terraform/aws/sles/k3s_instances.tf @@ -28,7 +28,7 @@ resource "aws_instance" "lh_aws_instance_controlplane_k3s" { tags = { Name = "${var.lh_aws_instance_name_controlplane}-${count.index}-${random_string.random_suffix.id}" DoNotDelete = "true" - Owner = "longhorn-infra" + Owner = var.resources_owner } } @@ -63,7 +63,7 @@ resource "aws_instance" "lh_aws_instance_worker_k3s" { tags = { Name = "${var.lh_aws_instance_name_worker}-${count.index}-${random_string.random_suffix.id}" DoNotDelete = "true" - Owner = "longhorn-infra" + Owner = var.resources_owner } } diff --git a/test_framework/terraform/aws/sles/main.tf b/test_framework/terraform/aws/sles/main.tf index 4fa5711c25..665dd5b946 100644 --- a/test_framework/terraform/aws/sles/main.tf +++ b/test_framework/terraform/aws/sles/main.tf @@ -33,7 +33,7 @@ resource "aws_vpc" "lh_aws_vpc" { tags = { Name = "${var.lh_aws_vpc_name}-${random_string.random_suffix.id}" - Owner = "longhorn-infra" + Owner = var.resources_owner } } @@ -43,7 +43,7 @@ resource "aws_internet_gateway" "lh_aws_igw" { tags = { Name = "lh_igw-${random_string.random_suffix.id}" - Owner = "longhorn-infra" + Owner = var.resources_owner } } @@ -85,6 +85,14 @@ resource "aws_security_group" "lh_aws_secgrp_controlplane" { cidr_blocks = ["0.0.0.0/0"] } + ingress { + description = "Allow longhorn-ui nodeport" + from_port = 30000 + to_port = 30000 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + ingress { description = "Allow UDP connection for longhorn-webhooks" from_port = 0 @@ -110,7 +118,7 @@ resource "aws_security_group" "lh_aws_secgrp_controlplane" { tags = { Name = "lh_aws_sec_grp_controlplane-${random_string.random_suffix.id}" - Owner = "longhorn-infra" + Owner = var.resources_owner } } @@ -154,7 +162,7 @@ resource "aws_security_group" "lh_aws_secgrp_worker" { tags = { Name = "lh_aws_sec_grp_worker-${random_string.random_suffix.id}" - Owner = "longhorn-infra" + Owner = var.resources_owner } } @@ -167,7 +175,7 @@ resource "aws_subnet" "lh_aws_public_subnet" { tags = { Name = "lh_public_subnet-${random_string.random_suffix.id}" - Owner = "longhorn-infra" + Owner = var.resources_owner } } @@ -179,7 +187,7 @@ resource "aws_subnet" "lh_aws_private_subnet" { tags = { Name = "lh_private_subnet-${random_string.random_suffix.id}" - Owner = "longhorn-infra" + Owner = var.resources_owner } } @@ -189,7 +197,7 @@ resource "aws_eip" "lh_aws_eip_nat_gw" { tags = { Name = "lh_eip_nat_gw-${random_string.random_suffix.id}" - Owner = "longhorn-infra" + Owner = var.resources_owner } } @@ -207,7 +215,7 @@ resource "aws_nat_gateway" "lh_aws_nat_gw" { tags = { Name = "lh_eip_nat_gw-${random_string.random_suffix.id}" - Owner = "longhorn-infra" + Owner = var.resources_owner } } @@ -227,7 +235,7 @@ resource "aws_route_table" "lh_aws_public_rt" { tags = { Name = "lh_aws_public_rt-${random_string.random_suffix.id}" - Owner = "longhorn-infra" + Owner = var.resources_owner } } @@ -246,7 +254,7 @@ resource "aws_route_table" "lh_aws_private_rt" { tags = { Name = "lh_aws_private_rt-${random_string.random_suffix.id}" - Owner = "longhorn-infra" + Owner = var.resources_owner } } @@ -279,7 +287,7 @@ resource "aws_key_pair" "lh_aws_pair_key" { tags = { Name = "lh_aws_key_pair-${random_string.random_suffix.id}" - Owner = "longhorn-infra" + Owner = var.resources_owner } } @@ -289,7 +297,7 @@ resource "aws_eip" "lh_aws_eip_controlplane" { tags = { Name = "lh_aws_eip_controlplane-${count.index}-${random_string.random_suffix.id}" - Owner = "longhorn-infra" + Owner = var.resources_owner } } @@ -303,7 +311,7 @@ resource "aws_ebs_volume" "lh_aws_hdd_volume" { tags = { Name = "lh-aws-hdd-volume-${count.index}-${random_string.random_suffix.id}" - Owner = "longhorn-infra" + Owner = var.resources_owner } } @@ -326,7 +334,7 @@ resource "aws_lb_target_group" "lh_aws_lb_tg_443" { tags = { Name = "lh-aws-lb-tg-443-${random_string.random_suffix.id}" - Owner = "longhorn-infra" + Owner = var.resources_owner } } @@ -347,7 +355,7 @@ resource "aws_lb" "lh_aws_lb" { tags = { Name = "lh-aws-lb-${random_string.random_suffix.id}" - Owner = "longhorn-infra" + Owner = var.resources_owner } } @@ -371,7 +379,7 @@ resource "aws_lb_listener" "lh_aws_lb_listener_443" { tags = { Name = "lh-aws-lb-listener-443-${random_string.random_suffix.id}" - Owner = "longhorn-infra" + Owner = var.resources_owner } } diff --git a/test_framework/terraform/aws/sles/output.tf b/test_framework/terraform/aws/sles/output.tf index d00ea9b97d..bd2b4c12f7 100644 --- a/test_framework/terraform/aws/sles/output.tf +++ b/test_framework/terraform/aws/sles/output.tf @@ -71,3 +71,10 @@ output "instance_mapping" { ) ) } + +output "controlplane_public_ip" { + depends_on = [ + aws_eip.lh_aws_eip_controlplane + ] + value = aws_eip.lh_aws_eip_controlplane[0].public_ip +} diff --git a/test_framework/terraform/aws/sles/rke2_instances.tf b/test_framework/terraform/aws/sles/rke2_instances.tf index 4521bb44b8..3ed6cade16 100644 --- a/test_framework/terraform/aws/sles/rke2_instances.tf +++ b/test_framework/terraform/aws/sles/rke2_instances.tf @@ -28,7 +28,7 @@ resource "aws_instance" "lh_aws_instance_controlplane_rke2" { tags = { Name = "${var.lh_aws_instance_name_controlplane}-${count.index}-${random_string.random_suffix.id}" DoNotDelete = "true" - Owner = "longhorn-infra" + Owner = var.resources_owner } } @@ -63,7 +63,7 @@ resource "aws_instance" "lh_aws_instance_worker_rke2" { tags = { Name = "${var.lh_aws_instance_name_worker}-${count.index}-${random_string.random_suffix.id}" DoNotDelete = "true" - Owner = "longhorn-infra" + Owner = var.resources_owner } } diff --git a/test_framework/terraform/aws/sles/rke_instances.tf b/test_framework/terraform/aws/sles/rke_instances.tf index fe84244bfe..b629d33037 100644 --- a/test_framework/terraform/aws/sles/rke_instances.tf +++ b/test_framework/terraform/aws/sles/rke_instances.tf @@ -28,7 +28,7 @@ resource "aws_instance" "lh_aws_instance_controlplane_rke" { tags = { Name = "${var.lh_aws_instance_name_controlplane}-${count.index}-${random_string.random_suffix.id}" DoNotDelete = "true" - Owner = "longhorn-infra" + Owner = var.resources_owner } } @@ -87,7 +87,7 @@ resource "aws_instance" "lh_aws_instance_worker_rke" { tags = { Name = "${var.lh_aws_instance_name_worker}-${count.index}-${random_string.random_suffix.id}" DoNotDelete = "true" - Owner = "longhorn-infra" + Owner = var.resources_owner } } diff --git a/test_framework/terraform/aws/sles/variables.tf b/test_framework/terraform/aws/sles/variables.tf index a63557d0e1..6e683a7d55 100644 --- a/test_framework/terraform/aws/sles/variables.tf +++ b/test_framework/terraform/aws/sles/variables.tf @@ -122,3 +122,8 @@ variable "cis_hardening" { type = bool default = false } + +variable "resources_owner" { + type = string + default = "longhorn-infra" +} \ No newline at end of file