diff --git a/manager/integration/Dockerfile b/manager/integration/Dockerfile index c00f519a06..4bf52c20ed 100644 --- a/manager/integration/Dockerfile +++ b/manager/integration/Dockerfile @@ -2,10 +2,11 @@ FROM registry.suse.com/bci/python:3.9 ARG KUBECTL_VERSION=v1.17.0 ARG YQ_VERSION=v4.24.2 +ARG TERRAFORM_VERSION=1.3.5 ARG ARCH=amd64 RUN zypper ref -f -RUN zypper in -y vim-small nfs-client xfsprogs e2fsprogs util-linux-systemd gcc python39-devel gawk java-11-openjdk tar awk gzip wget && \ +RUN zypper in -y vim-small nfs-client xfsprogs e2fsprogs util-linux-systemd gcc python39-devel gawk java-11-openjdk tar awk gzip wget unzip && \ rm -rf /var/cache/zypp/* RUN curl -sO https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_VERSION/bin/linux/${ARCH}/kubectl && \ @@ -16,7 +17,11 @@ RUN curl -sO https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_ ./get_helm.sh && \ wget -q "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${ARCH}" && \ mv yq_linux_${ARCH} /usr/local/bin/yq && \ - chmod +x /usr/local/bin/yq + chmod +x /usr/local/bin/yq && \ + wget -q https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_${ARCH}.zip && \ + unzip terraform_${TERRAFORM_VERSION}_linux_${ARCH}.zip && rm terraform_${TERRAFORM_VERSION}_linux_${ARCH}.zip && \ + mv terraform /usr/bin/terraform && \ + chmod +x /usr/bin/terraform RUN curl -L https://github.com/jonelo/jacksum/releases/download/v3.4.0/jacksum-3.4.0.jar --output /jacksum.jar @@ -28,5 +33,6 @@ WORKDIR /integration/tests ADD pipelines/utilities ./pipelines/utilities ADD pipelines/helm/scripts/upgrade-longhorn.sh ./pipelines/helm/scripts/upgrade-longhorn.sh +ADD pipelines/rancher/scripts/upgrade-longhorn.sh ./pipelines/rancher/scripts/upgrade-longhorn.sh -ENTRYPOINT ["./run.sh"] \ No newline at end of file +ENTRYPOINT ["./run.sh"] diff --git a/manager/integration/tests/conftest.py b/manager/integration/tests/conftest.py index dedee67054..033e4d896d 100644 --- a/manager/integration/tests/conftest.py +++ b/manager/integration/tests/conftest.py @@ -17,6 +17,11 @@ INCLUDE_CA_OPT = "--include-cluster-autoscaler-test" LH_INSTALL_METHOD = "--lh-install-method" +RANCHER_HOSTNAME = "--rancher-hostname" +RANCHER_ACCESS_KEY = "--rancher-access-key" +RANCHER_SECRET_KEY = "--rancher-secret-key" +RANCHER_CHART_INSTALL_VERSION = "--rancher-chart-install-version" +LONGHORN_REPO = "--longhorn-repo" UPGRADE_LH_REPO_URL = "--upgrade-lh-repo-url" UPGRADE_LH_REPO_BRANCH = "--upgrade-lh-repo-branch" UPGRADE_LH_MANAGER_IMAGE = "--upgrade-lh-manager-image" @@ -57,6 +62,35 @@ def pytest_addoption(parser): to determine how to upgrade longhorn for test_upgrade (default: manifest''') + parser.addoption(RANCHER_HOSTNAME, action="store", + default="", + help='''if longhorn install method is rancher, specify + where rancher is hosted''') + + parser.addoption(RANCHER_ACCESS_KEY, action="store", + default="", + help='''if longhorn install method is rancher, specify + the access key and secret key to have the permission to + operate it''') + + parser.addoption(RANCHER_SECRET_KEY, action="store", + default="", + help='''if longhorn install method is rancher, specify + the access key and secret key to have the permission to + operate it''') + + parser.addoption(RANCHER_CHART_INSTALL_VERSION, action="store", + default="", + help='''if longhorn install method is rancher, specify + the longhorn chart version you would like to install like + 102.2.1+up1.4.2''') + + parser.addoption(LONGHORN_REPO, action="store", + default="", + help='''if longhorn install method is rancher, specify + the longhorn dockerhub repo of longhorn components like + longhornio or rancher''') + longhorn_repo_url =\ "https://github.com/longhorn/longhorn.git" parser.addoption(UPGRADE_LH_REPO_URL, action="store", diff --git a/manager/integration/tests/test_upgrade.py b/manager/integration/tests/test_upgrade.py index 349f4cde66..dbc2762ec7 100644 --- a/manager/integration/tests/test_upgrade.py +++ b/manager/integration/tests/test_upgrade.py @@ -65,6 +65,31 @@ def longhorn_install_method(request): return request.config.getoption("--lh-install-method") +@pytest.fixture +def rancher_hostname(request): + return request.config.getoption("--rancher-hostname") + + +@pytest.fixture +def rancher_access_key(request): + return request.config.getoption("--rancher-access-key") + + +@pytest.fixture +def rancher_secret_key(request): + return request.config.getoption("--rancher-secret-key") + + +@pytest.fixture +def rancher_chart_install_version(request): + return request.config.getoption("--rancher-chart-install-version") + + +@pytest.fixture +def longhorn_repo(request): + return request.config.getoption("--longhorn-repo") + + @pytest.fixture def upgrade_longhorn_repo_url(request): return request.config.getoption("--upgrade-lh-repo-url") @@ -112,6 +137,11 @@ def longhorn_upgrade_type(): def longhorn_upgrade(longhorn_install_method, + rancher_hostname, + rancher_access_key, + rancher_secret_key, + rancher_chart_install_version, + longhorn_repo, longhorn_repo_url, longhorn_repo_branch, longhorn_manager_image, @@ -124,15 +154,27 @@ def longhorn_upgrade(longhorn_install_method, command = "../scripts/upgrade-longhorn.sh" elif longhorn_install_method == "helm": command = "./pipelines/helm/scripts/upgrade-longhorn.sh" - process = subprocess.Popen([command, - longhorn_repo_url, - longhorn_repo_branch, - longhorn_manager_image, - longhorn_engine_image, - longhorn_instance_manager_image, - longhorn_share_manager_image, - longhorn_backing_image_manager_image], - shell=False) + elif longhorn_install_method == "rancher": + command = "./pipelines/rancher/scripts/upgrade-longhorn.sh" + + if longhorn_install_method != "rancher": + process = subprocess.Popen([command, + longhorn_repo_url, + longhorn_repo_branch, + longhorn_manager_image, + longhorn_engine_image, + longhorn_instance_manager_image, + longhorn_share_manager_image, + longhorn_backing_image_manager_image], + shell=False) + else: + process = subprocess.Popen([command, + rancher_hostname, + rancher_access_key, + rancher_secret_key, + rancher_chart_install_version, + longhorn_repo], + shell=False) process.wait() if process.returncode == 0: longhorn_upgraded = True @@ -146,6 +188,11 @@ def longhorn_upgrade(longhorn_install_method, @pytest.mark.upgrade # NOQA def test_upgrade(longhorn_upgrade_type, longhorn_install_method, + rancher_hostname, + rancher_access_key, + rancher_secret_key, + rancher_chart_install_version, + longhorn_repo, upgrade_longhorn_repo_url, upgrade_longhorn_repo_branch, upgrade_longhorn_manager_image, @@ -191,6 +238,11 @@ def test_upgrade(longhorn_upgrade_type, 18. Verify the vol_rebuild is still healthy """ longhorn_install_method = longhorn_install_method + rancher_hostname = rancher_hostname + rancher_access_key = rancher_access_key + rancher_secret_key = rancher_secret_key + rancher_chart_install_version = rancher_chart_install_version + longhorn_repo = longhorn_repo longhorn_repo_url = upgrade_longhorn_repo_url longhorn_repo_branch = upgrade_longhorn_repo_branch longhorn_manager_image = upgrade_longhorn_manager_image @@ -320,6 +372,11 @@ def test_upgrade(longhorn_upgrade_type, # upgrade Longhorn manager assert longhorn_upgrade(longhorn_install_method, + rancher_hostname, + rancher_access_key, + rancher_secret_key, + rancher_chart_install_version, + longhorn_repo, longhorn_repo_url, longhorn_repo_branch, longhorn_manager_image, diff --git a/pipelines/rancher/Dockerfile.setup b/pipelines/rancher/Dockerfile.setup new file mode 100644 index 0000000000..18deaed96b --- /dev/null +++ b/pipelines/rancher/Dockerfile.setup @@ -0,0 +1,34 @@ +From alpine:latest + +ARG KUBECTL_VERSION=v1.20.2 + +ARG RKE_VERSION=v1.3.4 + +ARG TERRAFORM_VERSION=1.3.5 + +ARG YQ_VERSION=v4.24.2 + +ENV WORKSPACE /src/longhorn-tests + +WORKDIR $WORKSPACE + +RUN wget -q https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl && \ + mv kubectl /usr/local/bin/kubectl && \ + chmod +x /usr/local/bin/kubectl && \ + wget -q https://github.com/rancher/rke/releases/download/$RKE_VERSION/rke_linux-amd64 && \ + mv rke_linux-amd64 /usr/bin/rke && \ + chmod +x /usr/bin/rke && \ + wget -q https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \ + unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip && rm terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \ + mv terraform /usr/bin/terraform && \ + chmod +x /usr/bin/terraform && \ + wget -q "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" && \ + mv yq_linux_amd64 /usr/local/bin/yq && \ + chmod +x /usr/local/bin/yq && \ + apk add openssl openssh-client ca-certificates git rsync bash curl jq python3 py3-pip && \ + ssh-keygen -t rsa -b 4096 -N "" -f ~/.ssh/id_rsa && \ + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 && \ + chmod 700 get_helm.sh && \ + ./get_helm.sh + +COPY [".", "$WORKSPACE"] diff --git a/pipelines/rancher/Jenkinsfile b/pipelines/rancher/Jenkinsfile new file mode 100644 index 0000000000..d08b0eb3e2 --- /dev/null +++ b/pipelines/rancher/Jenkinsfile @@ -0,0 +1,208 @@ +def imageName = "${JOB_BASE_NAME}-${env.BUILD_NUMBER}" +def summary +def WORKSPACE = "/src/longhorn-tests" +def BUILD_TRIGGER_BY = "\n${currentBuild.getBuildCauses()[0].shortDescription}" + +// define optional parameters +def SELINUX_MODE = params.SELINUX_MODE ? params.SELINUX_MODE : "" + +def CREDS_ID = JOB_BASE_NAME == "longhorn-tests-regression" ? "AWS_CREDS_RANCHER_QA" : "AWS_CREDS" +def REGISTRATION_CODE_ID = params.ARCH == "amd64" ? "REGISTRATION_CODE" : "REGISTRATION_CODE_ARM64" + +// parameters for air gap installation +def AIR_GAP_INSTALLATION = params.AIR_GAP_INSTALLATION ? params.AIR_GAP_INSTALLATION : false +def CIS_HARDENING = params.CIS_HARDENING ? params.CIS_HARDENING : false +def REGISTRY_URL +def REGISTRY_USERNAME +def REGISTRY_PASSWORD + +// parameter for hdd test +def USE_HDD = params.USE_HDD ? params.USE_HDD : false + +node { + + withCredentials([ + usernamePassword(credentialsId: CREDS_ID, passwordVariable: 'AWS_SECRET_KEY', usernameVariable: 'AWS_ACCESS_KEY'), + string(credentialsId: 'DO_CREDS', variable: 'DO_TOKEN'), + string(credentialsId: REGISTRATION_CODE_ID, variable: 'REGISTRATION_CODE'), + ]) { + + if (params.SEND_SLACK_NOTIFICATION) { + notifyBuild('STARTED', BUILD_TRIGGER_BY, params.NOTIFY_SLACK_CHANNEL) + } + + checkout scm + + try { + + if (params.AIR_GAP_INSTALLATION) { + + stage('airgap build') { + sh "airgap/scripts/build.sh" + sh """ docker run -itd --name airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} \ + --env TF_VAR_longhorn_version=${LONGHORN_INSTALL_VERSION} \ + --env TF_VAR_do_token=${DO_TOKEN} \ + --env TF_VAR_aws_access_key=${AWS_ACCESS_KEY} \ + --env TF_VAR_aws_secret_key=${AWS_SECRET_KEY} \ + airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} + """ + } + + stage ('airgap setup') { + sh "docker exec airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} ./airgap/scripts/terraform-setup.sh" + REGISTRY_URL = sh ( + script: "docker exec airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} terraform -chdir=./airgap/terraform output -raw registry_url", + returnStdout: true + ) + println REGISTRY_URL + REGISTRY_USERNAME = sh ( + script: "docker exec airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} terraform -chdir=./airgap/terraform output -raw registry_username", + returnStdout: true + ) + REGISTRY_PASSWORD = sh ( + script: "docker exec airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} terraform -chdir=./airgap/terraform output -raw registry_password", + returnStdout: true + ) + } + + } + + stage('build') { + + echo "Using credentials: $CREDS_ID" + echo "Using registration coce: $REGISTRATION_CODE_ID" + + sh "pipelines/rancher/scripts/build.sh" + sh """ docker run -itd --name ${JOB_BASE_NAME}-${BUILD_NUMBER} \ + --env AIR_GAP_INSTALLATION=${AIR_GAP_INSTALLATION} \ + --env REGISTRY_URL=${REGISTRY_URL} \ + --env REGISTRY_USERNAME=${REGISTRY_USERNAME} \ + --env REGISTRY_PASSWORD=${REGISTRY_PASSWORD} \ + --env RANCHER_VERSION=${RANCHER_VERSION} \ + --env RANCHER_CHART_REPO_URI=${RANCHER_CHART_REPO_URI} \ + --env RANCHER_CHART_REPO_BRANCH=${RANCHER_CHART_REPO_BRANCH} \ + --env LONGHORN_REPO=${LONGHORN_REPO} \ + --env LONGHORN_TESTS_CUSTOM_IMAGE=${LONGHORN_TESTS_CUSTOM_IMAGE} \ + --env DISTRO=${DISTRO} \ + --env LONGHORN_INSTALL_VERSION=${LONGHORN_INSTALL_VERSION} \ + --env LONGHORN_STABLE_VERSION=${LONGHORN_STABLE_VERSION} \ + --env LONGHORN_TRANSIENT_VERSION=${LONGHORN_TRANSIENT_VERSION} \ + --env LONGHORN_TEST_CLOUDPROVIDER=${LONGHORN_TEST_CLOUDPROVIDER} \ + --env LONGHORN_UPGRADE_TEST=${LONGHORN_UPGRADE_TEST} \ + --env PYTEST_CUSTOM_OPTIONS="${PYTEST_CUSTOM_OPTIONS}" \ + --env BACKUP_STORE_TYPE="${BACKUP_STORE_TYPE}" \ + --env TF_VAR_create_load_balancer=true \ + --env TF_VAR_use_hdd=${USE_HDD} \ + --env TF_VAR_arch=${ARCH} \ + --env TF_VAR_k8s_distro_name=${K8S_DISTRO_NAME} \ + --env TF_VAR_k8s_distro_version=${K8S_DISTRO_VERSION} \ + --env TF_VAR_aws_availability_zone=${AWS_AVAILABILITY_ZONE} \ + --env TF_VAR_aws_region=${AWS_REGION} \ + --env TF_VAR_os_distro_version=${DISTRO_VERSION} \ + --env TF_VAR_do_token=${env.TF_VAR_do_token} \ + --env TF_VAR_lh_aws_access_key=${AWS_ACCESS_KEY} \ + --env TF_VAR_lh_aws_instance_name_controlplane="${JOB_BASE_NAME}-ctrl" \ + --env TF_VAR_lh_aws_instance_name_worker="${JOB_BASE_NAME}-wrk" \ + --env TF_VAR_lh_aws_instance_type_controlplane=${CONTROLPLANE_INSTANCE_TYPE} \ + --env TF_VAR_lh_aws_instance_type_worker=${WORKER_INSTANCE_TYPE}\ + --env TF_VAR_lh_aws_secret_key=${AWS_SECRET_KEY} \ + --env TF_VAR_selinux_mode=${SELINUX_MODE} \ + --env TF_VAR_registration_code=${REGISTRATION_CODE} \ + --env TF_VAR_cis_hardening=${CIS_HARDENING} \ + ${imageName} + """ + } + + timeout(60) { + stage ('terraform') { + sh "docker exec ${JOB_BASE_NAME}-${BUILD_NUMBER} pipelines/utilities/terraform_setup.sh" + } + } + + stage ('longhorn setup & tests') { + sh "docker exec ${JOB_BASE_NAME}-${BUILD_NUMBER} pipelines/rancher/scripts/longhorn-setup.sh" + } + + stage ('download support bundle') { + sh "docker exec ${JOB_BASE_NAME}-${BUILD_NUMBER} pipelines/utilities/download_support_bundle.sh ${JOB_BASE_NAME}-${BUILD_NUMBER}-bundle.zip" + sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/${JOB_BASE_NAME}-${BUILD_NUMBER}-bundle.zip ." + archiveArtifacts allowEmptyArchive: true, artifacts: '**/*.zip', followSymlinks: false + } + + stage ('report generation') { + sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/longhorn-test-junit-report.xml ." + + if(params.LONGHORN_UPGRADE_TEST && params.LONGHORN_TRANSIENT_VERSION) { + sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/longhorn-test-upgrade-from-stable-junit-report.xml ." + sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/longhorn-test-upgrade-from-transient-junit-report.xml ." + summary = junit 'longhorn-test-upgrade-from-stable-junit-report.xml, longhorn-test-upgrade-from-transient-junit-report.xml, longhorn-test-junit-report.xml' + } + else if(params.LONGHORN_UPGRADE_TEST) { + sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/longhorn-test-upgrade-from-stable-junit-report.xml ." + summary = junit 'longhorn-test-upgrade-from-stable-junit-report.xml, longhorn-test-junit-report.xml' + } + else { + summary = junit 'longhorn-test-junit-report.xml' + } + } + + } catch (e) { + currentBuild.result = "FAILED" + throw e + } finally { + stage ('releasing resources') { + if (sh (script: "docker container inspect airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} > /dev/null 2>&1", returnStatus: true) == 0) { + sh "docker exec airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} ./airgap/scripts/cleanup.sh" + sh "docker stop airgap-${JOB_BASE_NAME}-${BUILD_NUMBER}" + sh "docker rm -v airgap-${JOB_BASE_NAME}-${BUILD_NUMBER}" + sh "docker rmi airgap-${JOB_BASE_NAME}-${BUILD_NUMBER}" + } + + if (sh (script: "docker container inspect ${JOB_BASE_NAME}-${BUILD_NUMBER} > /dev/null 2>&1", returnStatus: true) == 0) { + sh "docker exec ${JOB_BASE_NAME}-${BUILD_NUMBER} pipelines/utilities/cleanup.sh" + sh "docker stop ${JOB_BASE_NAME}-${BUILD_NUMBER}" + sh "docker rm -v ${JOB_BASE_NAME}-${BUILD_NUMBER}" + sh "docker rmi ${imageName}" + } + + if (summary) { + summary_msg = "\nTest Summary - Failures: ${summary.failCount}, Skipped: ${summary.skipCount}, Passed: ${summary.passCount} -- Job completed in ${currentBuild.durationString.replace(' and counting', '')}" + } else { + summary_msg = "\n Test Failed: No Junit report" + } + + if(params.SEND_SLACK_NOTIFICATION){ + notifyBuild(currentBuild.result, summary_msg, params.NOTIFY_SLACK_CHANNEL) + } + } + } + } + +} + + +def notifyBuild(String buildStatus = 'STARTED', String summary_msg, String slack_channel) { + // build status of null means successful + buildStatus = buildStatus ?: 'SUCCESSFUL' + + // Default values + def colorName = 'RED' + def colorCode = '#FF0000' + def subject = "${buildStatus}: Job '${env.JOB_BASE_NAME} [${env.BUILD_NUMBER}]'" + def summary = "${subject} (${env.BUILD_URL})" + summary_msg + + // Override default values based on build status + if (buildStatus == 'STARTED') { + color = 'YELLOW' + colorCode = '#FFFF00' + } else if (buildStatus == 'SUCCESSFUL') { + color = 'GREEN' + colorCode = '#00FF00' + } else { + color = 'RED' + colorCode = '#FF0000' + } + + // Send notifications + slackSend (color: colorCode, message: summary, channel: slack_channel, tokenCredentialId: 'longhorn-tests-slack-token') +} diff --git a/pipelines/rancher/scripts/build.sh b/pipelines/rancher/scripts/build.sh new file mode 100755 index 0000000000..9b8fae9015 --- /dev/null +++ b/pipelines/rancher/scripts/build.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker build --no-cache -f ./pipelines/rancher/Dockerfile.setup -t "${JOB_BASE_NAME}-${BUILD_NUMBER}" . diff --git a/pipelines/rancher/scripts/longhorn-setup.sh b/pipelines/rancher/scripts/longhorn-setup.sh new file mode 100755 index 0000000000..b7a417ef67 --- /dev/null +++ b/pipelines/rancher/scripts/longhorn-setup.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +set -x + +source pipelines/utilities/kubeconfig.sh +source pipelines/utilities/selinux_workaround.sh +source pipelines/utilities/install_csi_snapshotter.sh +source pipelines/utilities/create_aws_secret.sh +source pipelines/utilities/install_backupstores.sh +source pipelines/utilities/create_longhorn_namespace.sh +source pipelines/utilities/longhorn_rancher_chart.sh +source pipelines/utilities/run_longhorn_test.sh + +# create and clean tmpdir +TMPDIR="/tmp/longhorn" +mkdir -p ${TMPDIR} +rm -rf "${TMPDIR}/" + +export LONGHORN_NAMESPACE="longhorn-system" +export LONGHORN_INSTALL_METHOD="rancher" + + +main(){ + set_kubeconfig + + if [[ ${DISTRO} == "rhel" ]] || [[ ${DISTRO} == "rockylinux" ]] || [[ ${DISTRO} == "oracle" ]]; then + apply_selinux_workaround + fi + + # set debugging mode off to avoid leaking aws secrets to the logs. + # DON'T REMOVE! + set +x + create_aws_secret + set -x + + create_longhorn_namespace + install_backupstores + install_csi_snapshotter + + install_rancher + get_rancher_api_key + + + if [[ "${LONGHORN_UPGRADE_TEST}" == true ]]; then + install_longhorn_rancher_chart "${LONGHORN_STABLE_VERSION}" + LONGHORN_UPGRADE_TYPE="from_stable" + LONGHORN_UPGRADE_TEST_POD_NAME="longhorn-test-upgrade-from-stable" + if [[ -n "${LONGHORN_TRANSIENT_VERSION}" ]]; then + RANCHER_CHART_INSTALL_VERSION="${LONGHORN_TRANSIENT_VERSION}" + # extract 1.4.2 from 102.2.1+up1.4.2 + RAW_VERSION=(${LONGHORN_TRANSIENT_VERSION/up/ }) + if [[ "${LONGHORN_REPO}" == "rancher" ]]; then + UPGRADE_LH_ENGINE_IMAGE="rancher/mirrored-longhornio-longhorn-engine:v${RAW_VERSION[1]}" + else + UPGRADE_LH_ENGINE_IMAGE="longhornio/longhorn-engine:v${RAW_VERSION[1]}" + fi + run_longhorn_upgrade_test + LONGHORN_UPGRADE_TYPE="from_transient" + LONGHORN_UPGRADE_TEST_POD_NAME="longhorn-test-upgrade-from-transient" + fi + RANCHER_CHART_INSTALL_VERSION="${LONGHORN_INSTALL_VERSION}" + RAW_VERSION=(${LONGHORN_INSTALL_VERSION/up/ }) + if [[ "${LONGHORN_REPO}" == "rancher" ]]; then + UPGRADE_LH_ENGINE_IMAGE="rancher/mirrored-longhornio-longhorn-engine:v${RAW_VERSION[1]}" + else + UPGRADE_LH_ENGINE_IMAGE="longhornio/longhorn-engine:v${RAW_VERSION[1]}" + fi + run_longhorn_upgrade_test + run_longhorn_test + else + install_longhorn_rancher_chart + run_longhorn_test + fi +} + +main diff --git a/pipelines/rancher/scripts/upgrade-longhorn.sh b/pipelines/rancher/scripts/upgrade-longhorn.sh new file mode 100755 index 0000000000..ddc658c09a --- /dev/null +++ b/pipelines/rancher/scripts/upgrade-longhorn.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -x + +export RANCHER_HOSTNAME="${1}" +export RANCHER_ACCESS_KEY="${2}" +export RANCHER_SECRET_KEY="${3}" +export LONGHORN_INSTALL_VERSION="${4}" +export LONGHORN_REPO="${5}" + +source pipelines/utilities/longhorn_rancher_chart.sh + +export LONGHORN_NAMESPACE="longhorn-system" + +upgrade_longhorn_rancher_chart diff --git a/pipelines/utilities/cleanup.sh b/pipelines/utilities/cleanup.sh new file mode 100755 index 0000000000..189d7dff7c --- /dev/null +++ b/pipelines/utilities/cleanup.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# terminate any terraform processes +TERRAFORM_PIDS=( `ps aux | grep -i terraform | grep -v grep | awk '{printf("%s ",$1)}'` ) +if [[ -n ${TERRAFORM_PIDS[@]} ]] ; then + for PID in "${TERRAFORM_PIDS[@]}"; do + kill "${TERRAFORM_PIDS}" + done +fi + +# wait 30 seconds for graceful terraform termination +sleep 30 + +terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} destroy -auto-approve -no-color diff --git a/pipelines/utilities/create_registry_secret.sh b/pipelines/utilities/create_registry_secret.sh new file mode 100755 index 0000000000..967e454df0 --- /dev/null +++ b/pipelines/utilities/create_registry_secret.sh @@ -0,0 +1,3 @@ +create_registry_secret(){ + kubectl -n ${LONGHORN_NAMESPACE} create secret docker-registry docker-registry-secret --docker-server=${REGISTRY_URL} --docker-username=${REGISTRY_USERNAME} --docker-password=${REGISTRY_PASSWORD} +} \ No newline at end of file diff --git a/pipelines/utilities/download_support_bundle.sh b/pipelines/utilities/download_support_bundle.sh new file mode 100755 index 0000000000..3788b882f5 --- /dev/null +++ b/pipelines/utilities/download_support_bundle.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +source pipelines/utilities/kubeconfig.sh + +set -e + +SUPPORT_BUNDLE_FILE_NAME=${1:-"lh-support-bundle.zip"} +SUPPORT_BUNDLE_ISSUE_URL=${2:-""} +SUPPORT_BUNDLE_ISSUE_DESC=${3:-"Auto-generated support bundle"} + +set_kubeconfig + +LH_FRONTEND_ADDR=`kubectl get svc -n longhorn-system longhorn-frontend -o json | jq -r '.spec.clusterIP + ":" + (.spec.ports[0].port|tostring)'` + +JSON_PAYLOAD="{\"issueURL\": \"${SUPPORT_BUNDLE_ISSUE_DESC}\", \"description\": \"${SUPPORT_BUNDLE_ISSUE_DESC}\"}" + +CURL_CMD="curl -XPOST http://${LH_FRONTEND_ADDR}/v1/supportbundles -H 'Accept: application/json' -H 'Accept-Encoding: gzip, deflate' -d '"${JSON_PAYLOAD}"'" + +SUPPORT_BUNDLE_URL=`kubectl exec -n longhorn-system svc/longhorn-frontend -- bash -c "${CURL_CMD}" | jq -r '.links.self + "/" + .name'` + +SUPPORT_BUNDLE_READY=false +while [[ ${SUPPORT_BUNDLE_READY} == false ]]; do + PERCENT=`kubectl exec -n longhorn-system svc/longhorn-frontend -- curl -H 'Accept: application/json' ${SUPPORT_BUNDLE_URL} | jq -r '.progressPercentage' || true` + echo ${PERCENT} + + if [[ ${PERCENT} == 100 ]]; then SUPPORT_BUNDLE_READY=true; fi +done + +kubectl exec -n longhorn-system svc/longhorn-frontend -- curl -H 'Accept-Encoding: gzip, deflate' ${SUPPORT_BUNDLE_URL}/download > ${SUPPORT_BUNDLE_FILE_NAME} diff --git a/pipelines/utilities/longhorn_rancher_chart.sh b/pipelines/utilities/longhorn_rancher_chart.sh new file mode 100755 index 0000000000..2b3ccada51 --- /dev/null +++ b/pipelines/utilities/longhorn_rancher_chart.sh @@ -0,0 +1,66 @@ +source pipelines/utilities/longhorn_status.sh + + +install_rancher() { + + RANCHER_HOSTNAME=`cat "test_framework/load_balancer_url"` + RANCHER_BOOTSTRAP_PASSWORD='p@ssw0rd' + + kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v1.12.2/cert-manager.crds.yaml + kubectl create namespace cert-manager + helm repo add jetstack https://charts.jetstack.io + helm repo update + helm install cert-manager jetstack/cert-manager --namespace cert-manager --version v1.12.2 + kubectl rollout status deployment cert-manager -n cert-manager + kubectl get pods --namespace cert-manager + + helm repo add rancher-latest https://releases.rancher.com/server-charts/latest + helm repo update + kubectl create namespace cattle-system + if [[ -z "${RANCHER_VERSION}" ]]; then + RANCHER_VERSION=$(helm search repo rancher-latest/rancher -o yaml | yq .[0].version) + fi + helm install rancher rancher-latest/rancher --version "${RANCHER_VERSION}" --namespace cattle-system --set bootstrapPassword="${RANCHER_BOOTSTRAP_PASSWORD}" --set hostname="${RANCHER_HOSTNAME}" --set replicas=3 --set ingress.tls.source=letsEncrypt --set letsEncrypt.email=yang.chiu@suse.com + kubectl -n cattle-system rollout status deploy/rancher +} + + +get_rancher_api_key() { + TOKEN=$(curl -X POST -s -k "https://${RANCHER_HOSTNAME}/v3-public/localproviders/local?action=login" -H 'Content-Type: application/json' -d "{\"username\":\"admin\", \"password\":\"${RANCHER_BOOTSTRAP_PASSWORD}\", \"responseType\": \"json\"}" | jq -r '.token' | tr -d '"') + ARR=(${TOKEN//:/ }) + RANCHER_ACCESS_KEY=${ARR[0]} + RANCHER_SECRET_KEY=${ARR[1]} +} + + +install_longhorn_rancher_chart() { + CHART_VERSION="${1:-${LONGHORN_INSTALL_VERSION}}" + terraform -chdir="pipelines/utilities/rancher/terraform_install" init + terraform -chdir="pipelines/utilities/rancher/terraform_install" apply \ + -var="api_url=https://${RANCHER_HOSTNAME}" \ + -var="access_key=${RANCHER_ACCESS_KEY}" \ + -var="secret_key=${RANCHER_SECRET_KEY}" \ + -var="rancher_chart_git_repo=${RANCHER_CHART_REPO_URI}" \ + -var="rancher_chart_git_branch=${RANCHER_CHART_REPO_BRANCH}" \ + -var="rancher_chart_install_version=${CHART_VERSION}" \ + -var="longhorn_repo=${LONGHORN_REPO}" \ + -var="registry_url=${REGISTRY_URL}" \ + -var="registry_user=${REGISTRY_USERNAME}" \ + -var="registry_passwd=${REGISTRY_PASSWORD}" \ + -var="registry_secret=docker-registry-secret" \ + -auto-approve -no-color + wait_longhorn_status_running +} + + +upgrade_longhorn_rancher_chart() { + terraform -chdir="pipelines/utilities/rancher/terraform_upgrade" init + terraform -chdir="pipelines/utilities/rancher/terraform_upgrade" apply \ + -var="api_url=https://${RANCHER_HOSTNAME}" \ + -var="access_key=${RANCHER_ACCESS_KEY}" \ + -var="secret_key=${RANCHER_SECRET_KEY}" \ + -var="rancher_chart_install_version=${LONGHORN_INSTALL_VERSION}" \ + -var="longhorn_repo=${LONGHORN_REPO}" \ + -auto-approve -no-color + wait_longhorn_status_running +} diff --git a/pipelines/utilities/rancher/terraform_install/main.tf b/pipelines/utilities/rancher/terraform_install/main.tf new file mode 100644 index 0000000000..7898b7967a --- /dev/null +++ b/pipelines/utilities/rancher/terraform_install/main.tf @@ -0,0 +1,61 @@ +terraform { + required_providers { + rancher2 = { + source = "rancher/rancher2" + version = "~> 1.24" + } + } +} + +provider "rancher2" { + api_url = var.api_url + insecure = true + access_key = var.access_key + secret_key = var.secret_key +} + +resource "rancher2_catalog_v2" "longhorn_repo" { + cluster_id = "local" + name = "longhorn-repo" + git_repo = var.rancher_chart_git_repo + git_branch = var.rancher_chart_git_branch +} + +resource "rancher2_app_v2" "longhorn_app" { + cluster_id = "local" + name = "longhorn-app" + namespace = "longhorn-system" + repo_name = "longhorn-repo" + chart_name = "longhorn" + chart_version = var.rancher_chart_install_version + values = <<-EOF +image: + csi: + attacher: + repository: ${var.longhorn_repo == "rancher" ? "${var.longhorn_repo}/mirrored-longhornio-" : "${var.longhorn_repo}/"}csi-attacher + nodeDriverRegistrar: + repository: ${var.longhorn_repo == "rancher" ? "${var.longhorn_repo}/mirrored-longhornio-" : "${var.longhorn_repo}/"}csi-node-driver-registrar + provisioner: + repository: ${var.longhorn_repo == "rancher" ? "${var.longhorn_repo}/mirrored-longhornio-" : "${var.longhorn_repo}/"}csi-provisioner + resizer: + repository: ${var.longhorn_repo == "rancher" ? "${var.longhorn_repo}/mirrored-longhornio-" : "${var.longhorn_repo}/"}csi-resizer + snapshotter: + repository: ${var.longhorn_repo == "rancher" ? "${var.longhorn_repo}/mirrored-longhornio-" : "${var.longhorn_repo}/"}csi-snapshotter + longhorn: + backingImageManager: + repository: ${var.longhorn_repo == "rancher" ? "${var.longhorn_repo}/mirrored-longhornio-" : "${var.longhorn_repo}/"}backing-image-manager + engine: + repository: ${var.longhorn_repo == "rancher" ? "${var.longhorn_repo}/mirrored-longhornio-" : "${var.longhorn_repo}/"}longhorn-engine + instanceManager: + repository: ${var.longhorn_repo == "rancher" ? "${var.longhorn_repo}/mirrored-longhornio-" : "${var.longhorn_repo}/"}longhorn-instance-manager + manager: + repository: ${var.longhorn_repo == "rancher" ? "${var.longhorn_repo}/mirrored-longhornio-" : "${var.longhorn_repo}/"}longhorn-manager + shareManager: + repository: ${var.longhorn_repo == "rancher" ? "${var.longhorn_repo}/mirrored-longhornio-" : "${var.longhorn_repo}/"}longhorn-share-manager + supportBundleManager: + repository: ${var.longhorn_repo == "rancher" ? "${var.longhorn_repo}/mirrored-longhornio-" : "${var.longhorn_repo}/"}support-bundle-kit + ui: + repository: ${var.longhorn_repo == "rancher" ? "${var.longhorn_repo}/mirrored-longhornio-" : "${var.longhorn_repo}/"}longhorn-ui +EOF + wait = true +} \ No newline at end of file diff --git a/pipelines/utilities/rancher/terraform_install/variable.tf b/pipelines/utilities/rancher/terraform_install/variable.tf new file mode 100644 index 0000000000..80ef298b93 --- /dev/null +++ b/pipelines/utilities/rancher/terraform_install/variable.tf @@ -0,0 +1,43 @@ +variable "api_url" { + type = string +} + +variable "access_key" { + type = string +} + +variable "secret_key" { + type = string +} + +variable "rancher_chart_git_repo" { + type = string +} + +variable "rancher_chart_git_branch" { + type = string +} + +variable "rancher_chart_install_version" { + type = string +} + +variable "longhorn_repo" { + type = string +} + +variable "registry_url" { + type = string +} + +variable "registry_user" { + type = string +} + +variable "registry_passwd" { + type = string +} + +variable "registry_secret" { + type = string +} diff --git a/pipelines/utilities/rancher/terraform_upgrade/main.tf b/pipelines/utilities/rancher/terraform_upgrade/main.tf new file mode 100644 index 0000000000..c949e6e28a --- /dev/null +++ b/pipelines/utilities/rancher/terraform_upgrade/main.tf @@ -0,0 +1,61 @@ +terraform { + required_providers { + rancher2 = { + source = "rancher/rancher2" + version = "~> 1.24" + } + } +} + +provider "rancher2" { + api_url = var.api_url + insecure = true + access_key = var.access_key + secret_key = var.secret_key +} + + +data "rancher2_catalog_v2" "longhorn_repo" { + cluster_id = "local" + name = "longhorn-repo" +} + + +resource "rancher2_app_v2" "longhorn_app" { + cluster_id = "local" + name = "longhorn-app" + namespace = "longhorn-system" + repo_name = "longhorn-repo" + chart_name = "longhorn" + chart_version = var.rancher_chart_install_version + values = <<-EOF +image: + csi: + attacher: + repository: ${var.longhorn_repo == "rancher" ? "${var.longhorn_repo}/mirrored-longhornio-" : "${var.longhorn_repo}/"}csi-attacher + nodeDriverRegistrar: + repository: ${var.longhorn_repo == "rancher" ? "${var.longhorn_repo}/mirrored-longhornio-" : "${var.longhorn_repo}/"}csi-node-driver-registrar + provisioner: + repository: ${var.longhorn_repo == "rancher" ? "${var.longhorn_repo}/mirrored-longhornio-" : "${var.longhorn_repo}/"}csi-provisioner + resizer: + repository: ${var.longhorn_repo == "rancher" ? "${var.longhorn_repo}/mirrored-longhornio-" : "${var.longhorn_repo}/"}csi-resizer + snapshotter: + repository: ${var.longhorn_repo == "rancher" ? "${var.longhorn_repo}/mirrored-longhornio-" : "${var.longhorn_repo}/"}csi-snapshotter + longhorn: + backingImageManager: + repository: ${var.longhorn_repo == "rancher" ? "${var.longhorn_repo}/mirrored-longhornio-" : "${var.longhorn_repo}/"}backing-image-manager + engine: + repository: ${var.longhorn_repo == "rancher" ? "${var.longhorn_repo}/mirrored-longhornio-" : "${var.longhorn_repo}/"}longhorn-engine + instanceManager: + repository: ${var.longhorn_repo == "rancher" ? "${var.longhorn_repo}/mirrored-longhornio-" : "${var.longhorn_repo}/"}longhorn-instance-manager + manager: + repository: ${var.longhorn_repo == "rancher" ? "${var.longhorn_repo}/mirrored-longhornio-" : "${var.longhorn_repo}/"}longhorn-manager + shareManager: + repository: ${var.longhorn_repo == "rancher" ? "${var.longhorn_repo}/mirrored-longhornio-" : "${var.longhorn_repo}/"}longhorn-share-manager + supportBundleManager: + repository: ${var.longhorn_repo == "rancher" ? "${var.longhorn_repo}/mirrored-longhornio-" : "${var.longhorn_repo}/"}support-bundle-kit + ui: + repository: ${var.longhorn_repo == "rancher" ? "${var.longhorn_repo}/mirrored-longhornio-" : "${var.longhorn_repo}/"}longhorn-ui +EOF + wait = true +} \ No newline at end of file diff --git a/pipelines/utilities/rancher/terraform_upgrade/variable.tf b/pipelines/utilities/rancher/terraform_upgrade/variable.tf new file mode 100644 index 0000000000..3b7a682abb --- /dev/null +++ b/pipelines/utilities/rancher/terraform_upgrade/variable.tf @@ -0,0 +1,19 @@ +variable "api_url" { + type = string +} + +variable "access_key" { + type = string +} + +variable "secret_key" { + type = string +} + +variable "rancher_chart_install_version" { + type = string +} + +variable "longhorn_repo" { + type = string +} diff --git a/pipelines/utilities/run_longhorn_test.sh b/pipelines/utilities/run_longhorn_test.sh index 204ce8f2fe..0122434a47 100755 --- a/pipelines/utilities/run_longhorn_test.sh +++ b/pipelines/utilities/run_longhorn_test.sh @@ -79,6 +79,11 @@ run_longhorn_upgrade_test(){ "--include-upgrade-test", "-k", "test_upgrade", "--lh-install-method", "'${LONGHORN_INSTALL_METHOD}'", + "--rancher-hostname", "'${RANCHER_HOSTNAME}'", + "--rancher-access-key", "'${RANCHER_ACCESS_KEY}'", + "--rancher-secret-key", "'${RANCHER_SECRET_KEY}'", + "--rancher-chart-install-version", "'${RANCHER_CHART_INSTALL_VERSION}'", + "--longhorn-repo", "'${LONGHORN_REPO}'", "--upgrade-lh-repo-url", "'${UPGRADE_LH_REPO_URL}'", "--upgrade-lh-repo-branch", "'${UPGRADE_LH_REPO_BRANCH}'", "--upgrade-lh-manager-image", "'${UPGRADE_LH_MANAGER_IMAGE}'", diff --git a/pipelines/utilities/selinux_workaround.sh b/pipelines/utilities/selinux_workaround.sh new file mode 100755 index 0000000000..e96b86be82 --- /dev/null +++ b/pipelines/utilities/selinux_workaround.sh @@ -0,0 +1,3 @@ +apply_selinux_workaround(){ + kubectl apply -f "https://raw.githubusercontent.com/longhorn/longhorn/master/deploy/prerequisite/longhorn-iscsi-selinux-workaround.yaml" +} \ No newline at end of file diff --git a/pipelines/utilities/terraform_setup.sh b/pipelines/utilities/terraform_setup.sh new file mode 100755 index 0000000000..2eb9ca468f --- /dev/null +++ b/pipelines/utilities/terraform_setup.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -x + +if [[ ${TF_VAR_arch} == "amd64" ]]; then + terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} init + terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} apply -auto-approve -no-color + if [[ ${TF_VAR_k8s_distro_name} == "rke" ]]; then + terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} apply -auto-approve -no-color -refresh-only + terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} output -raw rke_config > test_framework/rke.yml + sleep 30 + rke up --config test_framework/rke.yml + fi +else + terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} init + terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} apply -auto-approve -no-color +fi + +if [[ "${TF_VAR_create_load_balancer}" == true ]]; then + terraform -chdir=test_framework/terraform/${LONGHORN_TEST_CLOUDPROVIDER}/${DISTRO} output -raw load_balancer_url > test_framework/load_balancer_url +fi + +exit $?