diff --git a/manager/integration/Dockerfile b/manager/integration/Dockerfile index 6f3c45b7f3..8541f5edd9 100644 --- a/manager/integration/Dockerfile +++ b/manager/integration/Dockerfile @@ -42,5 +42,6 @@ ADD pipelines/helm/scripts/upgrade-longhorn.sh ./pipelines/helm/scripts/upgrade- ADD pipelines/rancher/scripts/upgrade-longhorn.sh ./pipelines/rancher/scripts/upgrade-longhorn.sh ADD pipelines/flux/scripts/upgrade-longhorn.sh ./pipelines/flux/scripts/upgrade-longhorn.sh ADD pipelines/argocd/scripts/upgrade-longhorn.sh ./pipelines/argocd/scripts/upgrade-longhorn.sh +ADD pipelines/fleet/scripts/upgrade-longhorn.sh ./pipelines/fleet/scripts/upgrade-longhorn.sh ENTRYPOINT ["./run.sh"] diff --git a/manager/integration/tests/test_upgrade.py b/manager/integration/tests/test_upgrade.py index 428d41c7b2..97b77d959d 100644 --- a/manager/integration/tests/test_upgrade.py +++ b/manager/integration/tests/test_upgrade.py @@ -205,6 +205,12 @@ def longhorn_upgrade(longhorn_install_method, longhorn_repo_url, longhorn_repo_branch], shell=False) + elif longhorn_install_method == "fleet": + command = "./pipelines/fleet/scripts/upgrade-longhorn.sh" + process = subprocess.Popen([command, + longhorn_repo_url, + longhorn_repo_branch], + shell=False) process.wait() if process.returncode == 0: diff --git a/pipelines/fleet/Dockerfile.setup b/pipelines/fleet/Dockerfile.setup new file mode 100644 index 0000000000..18deaed96b --- /dev/null +++ b/pipelines/fleet/Dockerfile.setup @@ -0,0 +1,34 @@ +From alpine:latest + +ARG KUBECTL_VERSION=v1.20.2 + +ARG RKE_VERSION=v1.3.4 + +ARG TERRAFORM_VERSION=1.3.5 + +ARG YQ_VERSION=v4.24.2 + +ENV WORKSPACE /src/longhorn-tests + +WORKDIR $WORKSPACE + +RUN wget -q https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl && \ + mv kubectl /usr/local/bin/kubectl && \ + chmod +x /usr/local/bin/kubectl && \ + wget -q https://github.com/rancher/rke/releases/download/$RKE_VERSION/rke_linux-amd64 && \ + mv rke_linux-amd64 /usr/bin/rke && \ + chmod +x /usr/bin/rke && \ + wget -q https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \ + unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip && rm terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \ + mv terraform /usr/bin/terraform && \ + chmod +x /usr/bin/terraform && \ + wget -q "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" && \ + mv yq_linux_amd64 /usr/local/bin/yq && \ + chmod +x /usr/local/bin/yq && \ + apk add openssl openssh-client ca-certificates git rsync bash curl jq python3 py3-pip && \ + ssh-keygen -t rsa -b 4096 -N "" -f ~/.ssh/id_rsa && \ + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 && \ + chmod 700 get_helm.sh && \ + ./get_helm.sh + +COPY [".", "$WORKSPACE"] diff --git a/pipelines/fleet/Jenkinsfile b/pipelines/fleet/Jenkinsfile new file mode 100644 index 0000000000..d2fdf43a09 --- /dev/null +++ b/pipelines/fleet/Jenkinsfile @@ -0,0 +1,163 @@ +def imageName = "${JOB_BASE_NAME}-${env.BUILD_NUMBER}" +def summary +def WORKSPACE = "/src/longhorn-tests" +def BUILD_TRIGGER_BY = "\n${currentBuild.getBuildCauses()[0].shortDescription}" + +// define optional parameters +def SELINUX_MODE = params.SELINUX_MODE ? params.SELINUX_MODE : "" + +def CREDS_ID = JOB_BASE_NAME == "longhorn-tests-regression" ? "AWS_CREDS_RANCHER_QA" : "AWS_CREDS" +def REGISTRATION_CODE_ID = params.ARCH == "amd64" ? "REGISTRATION_CODE" : "REGISTRATION_CODE_ARM64" + +// parameters for air gap installation +def AIR_GAP_INSTALLATION = params.AIR_GAP_INSTALLATION ? params.AIR_GAP_INSTALLATION : false +def CIS_HARDENING = params.CIS_HARDENING ? params.CIS_HARDENING : false +def REGISTRY_URL +def REGISTRY_USERNAME +def REGISTRY_PASSWORD + +// parameter for hdd test +def USE_HDD = params.USE_HDD ? params.USE_HDD : false + +node { + + withCredentials([ + usernamePassword(credentialsId: CREDS_ID, passwordVariable: 'AWS_SECRET_KEY', usernameVariable: 'AWS_ACCESS_KEY'), + string(credentialsId: REGISTRATION_CODE_ID, variable: 'REGISTRATION_CODE'), + ]) { + + if (params.SEND_SLACK_NOTIFICATION) { + notifyBuild('STARTED', BUILD_TRIGGER_BY, params.NOTIFY_SLACK_CHANNEL) + } + + checkout scm + + try { + + stage('build') { + + echo "Using credentials: $CREDS_ID" + echo "Using registration code: $REGISTRATION_CODE_ID" + + sh "pipelines/fleet/scripts/build.sh" + sh """ docker run -itd --name ${JOB_BASE_NAME}-${BUILD_NUMBER} \ + --env REGISTRY_URL=${REGISTRY_URL} \ + --env REGISTRY_USERNAME=${REGISTRY_USERNAME} \ + --env REGISTRY_PASSWORD=${REGISTRY_PASSWORD} \ + --env FLEET_REPO_URI=${FLEET_REPO_URI} \ + --env FLEET_REPO_VERSION=${FLEET_REPO_VERSION} \ + --env LONGHORN_TESTS_CUSTOM_IMAGE=${LONGHORN_TESTS_CUSTOM_IMAGE} \ + --env DISTRO=${DISTRO} \ + --env FLEET_REPO_STABLE_VERSION=${FLEET_REPO_STABLE_VERSION} \ + --env FLEET_REPO_TRANSIENT_VERSION=${FLEET_REPO_TRANSIENT_VERSION} \ + --env LONGHORN_TEST_CLOUDPROVIDER=${LONGHORN_TEST_CLOUDPROVIDER} \ + --env LONGHORN_UPGRADE_TEST=${LONGHORN_UPGRADE_TEST} \ + --env PYTEST_CUSTOM_OPTIONS="${PYTEST_CUSTOM_OPTIONS}" \ + --env BACKUP_STORE_TYPE="${BACKUP_STORE_TYPE}" \ + --env TF_VAR_use_hdd=${USE_HDD} \ + --env TF_VAR_arch=${ARCH} \ + --env TF_VAR_k8s_distro_name=${K8S_DISTRO_NAME} \ + --env TF_VAR_k8s_distro_version=${K8S_DISTRO_VERSION} \ + --env TF_VAR_aws_availability_zone=${AWS_AVAILABILITY_ZONE} \ + --env TF_VAR_aws_region=${AWS_REGION} \ + --env TF_VAR_os_distro_version=${DISTRO_VERSION} \ + --env TF_VAR_lh_aws_access_key=${AWS_ACCESS_KEY} \ + --env TF_VAR_lh_aws_instance_name_controlplane="${JOB_BASE_NAME}-ctrl" \ + --env TF_VAR_lh_aws_instance_name_worker="${JOB_BASE_NAME}-wrk" \ + --env TF_VAR_lh_aws_instance_type_controlplane=${CONTROLPLANE_INSTANCE_TYPE} \ + --env TF_VAR_lh_aws_instance_type_worker=${WORKER_INSTANCE_TYPE}\ + --env TF_VAR_lh_aws_secret_key=${AWS_SECRET_KEY} \ + --env TF_VAR_selinux_mode=${SELINUX_MODE} \ + --env TF_VAR_registration_code=${REGISTRATION_CODE} \ + --env TF_VAR_cis_hardening=${CIS_HARDENING} \ + ${imageName} + """ + } + + timeout(60) { + stage ('terraform') { + sh "docker exec ${JOB_BASE_NAME}-${BUILD_NUMBER} pipelines/utilities/terraform_setup.sh" + } + } + + stage ('longhorn setup & tests') { + sh "docker exec ${JOB_BASE_NAME}-${BUILD_NUMBER} pipelines/fleet/scripts/longhorn-setup.sh" + } + + stage ('download support bundle') { + sh "docker exec ${JOB_BASE_NAME}-${BUILD_NUMBER} pipelines/utilities/download_support_bundle.sh ${JOB_BASE_NAME}-${BUILD_NUMBER}-bundle.zip" + sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/${JOB_BASE_NAME}-${BUILD_NUMBER}-bundle.zip ." + archiveArtifacts allowEmptyArchive: true, artifacts: '**/*.zip', followSymlinks: false + } + + stage ('report generation') { + sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/longhorn-test-junit-report.xml ." + + if(params.LONGHORN_UPGRADE_TEST && params.FLEET_REPO_TRANSIENT_VERSION) { + sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/longhorn-test-upgrade-from-stable-junit-report.xml ." + sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/longhorn-test-upgrade-from-transient-junit-report.xml ." + summary = junit 'longhorn-test-upgrade-from-stable-junit-report.xml, longhorn-test-upgrade-from-transient-junit-report.xml, longhorn-test-junit-report.xml' + } + else if(params.LONGHORN_UPGRADE_TEST) { + sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/longhorn-test-upgrade-from-stable-junit-report.xml ." + summary = junit 'longhorn-test-upgrade-from-stable-junit-report.xml, longhorn-test-junit-report.xml' + } + else { + summary = junit 'longhorn-test-junit-report.xml' + } + } + + } catch (e) { + currentBuild.result = "FAILED" + throw e + } finally { + stage ('releasing resources') { + + if (sh (script: "docker container inspect ${JOB_BASE_NAME}-${BUILD_NUMBER} > /dev/null 2>&1", returnStatus: true) == 0) { + sh "docker exec ${JOB_BASE_NAME}-${BUILD_NUMBER} pipelines/utilities/cleanup.sh" + sh "docker stop ${JOB_BASE_NAME}-${BUILD_NUMBER}" + sh "docker rm -v ${JOB_BASE_NAME}-${BUILD_NUMBER}" + sh "docker rmi ${imageName}" + } + + if (summary) { + summary_msg = "\nTest Summary - Failures: ${summary.failCount}, Skipped: ${summary.skipCount}, Passed: ${summary.passCount} -- Job completed in ${currentBuild.durationString.replace(' and counting', '')}" + } else { + summary_msg = "\n Test Failed: No Junit report" + } + + if(params.SEND_SLACK_NOTIFICATION){ + notifyBuild(currentBuild.result, summary_msg, params.NOTIFY_SLACK_CHANNEL) + } + } + } + } + +} + + +def notifyBuild(String buildStatus = 'STARTED', String summary_msg, String slack_channel) { + // build status of null means successful + buildStatus = buildStatus ?: 'SUCCESSFUL' + + // Default values + def colorName = 'RED' + def colorCode = '#FF0000' + def subject = "${buildStatus}: Job '${env.JOB_BASE_NAME} [${env.BUILD_NUMBER}]'" + def summary = "${subject} (${env.BUILD_URL})" + summary_msg + + // Override default values based on build status + if (buildStatus == 'STARTED') { + color = 'YELLOW' + colorCode = '#FFFF00' + } else if (buildStatus == 'SUCCESSFUL') { + color = 'GREEN' + colorCode = '#00FF00' + } else { + color = 'RED' + colorCode = '#FF0000' + } + + // Send notifications + slackSend (color: colorCode, message: summary, channel: slack_channel, tokenCredentialId: 'longhorn-tests-slack-token') +} diff --git a/pipelines/fleet/scripts/build.sh b/pipelines/fleet/scripts/build.sh new file mode 100755 index 0000000000..0e4b6813ac --- /dev/null +++ b/pipelines/fleet/scripts/build.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker build --no-cache -f ./pipelines/fleet/Dockerfile.setup -t "${JOB_BASE_NAME}-${BUILD_NUMBER}" . diff --git a/pipelines/fleet/scripts/longhorn-setup.sh b/pipelines/fleet/scripts/longhorn-setup.sh new file mode 100755 index 0000000000..4332155876 --- /dev/null +++ b/pipelines/fleet/scripts/longhorn-setup.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +set -x + +source pipelines/utilities/kubeconfig.sh +source pipelines/utilities/selinux_workaround.sh +source pipelines/utilities/install_csi_snapshotter.sh +source pipelines/utilities/create_aws_secret.sh +source pipelines/utilities/install_backupstores.sh +source pipelines/utilities/create_longhorn_namespace.sh +source pipelines/utilities/fleet.sh +source pipelines/utilities/run_longhorn_test.sh + + +export LONGHORN_NAMESPACE="longhorn-system" +export LONGHORN_INSTALL_METHOD="fleet" + + +main(){ + set_kubeconfig + + if [[ ${DISTRO} == "rhel" ]] || [[ ${DISTRO} == "rockylinux" ]] || [[ ${DISTRO} == "oracle" ]]; then + apply_selinux_workaround + fi + + # set debugging mode off to avoid leaking aws secrets to the logs. + # DON'T REMOVE! + set +x + create_aws_secret + set -x + + create_longhorn_namespace + install_backupstores + install_csi_snapshotter + + install_fleet + + if [[ "${LONGHORN_UPGRADE_TEST}" == true ]]; then + create_fleet_git_repo "${FLEET_REPO_STABLE_VERSION}" + LONGHORN_UPGRADE_TYPE="from_stable" + LONGHORN_UPGRADE_TEST_POD_NAME="longhorn-test-upgrade-from-stable" + if [[ -n "${FLEET_REPO_TRANSIENT_VERSION}" ]]; then + UPGRADE_LH_REPO_URL="${FLEET_REPO_URI}" + UPGRADE_LH_REPO_BRANCH="${FLEET_REPO_TRANSIENT_VERSION}" + UPGRADE_LH_ENGINE_IMAGE="longhornio/longhorn-engine:${FLEET_REPO_TRANSIENT_VERSION}" + run_longhorn_upgrade_test + LONGHORN_UPGRADE_TYPE="from_transient" + LONGHORN_UPGRADE_TEST_POD_NAME="longhorn-test-upgrade-from-transient" + fi + UPGRADE_LH_REPO_URL="${FLEET_REPO_URI}" + UPGRADE_LH_REPO_BRANCH="${FLEET_REPO_VERSION}" + UPGRADE_LH_ENGINE_IMAGE="longhornio/longhorn-engine:${FLEET_REPO_VERSION}" + run_longhorn_upgrade_test + run_longhorn_test + else + create_fleet_git_repo + run_longhorn_test + fi +} + +main diff --git a/pipelines/fleet/scripts/upgrade-longhorn.sh b/pipelines/fleet/scripts/upgrade-longhorn.sh new file mode 100755 index 0000000000..13571ec608 --- /dev/null +++ b/pipelines/fleet/scripts/upgrade-longhorn.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -x + +export FLEET_REPO_URI="${1}" +export FLEET_REPO_VERSION="${2}" + +source pipelines/utilities/fleet.sh + +export LONGHORN_NAMESPACE="longhorn-system" + +create_fleet_git_repo diff --git a/pipelines/utilities/fleet.sh b/pipelines/utilities/fleet.sh new file mode 100755 index 0000000000..3c80f40f4c --- /dev/null +++ b/pipelines/utilities/fleet.sh @@ -0,0 +1,61 @@ +source pipelines/utilities/longhorn_status.sh + + +install_fleet(){ + helm repo add fleet https://rancher.github.io/fleet-helm-charts/ + helm -n cattle-fleet-system install --create-namespace --wait fleet-crd fleet/fleet-crd + helm -n cattle-fleet-system install --create-namespace --wait fleet fleet/fleet +} + + +create_fleet_git_repo(){ + REVISION="${1:-${FLEET_REPO_VERSION}}" + cat > longhorn-gitrepo.yaml <