ci: add pipeline for fleet

Signed-off-by: Yang Chiu <[email protected]>

manager/integration/Dockerfile

@@ -42,5 +42,6 @@ ADD pipelines/helm/scripts/upgrade-longhorn.sh ./pipelines/helm/scripts/upgrade-
 ADD pipelines/rancher/scripts/upgrade-longhorn.sh ./pipelines/rancher/scripts/upgrade-longhorn.sh
 ADD pipelines/flux/scripts/upgrade-longhorn.sh ./pipelines/flux/scripts/upgrade-longhorn.sh
 ADD pipelines/argocd/scripts/upgrade-longhorn.sh ./pipelines/argocd/scripts/upgrade-longhorn.sh
+ADD pipelines/fleet/scripts/upgrade-longhorn.sh ./pipelines/fleet/scripts/upgrade-longhorn.sh
 
 ENTRYPOINT ["./run.sh"]

manager/integration/tests/test_upgrade.py

@@ -205,6 +205,12 @@ def longhorn_upgrade(longhorn_install_method,
                                     longhorn_repo_url,
                                     longhorn_repo_branch],
                                    shell=False)
+    elif longhorn_install_method == "fleet":
+        command = "./pipelines/fleet/scripts/upgrade-longhorn.sh"
+        process = subprocess.Popen([command,
+                                    longhorn_repo_url,
+                                    longhorn_repo_branch],
+                                   shell=False)
 
     process.wait()
     if process.returncode == 0:

pipelines/fleet/Dockerfile.setup

@@ -0,0 +1,34 @@
+From alpine:latest
+
+ARG KUBECTL_VERSION=v1.20.2
+
+ARG RKE_VERSION=v1.3.4
+
+ARG TERRAFORM_VERSION=1.3.5
+
+ARG YQ_VERSION=v4.24.2
+
+ENV WORKSPACE /src/longhorn-tests
+
+WORKDIR $WORKSPACE
+
+RUN wget -q https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl && \
+    mv kubectl /usr/local/bin/kubectl && \
+    chmod +x /usr/local/bin/kubectl && \
+    wget -q https://github.com/rancher/rke/releases/download/$RKE_VERSION/rke_linux-amd64 && \
+    mv rke_linux-amd64 /usr/bin/rke && \
+    chmod +x /usr/bin/rke && \
+    wget -q https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \
+    unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip && rm terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \
+    mv terraform /usr/bin/terraform && \
+    chmod +x /usr/bin/terraform && \
+    wget -q "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" && \
+    mv yq_linux_amd64 /usr/local/bin/yq && \
+    chmod +x /usr/local/bin/yq && \
+    apk add openssl openssh-client ca-certificates git rsync bash curl jq python3 py3-pip && \
+    ssh-keygen -t rsa -b 4096 -N "" -f ~/.ssh/id_rsa && \
+    curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 && \
+    chmod 700 get_helm.sh && \
+    ./get_helm.sh
+
+COPY [".", "$WORKSPACE"]

pipelines/fleet/Jenkinsfile

@@ -0,0 +1,163 @@
+def imageName = "${JOB_BASE_NAME}-${env.BUILD_NUMBER}"
+def summary
+def WORKSPACE = "/src/longhorn-tests"
+def BUILD_TRIGGER_BY = "\n${currentBuild.getBuildCauses()[0].shortDescription}"
+
+// define optional parameters
+def SELINUX_MODE = params.SELINUX_MODE ? params.SELINUX_MODE : ""
+
+def CREDS_ID = JOB_BASE_NAME == "longhorn-tests-regression" ? "AWS_CREDS_RANCHER_QA" : "AWS_CREDS"
+def REGISTRATION_CODE_ID = params.ARCH == "amd64" ? "REGISTRATION_CODE" : "REGISTRATION_CODE_ARM64"
+
+// parameters for air gap installation
+def AIR_GAP_INSTALLATION = params.AIR_GAP_INSTALLATION ? params.AIR_GAP_INSTALLATION : false
+def CIS_HARDENING = params.CIS_HARDENING ? params.CIS_HARDENING : false
+def REGISTRY_URL
+def REGISTRY_USERNAME
+def REGISTRY_PASSWORD
+
+// parameter for hdd test
+def USE_HDD = params.USE_HDD ? params.USE_HDD : false
+
+node {
+
+    withCredentials([
+        usernamePassword(credentialsId: CREDS_ID, passwordVariable: 'AWS_SECRET_KEY', usernameVariable: 'AWS_ACCESS_KEY'),
+        string(credentialsId: REGISTRATION_CODE_ID, variable: 'REGISTRATION_CODE'),
+    ]) {
+
+        if (params.SEND_SLACK_NOTIFICATION) {
+            notifyBuild('STARTED', BUILD_TRIGGER_BY, params.NOTIFY_SLACK_CHANNEL)
+        }
+
+        checkout scm
+
+        try {
+
+            stage('build') {
+
+                echo "Using credentials: $CREDS_ID"
+                echo "Using registration code: $REGISTRATION_CODE_ID"
+
+                sh "pipelines/fleet/scripts/build.sh"
+                sh """ docker run -itd --name ${JOB_BASE_NAME}-${BUILD_NUMBER} \
+                                       --env REGISTRY_URL=${REGISTRY_URL} \
+                                       --env REGISTRY_USERNAME=${REGISTRY_USERNAME} \
+                                       --env REGISTRY_PASSWORD=${REGISTRY_PASSWORD} \
+                                       --env FLEET_REPO_URI=${FLEET_REPO_URI} \
+                                       --env FLEET_REPO_VERSION=${FLEET_REPO_VERSION} \
+                                       --env LONGHORN_TESTS_CUSTOM_IMAGE=${LONGHORN_TESTS_CUSTOM_IMAGE} \
+                                       --env DISTRO=${DISTRO} \
+                                       --env FLEET_REPO_STABLE_VERSION=${FLEET_REPO_STABLE_VERSION} \
+                                       --env FLEET_REPO_TRANSIENT_VERSION=${FLEET_REPO_TRANSIENT_VERSION} \
+                                       --env LONGHORN_TEST_CLOUDPROVIDER=${LONGHORN_TEST_CLOUDPROVIDER} \
+                                       --env LONGHORN_UPGRADE_TEST=${LONGHORN_UPGRADE_TEST} \
+                                       --env PYTEST_CUSTOM_OPTIONS="${PYTEST_CUSTOM_OPTIONS}" \
+                                       --env BACKUP_STORE_TYPE="${BACKUP_STORE_TYPE}" \
+                                       --env TF_VAR_use_hdd=${USE_HDD} \
+                                       --env TF_VAR_arch=${ARCH} \
+                                       --env TF_VAR_k8s_distro_name=${K8S_DISTRO_NAME} \
+                                       --env TF_VAR_k8s_distro_version=${K8S_DISTRO_VERSION} \
+                                       --env TF_VAR_aws_availability_zone=${AWS_AVAILABILITY_ZONE} \
+                                       --env TF_VAR_aws_region=${AWS_REGION} \
+                                       --env TF_VAR_os_distro_version=${DISTRO_VERSION} \
+                                       --env TF_VAR_lh_aws_access_key=${AWS_ACCESS_KEY} \
+                                       --env TF_VAR_lh_aws_instance_name_controlplane="${JOB_BASE_NAME}-ctrl" \
+                                       --env TF_VAR_lh_aws_instance_name_worker="${JOB_BASE_NAME}-wrk" \
+                                       --env TF_VAR_lh_aws_instance_type_controlplane=${CONTROLPLANE_INSTANCE_TYPE} \
+                                       --env TF_VAR_lh_aws_instance_type_worker=${WORKER_INSTANCE_TYPE}\
+                                       --env TF_VAR_lh_aws_secret_key=${AWS_SECRET_KEY} \
+                                       --env TF_VAR_selinux_mode=${SELINUX_MODE} \
+                                       --env TF_VAR_registration_code=${REGISTRATION_CODE} \
+                                       --env TF_VAR_cis_hardening=${CIS_HARDENING} \
+                                       ${imageName}
+                """
+            }
+
+            timeout(60) {
+                stage ('terraform') {
+                    sh "docker exec ${JOB_BASE_NAME}-${BUILD_NUMBER} pipelines/utilities/terraform_setup.sh"
+                }
+			}
+
+            stage ('longhorn setup & tests') {
+                sh "docker exec ${JOB_BASE_NAME}-${BUILD_NUMBER} pipelines/fleet/scripts/longhorn-setup.sh"
+            }
+
+            stage ('download support bundle') {
+                sh "docker exec ${JOB_BASE_NAME}-${BUILD_NUMBER} pipelines/utilities/download_support_bundle.sh  ${JOB_BASE_NAME}-${BUILD_NUMBER}-bundle.zip"
+                sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/${JOB_BASE_NAME}-${BUILD_NUMBER}-bundle.zip ."
+				archiveArtifacts allowEmptyArchive: true, artifacts: '**/*.zip', followSymlinks: false
+			}
+
+            stage ('report generation') {
+                sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/longhorn-test-junit-report.xml ."
+
+                if(params.LONGHORN_UPGRADE_TEST && params.FLEET_REPO_TRANSIENT_VERSION) {
+                    sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/longhorn-test-upgrade-from-stable-junit-report.xml ."
+                    sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/longhorn-test-upgrade-from-transient-junit-report.xml ."
+                    summary = junit 'longhorn-test-upgrade-from-stable-junit-report.xml, longhorn-test-upgrade-from-transient-junit-report.xml, longhorn-test-junit-report.xml'
+                }
+                else if(params.LONGHORN_UPGRADE_TEST) {
+                    sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/longhorn-test-upgrade-from-stable-junit-report.xml ."
+                    summary = junit 'longhorn-test-upgrade-from-stable-junit-report.xml, longhorn-test-junit-report.xml'
+                }
+                else {
+                    summary = junit 'longhorn-test-junit-report.xml'
+                }
+            }
+
+        } catch (e) {
+            currentBuild.result = "FAILED"
+            throw e
+        } finally {
+            stage ('releasing resources') {
+
+                if (sh (script: "docker container inspect ${JOB_BASE_NAME}-${BUILD_NUMBER} > /dev/null 2>&1", returnStatus: true) == 0) {
+                    sh "docker exec ${JOB_BASE_NAME}-${BUILD_NUMBER} pipelines/utilities/cleanup.sh"
+                    sh "docker stop ${JOB_BASE_NAME}-${BUILD_NUMBER}"
+                    sh "docker rm -v ${JOB_BASE_NAME}-${BUILD_NUMBER}"
+                    sh "docker rmi ${imageName}"
+                }
+
+                if (summary) {
+                    summary_msg = "\nTest Summary - Failures: ${summary.failCount}, Skipped: ${summary.skipCount}, Passed: ${summary.passCount}  -- Job completed in ${currentBuild.durationString.replace(' and counting', '')}"
+                } else {
+                    summary_msg = "\n Test Failed: No Junit report"
+                }
+
+                if(params.SEND_SLACK_NOTIFICATION){
+                    notifyBuild(currentBuild.result, summary_msg, params.NOTIFY_SLACK_CHANNEL)
+                }
+            }
+        }
+    }
+
+}
+
+
+def notifyBuild(String buildStatus = 'STARTED', String summary_msg, String slack_channel) {
+  // build status of null means successful
+  buildStatus =  buildStatus ?: 'SUCCESSFUL'
+
+  // Default values
+  def colorName = 'RED'
+  def colorCode = '#FF0000'
+  def subject = "${buildStatus}: Job '${env.JOB_BASE_NAME} [${env.BUILD_NUMBER}]'"
+  def summary = "${subject} (${env.BUILD_URL})" + summary_msg
+
+  // Override default values based on build status
+  if (buildStatus == 'STARTED') {
+    color = 'YELLOW'
+    colorCode = '#FFFF00'
+  } else if (buildStatus == 'SUCCESSFUL') {
+    color = 'GREEN'
+    colorCode = '#00FF00'
+  } else {
+    color = 'RED'
+    colorCode = '#FF0000'
+  }
+
+  // Send notifications
+  slackSend (color: colorCode, message: summary, channel: slack_channel,  tokenCredentialId: 'longhorn-tests-slack-token')
+}

pipelines/fleet/scripts/build.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker build --no-cache -f ./pipelines/fleet/Dockerfile.setup -t "${JOB_BASE_NAME}-${BUILD_NUMBER}" .

pipelines/fleet/scripts/longhorn-setup.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+set -x
+
+source pipelines/utilities/kubeconfig.sh
+source pipelines/utilities/selinux_workaround.sh
+source pipelines/utilities/install_csi_snapshotter.sh
+source pipelines/utilities/create_aws_secret.sh
+source pipelines/utilities/install_backupstores.sh
+source pipelines/utilities/create_longhorn_namespace.sh
+source pipelines/utilities/fleet.sh
+source pipelines/utilities/run_longhorn_test.sh
+
+
+export LONGHORN_NAMESPACE="longhorn-system"
+export LONGHORN_INSTALL_METHOD="fleet"
+
+
+main(){
+  set_kubeconfig
+
+  if [[ ${DISTRO} == "rhel" ]] || [[ ${DISTRO} == "rockylinux" ]] || [[ ${DISTRO} == "oracle" ]]; then
+    apply_selinux_workaround
+  fi
+
+  # set debugging mode off to avoid leaking aws secrets to the logs.
+  # DON'T REMOVE!
+  set +x
+  create_aws_secret
+  set -x
+
+  create_longhorn_namespace
+  install_backupstores
+  install_csi_snapshotter
+
+  install_fleet
+
+  if [[ "${LONGHORN_UPGRADE_TEST}" == true ]]; then
+    create_fleet_git_repo "${FLEET_REPO_STABLE_VERSION}"
+    LONGHORN_UPGRADE_TYPE="from_stable"
+    LONGHORN_UPGRADE_TEST_POD_NAME="longhorn-test-upgrade-from-stable"
+    if [[ -n "${FLEET_REPO_TRANSIENT_VERSION}" ]]; then
+      UPGRADE_LH_REPO_URL="${FLEET_REPO_URI}"
+      UPGRADE_LH_REPO_BRANCH="${FLEET_REPO_TRANSIENT_VERSION}"
+      UPGRADE_LH_ENGINE_IMAGE="longhornio/longhorn-engine:${FLEET_REPO_TRANSIENT_VERSION}"
+      run_longhorn_upgrade_test
+      LONGHORN_UPGRADE_TYPE="from_transient"
+      LONGHORN_UPGRADE_TEST_POD_NAME="longhorn-test-upgrade-from-transient"
+    fi
+    UPGRADE_LH_REPO_URL="${FLEET_REPO_URI}"
+    UPGRADE_LH_REPO_BRANCH="${FLEET_REPO_VERSION}"
+    UPGRADE_LH_ENGINE_IMAGE="longhornio/longhorn-engine:${FLEET_REPO_VERSION}"
+    run_longhorn_upgrade_test
+    run_longhorn_test
+  else
+    create_fleet_git_repo
+    run_longhorn_test
+  fi
+}
+
+main

pipelines/fleet/scripts/upgrade-longhorn.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -x
+
+export FLEET_REPO_URI="${1}"
+export FLEET_REPO_VERSION="${2}"
+
+source pipelines/utilities/fleet.sh
+
+export LONGHORN_NAMESPACE="longhorn-system"
+
+create_fleet_git_repo

pipelines/utilities/fleet.sh

@@ -0,0 +1,61 @@
+source pipelines/utilities/longhorn_status.sh
+
+
+install_fleet(){
+  helm repo add fleet https://rancher.github.io/fleet-helm-charts/
+  helm -n cattle-fleet-system install --create-namespace --wait fleet-crd fleet/fleet-crd
+  helm -n cattle-fleet-system install --create-namespace --wait fleet fleet/fleet
+}
+
+
+create_fleet_git_repo(){
+  REVISION="${1:-${FLEET_REPO_VERSION}}"
+  cat > longhorn-gitrepo.yaml <<EOF
+apiVersion: fleet.cattle.io/v1alpha1
+kind: GitRepo
+metadata:
+  name: longhorn
+  # This namespace is special and auto-wired to deploy to the local cluster
+  namespace: fleet-local
+spec:
+  # Everything from this repo will be run in this cluster. You trust me right?
+  repo: ${FLEET_REPO_URI}
+  revision: ${REVISION}
+  paths:
+  - .
+EOF
+  kubectl apply -f longhorn-gitrepo.yaml
+  wait_for_bundle_deployment_applied
+  wait_for_bundle_deployment_ready
+  wait_longhorn_status_running
+}
+
+
+wait_for_bundle_deployment_applied(){
+  local RETRY_COUNTS=60 # in seconds
+  local RETRY_INTERVAL="1s"
+
+  RETRIES=0
+  while [[ $(kubectl -n fleet-local get gitrepo longhorn -o jsonpath='{.status.summary.waitApplied}') != 1 ]]; do
+    echo "Wait for fleet bundle deployment applied ... re-checking in 1s"
+    sleep ${RETRY_INTERVAL}
+    RETRIES=$((RETRIES+1))
+
+    if [[ ${RETRIES} -eq ${RETRY_COUNTS} ]]; then echo "Error: apply fleet bundle deployment timeout"; exit 1 ; fi
+  done
+}
+
+
+wait_for_bundle_deployment_ready(){
+  local RETRY_COUNTS=10 # in minutes
+  local RETRY_INTERVAL="1m"
+
+  RETRIES=0
+  while [[ $(kubectl -n fleet-local get gitrepo longhorn -o jsonpath='{.status.readyClusters}') != 1 ]]; do
+    echo "Wait for fleet bundle deployment ready ... re-checking in 1m"
+    sleep ${RETRY_INTERVAL}
+    RETRIES=$((RETRIES+1))
+
+    if [[ ${RETRIES} -eq ${RETRY_COUNTS} ]]; then echo "Error: fleet bundle deployment timeout"; exit 1 ; fi
+  done
+}