ci: implement argocd install/upgrade regression test pipeline

Signed-off-by: Yang Chiu <[email protected]>

manager/integration/Dockerfile

@@ -24,7 +24,10 @@ RUN curl -sO https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_
     chmod +x /usr/bin/terraform && \
     curl -s -o flux_install.sh https://fluxcd.io/install.sh && \
     chmod 700 flux_install.sh && \
-    ./flux_install.sh
+    ./flux_install.sh && \
+    curl -sSL -o argocd-linux-${ARCH} https://github.com/argoproj/argo-cd/releases/latest/download/argocd-linux-${ARCH} && \
+    install -m 555 argocd-linux-${ARCH} /usr/local/bin/argocd && \
+    rm argocd-linux-${ARCH}
 
 RUN curl -L https://github.com/jonelo/jacksum/releases/download/v3.4.0/jacksum-3.4.0.jar --output /jacksum.jar
 
@@ -38,5 +41,6 @@ ADD pipelines/utilities ./pipelines/utilities
 ADD pipelines/helm/scripts/upgrade-longhorn.sh ./pipelines/helm/scripts/upgrade-longhorn.sh
 ADD pipelines/rancher/scripts/upgrade-longhorn.sh ./pipelines/rancher/scripts/upgrade-longhorn.sh
 ADD pipelines/flux/scripts/upgrade-longhorn.sh ./pipelines/flux/scripts/upgrade-longhorn.sh
+ADD pipelines/argocd/scripts/upgrade-longhorn.sh ./pipelines/argocd/scripts/upgrade-longhorn.sh
 
 ENTRYPOINT ["./run.sh"]

manager/integration/tests/test_upgrade.py

@@ -199,6 +199,12 @@ def longhorn_upgrade(longhorn_install_method,
                                     flux_helm_chart_url,
                                     flux_helm_chart_version],
                                    shell=False)
+    elif longhorn_install_method == "argocd":
+        command = "./pipelines/argocd/scripts/upgrade-longhorn.sh"
+        process = subprocess.Popen([command,
+                                    longhorn_repo_url,
+                                    longhorn_repo_branch],
+                                   shell=False)
 
     process.wait()
     if process.returncode == 0:
@@ -265,11 +271,6 @@ def test_upgrade(longhorn_upgrade_type,
     18. Verify the vol_rebuild is still healthy
     """
     longhorn_install_method = longhorn_install_method
-    rancher_hostname = rancher_hostname
-    rancher_access_key = rancher_access_key
-    rancher_secret_key = rancher_secret_key
-    rancher_chart_install_version = rancher_chart_install_version
-    longhorn_repo = longhorn_repo
     longhorn_repo_url = upgrade_longhorn_repo_url
     longhorn_repo_branch = upgrade_longhorn_repo_branch
     longhorn_manager_image = upgrade_longhorn_manager_image

pipelines/argocd/Dockerfile.setup

@@ -0,0 +1,34 @@
+From alpine:latest
+
+ARG KUBECTL_VERSION=v1.20.2
+
+ARG RKE_VERSION=v1.3.4
+
+ARG TERRAFORM_VERSION=1.3.5
+
+ARG YQ_VERSION=v4.24.2
+
+ENV WORKSPACE /src/longhorn-tests
+
+WORKDIR $WORKSPACE
+
+RUN wget -q https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl && \
+    mv kubectl /usr/local/bin/kubectl && \
+    chmod +x /usr/local/bin/kubectl && \
+    wget -q https://github.com/rancher/rke/releases/download/$RKE_VERSION/rke_linux-amd64 && \
+    mv rke_linux-amd64 /usr/bin/rke && \
+    chmod +x /usr/bin/rke && \
+    wget -q https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \
+    unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip && rm terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \
+    mv terraform /usr/bin/terraform && \
+    chmod +x /usr/bin/terraform && \
+    wget -q "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" && \
+    mv yq_linux_amd64 /usr/local/bin/yq && \
+    chmod +x /usr/local/bin/yq && \
+    apk add openssl openssh-client ca-certificates git rsync bash curl jq python3 py3-pip && \
+    ssh-keygen -t rsa -b 4096 -N "" -f ~/.ssh/id_rsa && \
+    curl -sSL -o argocd-linux-amd64 https://github.com/argoproj/argo-cd/releases/latest/download/argocd-linux-amd64 && \
+    install -m 555 argocd-linux-amd64 /usr/local/bin/argocd && \
+    rm argocd-linux-amd64
+
+COPY [".", "$WORKSPACE"]

pipelines/argocd/Jenkinsfile

@@ -0,0 +1,204 @@
+def imageName = "${JOB_BASE_NAME}-${env.BUILD_NUMBER}"
+def summary
+def WORKSPACE = "/src/longhorn-tests"
+def BUILD_TRIGGER_BY = "\n${currentBuild.getBuildCauses()[0].shortDescription}"
+
+// define optional parameters
+def SELINUX_MODE = params.SELINUX_MODE ? params.SELINUX_MODE : ""
+
+def CREDS_ID = JOB_BASE_NAME == "longhorn-tests-regression" ? "AWS_CREDS_RANCHER_QA" : "AWS_CREDS"
+def REGISTRATION_CODE_ID = params.ARCH == "amd64" ? "REGISTRATION_CODE" : "REGISTRATION_CODE_ARM64"
+
+// parameters for air gap installation
+def AIR_GAP_INSTALLATION = params.AIR_GAP_INSTALLATION ? params.AIR_GAP_INSTALLATION : false
+def CIS_HARDENING = params.CIS_HARDENING ? params.CIS_HARDENING : false
+def REGISTRY_URL
+def REGISTRY_USERNAME
+def REGISTRY_PASSWORD
+
+// parameter for hdd test
+def USE_HDD = params.USE_HDD ? params.USE_HDD : false
+
+node {
+
+    withCredentials([
+        usernamePassword(credentialsId: CREDS_ID, passwordVariable: 'AWS_SECRET_KEY', usernameVariable: 'AWS_ACCESS_KEY'),
+        string(credentialsId: 'DO_CREDS', variable: 'DO_TOKEN'),
+        string(credentialsId: REGISTRATION_CODE_ID, variable: 'REGISTRATION_CODE'),
+    ]) {
+
+        if (params.SEND_SLACK_NOTIFICATION) {
+            notifyBuild('STARTED', BUILD_TRIGGER_BY, params.NOTIFY_SLACK_CHANNEL)
+        }
+
+        checkout scm
+
+        try {
+
+            if (params.AIR_GAP_INSTALLATION) {
+
+                stage('airgap build') {
+                    sh "airgap/scripts/build.sh"
+                    sh """ docker run -itd --name airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} \
+                                           --env TF_VAR_longhorn_version=${LONGHORN_INSTALL_VERSION} \
+                                           --env TF_VAR_do_token=${DO_TOKEN} \
+                                           --env TF_VAR_aws_access_key=${AWS_ACCESS_KEY} \
+                                           --env TF_VAR_aws_secret_key=${AWS_SECRET_KEY} \
+                                           airgap-${JOB_BASE_NAME}-${BUILD_NUMBER}
+                    """
+                }
+
+                stage ('airgap setup') {
+                    sh "docker exec airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} ./airgap/scripts/terraform-setup.sh"
+                    REGISTRY_URL = sh (
+                        script: "docker exec airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} terraform -chdir=./airgap/terraform output -raw registry_url",
+                        returnStdout: true
+                    )
+                    println REGISTRY_URL
+                    REGISTRY_USERNAME = sh (
+                        script: "docker exec airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} terraform -chdir=./airgap/terraform output -raw registry_username",
+                        returnStdout: true
+                    )
+                    REGISTRY_PASSWORD = sh (
+                        script: "docker exec airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} terraform -chdir=./airgap/terraform output -raw registry_password",
+                        returnStdout: true
+                    )
+                }
+
+            }
+
+            stage('build') {
+
+                echo "Using credentials: $CREDS_ID"
+                echo "Using registration coce: $REGISTRATION_CODE_ID"
+
+                sh "pipelines/argocd/scripts/build.sh"
+                sh """ docker run -itd --name ${JOB_BASE_NAME}-${BUILD_NUMBER} \
+                                       --env AIR_GAP_INSTALLATION=${AIR_GAP_INSTALLATION} \
+                                       --env REGISTRY_URL=${REGISTRY_URL} \
+                                       --env REGISTRY_USERNAME=${REGISTRY_USERNAME} \
+                                       --env REGISTRY_PASSWORD=${REGISTRY_PASSWORD} \
+                                       --env LONGHORN_REPO_URI=${LONGHORN_REPO_URI} \
+                                       --env LONGHORN_INSTALL_VERSION=${LONGHORN_INSTALL_VERSION} \
+                                       --env LONGHORN_TESTS_CUSTOM_IMAGE=${LONGHORN_TESTS_CUSTOM_IMAGE} \
+                                       --env DISTRO=${DISTRO} \
+                                       --env LONGHORN_STABLE_VERSION=${LONGHORN_STABLE_VERSION} \
+                                       --env LONGHORN_TRANSIENT_VERSION=${LONGHORN_TRANSIENT_VERSION} \
+                                       --env LONGHORN_TEST_CLOUDPROVIDER=${LONGHORN_TEST_CLOUDPROVIDER} \
+                                       --env LONGHORN_UPGRADE_TEST=${LONGHORN_UPGRADE_TEST} \
+                                       --env PYTEST_CUSTOM_OPTIONS="${PYTEST_CUSTOM_OPTIONS}" \
+                                       --env BACKUP_STORE_TYPE="${BACKUP_STORE_TYPE}" \
+                                       --env TF_VAR_use_hdd=${USE_HDD} \
+                                       --env TF_VAR_arch=${ARCH} \
+                                       --env TF_VAR_k8s_distro_name=${K8S_DISTRO_NAME} \
+                                       --env TF_VAR_k8s_distro_version=${K8S_DISTRO_VERSION} \
+                                       --env TF_VAR_aws_availability_zone=${AWS_AVAILABILITY_ZONE} \
+                                       --env TF_VAR_aws_region=${AWS_REGION} \
+                                       --env TF_VAR_os_distro_version=${DISTRO_VERSION} \
+                                       --env TF_VAR_do_token=${env.TF_VAR_do_token} \
+                                       --env TF_VAR_lh_aws_access_key=${AWS_ACCESS_KEY} \
+                                       --env TF_VAR_lh_aws_instance_name_controlplane="${JOB_BASE_NAME}-ctrl" \
+                                       --env TF_VAR_lh_aws_instance_name_worker="${JOB_BASE_NAME}-wrk" \
+                                       --env TF_VAR_lh_aws_instance_type_controlplane=${CONTROLPLANE_INSTANCE_TYPE} \
+                                       --env TF_VAR_lh_aws_instance_type_worker=${WORKER_INSTANCE_TYPE}\
+                                       --env TF_VAR_lh_aws_secret_key=${AWS_SECRET_KEY} \
+                                       --env TF_VAR_selinux_mode=${SELINUX_MODE} \
+                                       --env TF_VAR_registration_code=${REGISTRATION_CODE} \
+                                       --env TF_VAR_cis_hardening=${CIS_HARDENING} \
+                                       ${imageName}
+                """
+            }
+
+            timeout(60) {
+                stage ('terraform') {
+                    sh "docker exec ${JOB_BASE_NAME}-${BUILD_NUMBER} pipelines/utilities/terraform_setup.sh"
+                }
+			}
+
+            stage ('longhorn setup & tests') {
+                sh "docker exec ${JOB_BASE_NAME}-${BUILD_NUMBER} pipelines/argocd/scripts/longhorn-setup.sh"
+            }
+
+            stage ('download support bundle') {
+                sh "docker exec ${JOB_BASE_NAME}-${BUILD_NUMBER} pipelines/utilities/download_support_bundle.sh  ${JOB_BASE_NAME}-${BUILD_NUMBER}-bundle.zip"
+                sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/${JOB_BASE_NAME}-${BUILD_NUMBER}-bundle.zip ."
+				archiveArtifacts allowEmptyArchive: true, artifacts: '**/*.zip', followSymlinks: false
+			}
+
+            stage ('report generation') {
+                sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/longhorn-test-junit-report.xml ."
+
+                if(params.LONGHORN_UPGRADE_TEST && params.LONGHORN_TRANSIENT_VERSION) {
+                    sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/longhorn-test-upgrade-from-stable-junit-report.xml ."
+                    sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/longhorn-test-upgrade-from-transient-junit-report.xml ."
+                    summary = junit 'longhorn-test-upgrade-from-stable-junit-report.xml, longhorn-test-upgrade-from-transient-junit-report.xml, longhorn-test-junit-report.xml'
+                }
+                else if(params.LONGHORN_UPGRADE_TEST) {
+                    sh "docker cp ${JOB_BASE_NAME}-${BUILD_NUMBER}:${WORKSPACE}/longhorn-test-upgrade-from-stable-junit-report.xml ."
+                    summary = junit 'longhorn-test-upgrade-from-stable-junit-report.xml, longhorn-test-junit-report.xml'
+                }
+                else {
+                    summary = junit 'longhorn-test-junit-report.xml'
+                }
+            }
+
+        } catch (e) {
+            currentBuild.result = "FAILED"
+            throw e
+        } finally {
+            stage ('releasing resources') {
+                if (sh (script: "docker container inspect airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} > /dev/null 2>&1", returnStatus: true) == 0) {
+                    sh "docker exec airgap-${JOB_BASE_NAME}-${BUILD_NUMBER} ./airgap/scripts/cleanup.sh"
+                    sh "docker stop airgap-${JOB_BASE_NAME}-${BUILD_NUMBER}"
+                    sh "docker rm -v airgap-${JOB_BASE_NAME}-${BUILD_NUMBER}"
+                    sh "docker rmi airgap-${JOB_BASE_NAME}-${BUILD_NUMBER}"
+                }
+
+                if (sh (script: "docker container inspect ${JOB_BASE_NAME}-${BUILD_NUMBER} > /dev/null 2>&1", returnStatus: true) == 0) {
+                    sh "docker exec ${JOB_BASE_NAME}-${BUILD_NUMBER} pipelines/utilities/cleanup.sh"
+                    sh "docker stop ${JOB_BASE_NAME}-${BUILD_NUMBER}"
+                    sh "docker rm -v ${JOB_BASE_NAME}-${BUILD_NUMBER}"
+                    sh "docker rmi ${imageName}"
+                }
+
+                if (summary) {
+                    summary_msg = "\nTest Summary - Failures: ${summary.failCount}, Skipped: ${summary.skipCount}, Passed: ${summary.passCount}  -- Job completed in ${currentBuild.durationString.replace(' and counting', '')}"
+                } else {
+                    summary_msg = "\n Test Failed: No Junit report"
+                }
+
+                if(params.SEND_SLACK_NOTIFICATION){
+                    notifyBuild(currentBuild.result, summary_msg, params.NOTIFY_SLACK_CHANNEL)
+                }
+            }
+        }
+    }
+
+}
+
+
+def notifyBuild(String buildStatus = 'STARTED', String summary_msg, String slack_channel) {
+  // build status of null means successful
+  buildStatus =  buildStatus ?: 'SUCCESSFUL'
+
+  // Default values
+  def colorName = 'RED'
+  def colorCode = '#FF0000'
+  def subject = "${buildStatus}: Job '${env.JOB_BASE_NAME} [${env.BUILD_NUMBER}]'"
+  def summary = "${subject} (${env.BUILD_URL})" + summary_msg
+
+  // Override default values based on build status
+  if (buildStatus == 'STARTED') {
+    color = 'YELLOW'
+    colorCode = '#FFFF00'
+  } else if (buildStatus == 'SUCCESSFUL') {
+    color = 'GREEN'
+    colorCode = '#00FF00'
+  } else {
+    color = 'RED'
+    colorCode = '#FF0000'
+  }
+
+  // Send notifications
+  slackSend (color: colorCode, message: summary, channel: slack_channel,  tokenCredentialId: 'longhorn-tests-slack-token')
+}

pipelines/argocd/scripts/build.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker build --no-cache -f ./pipelines/argocd/Dockerfile.setup -t "${JOB_BASE_NAME}-${BUILD_NUMBER}" .

pipelines/argocd/scripts/longhorn-setup.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+set -x
+
+source pipelines/utilities/kubeconfig.sh
+source pipelines/utilities/selinux_workaround.sh
+source pipelines/utilities/install_csi_snapshotter.sh
+source pipelines/utilities/create_aws_secret.sh
+source pipelines/utilities/install_backupstores.sh
+source pipelines/utilities/create_longhorn_namespace.sh
+source pipelines/utilities/argocd.sh
+source pipelines/utilities/run_longhorn_test.sh
+
+
+export LONGHORN_NAMESPACE="longhorn-system"
+export LONGHORN_INSTALL_METHOD="argocd"
+
+
+main(){
+  set_kubeconfig
+
+  if [[ ${DISTRO} == "rhel" ]] || [[ ${DISTRO} == "rockylinux" ]] || [[ ${DISTRO} == "oracle" ]]; then
+    apply_selinux_workaround
+  fi
+
+  # set debugging mode off to avoid leaking aws secrets to the logs.
+  # DON'T REMOVE!
+  set +x
+  create_aws_secret
+  set -x
+
+  create_longhorn_namespace
+  install_backupstores
+  install_csi_snapshotter
+
+  install_argocd
+  init_argocd
+
+  if [[ "${LONGHORN_UPGRADE_TEST}" == true ]]; then
+    create_argocd_app "${LONGHORN_STABLE_VERSION}"
+    sync_argocd_app
+    LONGHORN_UPGRADE_TYPE="from_stable"
+    LONGHORN_UPGRADE_TEST_POD_NAME="longhorn-test-upgrade-from-stable"
+    if [[ -n "${LONGHORN_TRANSIENT_VERSION}" ]]; then
+      UPGRADE_LH_REPO_URL="${LONGHORN_REPO_URI}"
+      UPGRADE_LH_REPO_BRANCH="${LONGHORN_TRANSIENT_VERSION}"
+      UPGRADE_LH_ENGINE_IMAGE="longhornio/longhorn-engine:${LONGHORN_TRANSIENT_VERSION}"
+      run_longhorn_upgrade_test
+      LONGHORN_UPGRADE_TYPE="from_transient"
+      LONGHORN_UPGRADE_TEST_POD_NAME="longhorn-test-upgrade-from-transient"
+    fi
+    UPGRADE_LH_REPO_URL="${LONGHORN_REPO_URI}"
+    UPGRADE_LH_REPO_BRANCH="${LONGHORN_INSTALL_VERSION}"
+    UPGRADE_LH_ENGINE_IMAGE="longhornio/longhorn-engine:${LONGHORN_INSTALL_VERSION}"
+    run_longhorn_upgrade_test
+    run_longhorn_test
+  else
+    create_argocd_app
+    sync_argocd_app
+    run_longhorn_test
+  fi
+}
+
+main

pipelines/argocd/scripts/upgrade-longhorn.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -x
+
+export LONGHORN_REPO_URI="${1}"
+export LONGHORN_INSTALL_VERSION="${2}"
+
+source pipelines/utilities/argocd.sh
+source pipelines/utilities/kubeconfig.sh
+
+export LONGHORN_NAMESPACE="longhorn-system"
+
+construct_kubeconfig
+
+init_argocd
+
+kubectl config get-contexts
+kubectl config view
+
+update_argocd_app_target_revision "${LONGHORN_INSTALL_VERSION}"
+sync_argocd_app