Merge pull request #1476 from yangchiu/rancher-pipeline

ci: implement rancher chart install/upgrade regression test pipeline

manager/integration/Dockerfile

@@ -2,10 +2,11 @@ FROM registry.suse.com/bci/python:3.9
 
 ARG KUBECTL_VERSION=v1.17.0
 ARG YQ_VERSION=v4.24.2
+ARG TERRAFORM_VERSION=1.3.5
 ARG ARCH=amd64
 
 RUN zypper ref -f
-RUN zypper in -y vim-small nfs-client xfsprogs e2fsprogs util-linux-systemd gcc python39-devel gawk java-11-openjdk tar awk gzip wget && \
+RUN zypper in -y vim-small nfs-client xfsprogs e2fsprogs util-linux-systemd gcc python39-devel gawk java-11-openjdk tar awk gzip wget unzip && \
     rm -rf /var/cache/zypp/*
 
 RUN curl -sO https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_VERSION/bin/linux/${ARCH}/kubectl && \
@@ -16,7 +17,11 @@ RUN curl -sO https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_
     ./get_helm.sh && \
     wget -q "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${ARCH}" && \
     mv yq_linux_${ARCH} /usr/local/bin/yq && \
-    chmod +x /usr/local/bin/yq
+    chmod +x /usr/local/bin/yq && \
+    wget -q https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_${ARCH}.zip && \
+    unzip terraform_${TERRAFORM_VERSION}_linux_${ARCH}.zip && rm terraform_${TERRAFORM_VERSION}_linux_${ARCH}.zip && \
+    mv terraform /usr/bin/terraform && \
+    chmod +x /usr/bin/terraform
 
 RUN curl -L https://github.com/jonelo/jacksum/releases/download/v3.4.0/jacksum-3.4.0.jar --output /jacksum.jar
 
@@ -28,5 +33,6 @@ WORKDIR /integration/tests
 
 ADD pipelines/utilities ./pipelines/utilities
 ADD pipelines/helm/scripts/upgrade-longhorn.sh ./pipelines/helm/scripts/upgrade-longhorn.sh
+ADD pipelines/rancher/scripts/upgrade-longhorn.sh ./pipelines/rancher/scripts/upgrade-longhorn.sh
 
-ENTRYPOINT ["./run.sh"]
+ENTRYPOINT ["./run.sh"]

manager/integration/tests/conftest.py

@@ -17,6 +17,11 @@
 INCLUDE_CA_OPT = "--include-cluster-autoscaler-test"
 
 LH_INSTALL_METHOD = "--lh-install-method"
+RANCHER_HOSTNAME = "--rancher-hostname"
+RANCHER_ACCESS_KEY = "--rancher-access-key"
+RANCHER_SECRET_KEY = "--rancher-secret-key"
+RANCHER_CHART_INSTALL_VERSION = "--rancher-chart-install-version"
+LONGHORN_REPO = "--longhorn-repo"
 UPGRADE_LH_REPO_URL = "--upgrade-lh-repo-url"
 UPGRADE_LH_REPO_BRANCH = "--upgrade-lh-repo-branch"
 UPGRADE_LH_MANAGER_IMAGE = "--upgrade-lh-manager-image"
@@ -57,6 +62,35 @@ def pytest_addoption(parser):
                      to determine how to upgrade longhorn for test_upgrade
                      (default: manifest''')
 
+    parser.addoption(RANCHER_HOSTNAME, action="store",
+                     default="",
+                     help='''if longhorn install method is rancher, specify
+                     where rancher is hosted''')
+
+    parser.addoption(RANCHER_ACCESS_KEY, action="store",
+                     default="",
+                     help='''if longhorn install method is rancher, specify
+                     the access key and secret key to have the permission to
+                     operate it''')
+
+    parser.addoption(RANCHER_SECRET_KEY, action="store",
+                     default="",
+                     help='''if longhorn install method is rancher, specify
+                     the access key and secret key to have the permission to
+                     operate it''')
+
+    parser.addoption(RANCHER_CHART_INSTALL_VERSION, action="store",
+                     default="",
+                     help='''if longhorn install method is rancher, specify
+                     the longhorn chart version you would like to install like
+                     102.2.1+up1.4.2''')
+
+    parser.addoption(LONGHORN_REPO, action="store",
+                     default="",
+                     help='''if longhorn install method is rancher, specify
+                     the longhorn dockerhub repo of longhorn components like
+                     longhornio or rancher''')
+
     longhorn_repo_url =\
         "https://github.com/longhorn/longhorn.git"
     parser.addoption(UPGRADE_LH_REPO_URL, action="store",

manager/integration/tests/test_upgrade.py

@@ -65,6 +65,31 @@ def longhorn_install_method(request):
     return request.config.getoption("--lh-install-method")
 
 
+@pytest.fixture
+def rancher_hostname(request):
+    return request.config.getoption("--rancher-hostname")
+
+
+@pytest.fixture
+def rancher_access_key(request):
+    return request.config.getoption("--rancher-access-key")
+
+
+@pytest.fixture
+def rancher_secret_key(request):
+    return request.config.getoption("--rancher-secret-key")
+
+
+@pytest.fixture
+def rancher_chart_install_version(request):
+    return request.config.getoption("--rancher-chart-install-version")
+
+
+@pytest.fixture
+def longhorn_repo(request):
+    return request.config.getoption("--longhorn-repo")
+
+
 @pytest.fixture
 def upgrade_longhorn_repo_url(request):
     return request.config.getoption("--upgrade-lh-repo-url")
@@ -112,6 +137,11 @@ def longhorn_upgrade_type():
 
 
 def longhorn_upgrade(longhorn_install_method,
+                     rancher_hostname,
+                     rancher_access_key,
+                     rancher_secret_key,
+                     rancher_chart_install_version,
+                     longhorn_repo,
                      longhorn_repo_url,
                      longhorn_repo_branch,
                      longhorn_manager_image,
@@ -124,15 +154,27 @@ def longhorn_upgrade(longhorn_install_method,
         command = "../scripts/upgrade-longhorn.sh"
     elif longhorn_install_method == "helm":
         command = "./pipelines/helm/scripts/upgrade-longhorn.sh"
-    process = subprocess.Popen([command,
-                                longhorn_repo_url,
-                                longhorn_repo_branch,
-                                longhorn_manager_image,
-                                longhorn_engine_image,
-                                longhorn_instance_manager_image,
-                                longhorn_share_manager_image,
-                                longhorn_backing_image_manager_image],
-                               shell=False)
+    elif longhorn_install_method == "rancher":
+        command = "./pipelines/rancher/scripts/upgrade-longhorn.sh"
+
+    if longhorn_install_method != "rancher":
+        process = subprocess.Popen([command,
+                                    longhorn_repo_url,
+                                    longhorn_repo_branch,
+                                    longhorn_manager_image,
+                                    longhorn_engine_image,
+                                    longhorn_instance_manager_image,
+                                    longhorn_share_manager_image,
+                                    longhorn_backing_image_manager_image],
+                                   shell=False)
+    else:
+        process = subprocess.Popen([command,
+                                    rancher_hostname,
+                                    rancher_access_key,
+                                    rancher_secret_key,
+                                    rancher_chart_install_version,
+                                    longhorn_repo],
+                                   shell=False)
     process.wait()
     if process.returncode == 0:
         longhorn_upgraded = True
@@ -146,6 +188,11 @@ def longhorn_upgrade(longhorn_install_method,
 @pytest.mark.upgrade  # NOQA
 def test_upgrade(longhorn_upgrade_type,
                  longhorn_install_method,
+                 rancher_hostname,
+                 rancher_access_key,
+                 rancher_secret_key,
+                 rancher_chart_install_version,
+                 longhorn_repo,
                  upgrade_longhorn_repo_url,
                  upgrade_longhorn_repo_branch,
                  upgrade_longhorn_manager_image,
@@ -191,6 +238,11 @@ def test_upgrade(longhorn_upgrade_type,
     18. Verify the vol_rebuild is still healthy
     """
     longhorn_install_method = longhorn_install_method
+    rancher_hostname = rancher_hostname
+    rancher_access_key = rancher_access_key
+    rancher_secret_key = rancher_secret_key
+    rancher_chart_install_version = rancher_chart_install_version
+    longhorn_repo = longhorn_repo
     longhorn_repo_url = upgrade_longhorn_repo_url
     longhorn_repo_branch = upgrade_longhorn_repo_branch
     longhorn_manager_image = upgrade_longhorn_manager_image
@@ -320,6 +372,11 @@ def test_upgrade(longhorn_upgrade_type,
 
     # upgrade Longhorn manager
     assert longhorn_upgrade(longhorn_install_method,
+                            rancher_hostname,
+                            rancher_access_key,
+                            rancher_secret_key,
+                            rancher_chart_install_version,
+                            longhorn_repo,
                             longhorn_repo_url,
                             longhorn_repo_branch,
                             longhorn_manager_image,

pipelines/rancher/Dockerfile.setup

@@ -0,0 +1,34 @@
+From alpine:latest
+
+ARG KUBECTL_VERSION=v1.20.2
+
+ARG RKE_VERSION=v1.3.4
+
+ARG TERRAFORM_VERSION=1.3.5
+
+ARG YQ_VERSION=v4.24.2
+
+ENV WORKSPACE /src/longhorn-tests
+
+WORKDIR $WORKSPACE
+
+RUN wget -q https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl && \
+    mv kubectl /usr/local/bin/kubectl && \
+    chmod +x /usr/local/bin/kubectl && \
+    wget -q https://github.com/rancher/rke/releases/download/$RKE_VERSION/rke_linux-amd64 && \
+    mv rke_linux-amd64 /usr/bin/rke && \
+    chmod +x /usr/bin/rke && \
+    wget -q https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \
+    unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip && rm terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \
+    mv terraform /usr/bin/terraform && \
+    chmod +x /usr/bin/terraform && \
+    wget -q "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" && \
+    mv yq_linux_amd64 /usr/local/bin/yq && \
+    chmod +x /usr/local/bin/yq && \
+    apk add openssl openssh-client ca-certificates git rsync bash curl jq python3 py3-pip && \
+    ssh-keygen -t rsa -b 4096 -N "" -f ~/.ssh/id_rsa && \
+    curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 && \
+    chmod 700 get_helm.sh && \
+    ./get_helm.sh
+
+COPY [".", "$WORKSPACE"]