ci: implement flux install/upgrade regression test pipeline

Signed-off-by: Yang Chiu <[email protected]>

manager/integration/Dockerfile

@@ -21,7 +21,10 @@ RUN curl -sO https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_
     wget -q https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_${ARCH}.zip && \
     unzip terraform_${TERRAFORM_VERSION}_linux_${ARCH}.zip && rm terraform_${TERRAFORM_VERSION}_linux_${ARCH}.zip && \
     mv terraform /usr/bin/terraform && \
-    chmod +x /usr/bin/terraform
+    chmod +x /usr/bin/terraform && \
+    curl -s -o flux_install.sh https://fluxcd.io/install.sh && \
+    chmod 700 flux_install.sh && \
+    ./flux_install.sh
 
 RUN curl -L https://github.com/jonelo/jacksum/releases/download/v3.4.0/jacksum-3.4.0.jar --output /jacksum.jar
 
@@ -34,5 +37,6 @@ WORKDIR /integration/tests
 ADD pipelines/utilities ./pipelines/utilities
 ADD pipelines/helm/scripts/upgrade-longhorn.sh ./pipelines/helm/scripts/upgrade-longhorn.sh
 ADD pipelines/rancher/scripts/upgrade-longhorn.sh ./pipelines/rancher/scripts/upgrade-longhorn.sh
+ADD pipelines/flux/scripts/upgrade-longhorn.sh ./pipelines/flux/scripts/upgrade-longhorn.sh
 
 ENTRYPOINT ["./run.sh"]

manager/integration/deploy/test.yaml

@@ -5,50 +5,13 @@ metadata:
   namespace: default
 ---
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: longhorn-test-role
-rules:
-- apiGroups: [""]
-  resources: ["nodes", "nodes/status", "pods", "pods/exec", "persistentvolumes", "persistentvolumeclaims",
-              "persistentvolumeclaims/status", "secrets", "services", "serviceaccounts", "namespaces", "configmaps"]
-  verbs: ["*"]
-- apiGroups: ["scheduling.k8s.io"]
-  resources: ["priorityclasses"]
-  verbs: ["*"]
-- apiGroups: ["storage.k8s.io"]
-  resources: ["storageclasses"]
-  verbs: ["*"]
-- apiGroups: ["apps"]
-  resources: ["statefulsets", "deployments", "daemonsets"]
-  verbs: ["*"]
-- apiGroups: ["rbac.authorization.k8s.io"]
-  resources: ["clusterroles", "clusterrolebindings", "rolebindings", "roles"]
-  verbs: ["*"]
-- apiGroups: ["apiextensions.k8s.io"]
-  resources: ["customresourcedefinitions"]
-  verbs: ["*"]
-- apiGroups: ["policy"]
-  resources: ["podsecuritypolicies", "poddisruptionbudgets"]
-  verbs: ["*"]
-- apiGroups: ["snapshot.storage.k8s.io"]
-  resources: ["*"]
-  verbs: ["*"]
-- apiGroups: ["batch"]
-  resources: ["cronjobs", "jobs"]
-  verbs: ["*"]
-- apiGroups: ["longhorn.io"]
-  resources: ["supportbundles", "supportbundles/status"]
-  verbs: ["*"]
----
-apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   name: longhorn-test-bind
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: longhorn-test-role
+  name: cluster-admin
 subjects:
 - kind: ServiceAccount
   name: longhorn-test-service-account

manager/integration/tests/common.py

@@ -4444,7 +4444,6 @@ def wait_for_backup_restore_completed(client, name, backup_name):
     complete = False
     for i in range(RETRY_COUNTS):
         v = client.by_id_volume(name)
-        print(f"volume = {v}")
         if v.controllers and len(v.controllers) != 0 and \
                 v.controllers[0].lastRestoredBackup == backup_name:
             complete = True

manager/integration/tests/conftest.py

@@ -22,6 +22,8 @@
 RANCHER_SECRET_KEY = "--rancher-secret-key"
 RANCHER_CHART_INSTALL_VERSION = "--rancher-chart-install-version"
 LONGHORN_REPO = "--longhorn-repo"
+FLUX_HELM_CHART_URL = "--flux-helm-chart-url"
+FLUX_HELM_CHART_VERSION = "--flux-helm-chart-version"
 UPGRADE_LH_REPO_URL = "--upgrade-lh-repo-url"
 UPGRADE_LH_REPO_BRANCH = "--upgrade-lh-repo-branch"
 UPGRADE_LH_MANAGER_IMAGE = "--upgrade-lh-manager-image"
@@ -91,6 +93,16 @@ def pytest_addoption(parser):
                      the longhorn dockerhub repo of longhorn components like
                      longhornio or rancher''')
 
+    parser.addoption(FLUX_HELM_CHART_URL, action="store",
+                     default="https://charts.longhorn.io",
+                     help='''if longhorn install method is flux, specify the
+                     url of flux helm repository resource''')
+
+    parser.addoption(FLUX_HELM_CHART_VERSION, action="store",
+                     default="",
+                     help='''if longhorn install method is flux, specify the
+                     chart version when create flux helm release resource''')
+
     longhorn_repo_url =\
         "https://github.com/longhorn/longhorn.git"
     parser.addoption(UPGRADE_LH_REPO_URL, action="store",

manager/integration/tests/test_upgrade.py

@@ -90,6 +90,16 @@ def longhorn_repo(request):
     return request.config.getoption("--longhorn-repo")
 
 
+@pytest.fixture
+def flux_helm_chart_url(request):
+    return request.config.getoption("--flux-helm-chart-url")
+
+
+@pytest.fixture
+def flux_helm_chart_version(request):
+    return request.config.getoption("--flux-helm-chart-version")
+
+
 @pytest.fixture
 def upgrade_longhorn_repo_url(request):
     return request.config.getoption("--upgrade-lh-repo-url")
@@ -142,6 +152,8 @@ def longhorn_upgrade(longhorn_install_method,
                      rancher_secret_key,
                      rancher_chart_install_version,
                      longhorn_repo,
+                     flux_helm_chart_url,
+                     flux_helm_chart_version,
                      longhorn_repo_url,
                      longhorn_repo_branch,
                      longhorn_manager_image,
@@ -152,12 +164,17 @@ def longhorn_upgrade(longhorn_install_method,
 
     if longhorn_install_method == "manifest":
         command = "../scripts/upgrade-longhorn.sh"
+        process = subprocess.Popen([command,
+                                    longhorn_repo_url,
+                                    longhorn_repo_branch,
+                                    longhorn_manager_image,
+                                    longhorn_engine_image,
+                                    longhorn_instance_manager_image,
+                                    longhorn_share_manager_image,
+                                    longhorn_backing_image_manager_image],
+                                   shell=False)
     elif longhorn_install_method == "helm":
         command = "./pipelines/helm/scripts/upgrade-longhorn.sh"
-    elif longhorn_install_method == "rancher":
-        command = "./pipelines/rancher/scripts/upgrade-longhorn.sh"
-
-    if longhorn_install_method != "rancher":
         process = subprocess.Popen([command,
                                     longhorn_repo_url,
                                     longhorn_repo_branch,
@@ -167,14 +184,22 @@ def longhorn_upgrade(longhorn_install_method,
                                     longhorn_share_manager_image,
                                     longhorn_backing_image_manager_image],
                                    shell=False)
-    else:
+    elif longhorn_install_method == "rancher":
+        command = "./pipelines/rancher/scripts/upgrade-longhorn.sh"
         process = subprocess.Popen([command,
                                     rancher_hostname,
                                     rancher_access_key,
                                     rancher_secret_key,
                                     rancher_chart_install_version,
                                     longhorn_repo],
                                    shell=False)
+    elif longhorn_install_method == "flux":
+        command = "./pipelines/flux/scripts/upgrade-longhorn.sh"
+        process = subprocess.Popen([command,
+                                    flux_helm_chart_url,
+                                    flux_helm_chart_version],
+                                   shell=False)
+
     process.wait()
     if process.returncode == 0:
         longhorn_upgraded = True
@@ -193,6 +218,8 @@ def test_upgrade(longhorn_upgrade_type,
                  rancher_secret_key,
                  rancher_chart_install_version,
                  longhorn_repo,
+                 flux_helm_chart_url,
+                 flux_helm_chart_version,
                  upgrade_longhorn_repo_url,
                  upgrade_longhorn_repo_branch,
                  upgrade_longhorn_manager_image,
@@ -377,6 +404,8 @@ def test_upgrade(longhorn_upgrade_type,
                             rancher_secret_key,
                             rancher_chart_install_version,
                             longhorn_repo,
+                            flux_helm_chart_url,
+                            flux_helm_chart_version,
                             longhorn_repo_url,
                             longhorn_repo_branch,
                             longhorn_manager_image,

pipelines/flux/Dockerfile.setup

@@ -0,0 +1,37 @@
+From alpine:latest
+
+ARG KUBECTL_VERSION=v1.20.2
+
+ARG RKE_VERSION=v1.3.4
+
+ARG TERRAFORM_VERSION=1.3.5
+
+ARG YQ_VERSION=v4.24.2
+
+ENV WORKSPACE /src/longhorn-tests
+
+WORKDIR $WORKSPACE
+
+RUN wget -q https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl && \
+    mv kubectl /usr/local/bin/kubectl && \
+    chmod +x /usr/local/bin/kubectl && \
+    wget -q https://github.com/rancher/rke/releases/download/$RKE_VERSION/rke_linux-amd64 && \
+    mv rke_linux-amd64 /usr/bin/rke && \
+    chmod +x /usr/bin/rke && \
+    wget -q https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \
+    unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip && rm terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \
+    mv terraform /usr/bin/terraform && \
+    chmod +x /usr/bin/terraform && \
+    wget -q "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" && \
+    mv yq_linux_amd64 /usr/local/bin/yq && \
+    chmod +x /usr/local/bin/yq && \
+    apk add openssl openssh-client ca-certificates git rsync bash curl jq python3 py3-pip && \
+    ssh-keygen -t rsa -b 4096 -N "" -f ~/.ssh/id_rsa && \
+    curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 && \
+    chmod 700 get_helm.sh && \
+    ./get_helm.sh && \
+    curl -s -o flux_install.sh https://fluxcd.io/install.sh && \
+    chmod 700 flux_install.sh && \
+    ./flux_install.sh
+
+COPY [".", "$WORKSPACE"]