test: add Dockerfile and test.yaml to build negative testing image an…

…d run it as a pod Signed-off-by: Yang Chiu <[email protected]>
longhorn · Sep 11, 2023 · 02d398e · 02d398e
1 parent cddb5da
commit 02d398e
Show file tree

Hide file tree

Showing 17 changed files with 473 additions and 452 deletions.
diff --git a/e2e/Dockerfile b/e2e/Dockerfile
@@ -0,0 +1,27 @@
+FROM registry.suse.com/bci/python:3.9
+
+ARG KUBECTL_VERSION=v1.17.0
+ARG YQ_VERSION=v4.24.2
+ARG ARCH=amd64
+
+RUN zypper ref -f
+RUN zypper in -y vim-small nfs-client xfsprogs e2fsprogs util-linux-systemd gcc python39-devel gawk java-11-openjdk tar awk gzip wget && \
+    rm -rf /var/cache/zypp/*
+
+RUN curl -sO https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_VERSION/bin/linux/${ARCH}/kubectl && \
+    mv kubectl /usr/local/bin/kubectl && \
+    chmod +x /usr/local/bin/kubectl && \
+    curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 && \
+    chmod 700 get_helm.sh && \
+    ./get_helm.sh && \
+    wget -q "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${ARCH}" && \
+    mv yq_linux_${ARCH} /usr/local/bin/yq && \
+    chmod +x /usr/local/bin/yq && \
+    curl -L https://github.com/jonelo/jacksum/releases/download/v3.4.0/jacksum-3.4.0.jar --output /jacksum.jar
+
+ADD e2e /e2e
+WORKDIR /e2e
+
+RUN pip install -r requirements.txt
+
+ENTRYPOINT ["./run.sh"]
diff --git a/e2e/deploy/test.yaml b/e2e/deploy/test.yaml
@@ -0,0 +1,97 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: longhorn-test-service-account
+  namespace: default
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: longhorn-test-bind
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: longhorn-test-service-account
+  namespace: default
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: longhorn-test
+  namespace: default
+  labels:
+    longhorn-test: test-job
+spec:
+  containers:
+  - name: longhorn-test
+    image: longhornio/longhorn-e2e-test:master-head
+    #args: [
+    #       "-t", "Replica Rebuilding While Replica Deletion",
+    #       ]
+    #TEST_FRAMEWORK_ARGS_PLACEHOLDER
+    imagePullPolicy: Always
+    securityContext:
+      privileged: true
+    env:
+    - name: LONGHORN_JUNIT_REPORT_PATH
+      value: /tmp/test-report/longhorn-test-junit-report.xml
+    - name: LONGHORN_BACKUPSTORES
+      value: "s3://backupbucket@us-east-1/backupstore$minio-secret, nfs://longhorn-test-nfs-svc.default:/opt/backupstore"
+    - name: LONGHORN_BACKUPSTORE_POLL_INTERVAL
+      value: "30"
+    - name: LONGHORN_DISK_TYPE
+      value: "ssd"
+    - name: LONGHORN_UPGRADE_TYPE
+      value: "from_stable"
+    - name: NODE_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: spec.nodeName
+    - name: MANAGED_K8S_CLUSTER
+      value: "false"
+    volumeMounts:
+    - name: dev
+      mountPath: /dev
+    - name: proc
+      mountPath: /host/proc
+    - name: disk-directory
+      mountPath: /tmp/longhorn-test
+      mountPropagation: Bidirectional
+    - name: longhorn
+      mountPath: /var/lib/longhorn/
+      mountPropagation: Bidirectional
+    - name: test-report
+      mountPath: /tmp/test-report
+  - name: longhorn-test-report
+    image: busybox:1.34.0
+    securityContext:
+      privileged: true
+    command: [ "tail", "-f", "/dev/null" ]
+    volumeMounts:
+    - name: test-report
+      mountPath: /tmp/test-report
+  dnsConfig:
+    nameservers:
+      - 8.8.8.8
+      - 1.1.1.1
+  volumes:
+  - name: dev
+    hostPath:
+      path: /dev/
+  - name: proc
+    hostPath:
+      path: /proc/
+  - name: disk-directory
+    hostPath:
+      path: /tmp/longhorn-test/
+  - name: longhorn
+    hostPath:
+      path: /var/lib/longhorn/
+  - name: test-report
+    hostPath:
+      path: /tmp/test-report/
+  restartPolicy: Never
+  serviceAccountName: longhorn-test-service-account
diff --git a/e2e/keywords/volume.resource b/e2e/keywords/volume.resource
@@ -1,60 +1,34 @@
 *** Settings ***
-Documentation       Longhorn volume related keywords
-
-Library             ../libs/keywords/volume_keywords.py
-Resource            ./node.resource
+Documentation    Volume Keywords
 
+Library    ../libs/keywords/volume_keywords.py
 
 *** Keywords ***
-Create Volume With Fields
-    [Arguments]    ${list_of_fields}
-    ${volume_name}    ${manifest} =    Create Volume Base Manifest
+Create a volume ${size} GB with ${replica_count} replicas
+    ${volume_name} =    create_volume   ${size}    ${replica_count}
+    attach_volume    ${volume_name}
+    Set Test Variable    ${volume_name}
 
-    FOR    ${field}    IN    @{list_of_fields}
-        ${manifest} =    Update Manifest With Fields    ${manifest}    ${field}
-    END
+Write data to the volume
+    ${volume_data_checksum} =     write_volume_random_data   ${volume_name}    1024
+    Set Test Variable    ${volume_data_checksum}
 
-    Create Volume With Manifest    ${manifest}
+Delete replica ${replica_0} to trigger replica ${replica_0} rebuilding
+    delete_replica    ${volume_name}    ${replica_0}
+    wait_for_replica_rebuilding_start    ${volume_name}    ${replica_0}
 
-Create Volume Base Manifest
-    ${volume_name}    ${manifest} =    create_volume_manifest
-    Set Test Variable    ${volume_name}
-    RETURN    ${volume_name}    ${manifest}
-
-Update Manifest With Fields
-    [Arguments]    ${manifest}    ${field}
-    update_manifest_field    ${manifest}    ${field}
-    RETURN    ${manifest}
-
-Create Volume With Manifest
-    [Arguments]    ${manifest}
-    create_volume_using_manifest    ${manifest}
-
-Attach volume to node ${node_index}
-    Get the index of the cluster nodes
-    ${target_node_index} =    Evaluate    ${node_index}-1
-    ${volume_attached_node} =    attach_volume    ${volume_name}    ${cluster_node_index}[${target_node_index}]
-    ${non_volume_attached_node} =    get_non_volume_attached_node    ${volume_attached_node}
-    Set Test Variable    ${volume_attached_node}
-    Set Test Variable    ${non_volume_attached_node}
-
-Write data into mount point
-    ${volume_data_checksum} =    write_volume_random_data    ${volume_name}    512
-    Set Test Variable    ${volume_data_checksum}
+During replica ${replica_0} rebuilding, delete replica ${replica_1}
+    wait_for_replica_rebuilding_start    ${volume_name}    ${replica_0}
+    delete_replica    ${volume_name}    ${replica_1}
 
-Check data is intact
-    check_data    ${volume_name}    ${volume_data_checksum}
+Wait until replica ${replica_0} rebuilt, delete replica ${replica_2}
+    wait_for_replica_rebuilding_complete    ${volume_name}    ${replica_0}
+    delete_replica    ${volume_name}    ${replica_2}
 
-Volume ${volume_name} data checksum should be ${volume_data_checksum}
+Check data is intact
     check_data    ${volume_name}    ${volume_data_checksum}
 
-Volume state should eventually be ${expected_volume_state}
-    Run keyword And Continue On Failure
-    ...    Wait Until Keyword Succeeds
-    ...    ${retry_timeout_second} seconds
-    ...    ${retry_interval} seconds
-    ...    Volume state should be ${expected_volume_state}
-
-Volume state should be ${expected_volume_state}
-    ${volume_current_state} =    get_volume_state    ${volume_name}
-    Should Be Equal    ${volume_current_state}    ${expected_volume_state}
+Wait until all replicas rebuilt
+    wait_for_replica_rebuilding_complete    ${volume_name}    0
+    wait_for_replica_rebuilding_complete    ${volume_name}    1
+    wait_for_replica_rebuilding_complete    ${volume_name}    2
diff --git a/e2e/libs/keywords/volume_keywords.py b/e2e/libs/keywords/volume_keywords.py
@@ -1,77 +1,73 @@
-import logging
-
-from common_keywords import common_keywords
-from node import Nodes
-from utils import common_utils
+from pathlib import Path
+from volume import Volume
+from utility import Utility
+from node import Node
+from node_exec import NodeExec
 
 class volume_keywords:
 
+    ROBOT_LIBRARY_SCOPE = 'TEST'
+
     def __init__(self):
-        self.volume = common_keywords.volume_instance
-        self.node_exec = common_keywords.node_exec_instance
+        Utility().init_k8s_api_client()
+
+
+    def set_test_name(self, test_name):
+        self.namespace = test_name.lower().replace(' ', '-')
+        self.node_exec = NodeExec(self.namespace)
+        self.volume = Volume(self.node_exec)
 
-    def create_volume(self, size, replica_count, volume_type='RWO'):
-        volume_name = common_utils.generate_volume_name()
-        self.volume.create(volume_name, size, replica_count, volume_type)
+
+    def create_volume(self, size, replica_count):
+        print('create_volume')
+        volume_name = Utility().generate_volume_name()
+        self.volume.create(volume_name, size, replica_count)
         return volume_name
 
-    def create_volume_manifest(self):
-        volume_name = common_utils.generate_volume_name()
-        manifest = {
-            "apiVersion": "longhorn.io/v1beta2",
-            "kind": "Volume",
-            "metadata": {"name": volume_name},
-            "spec": {
-                "frontend": "blockdev",
-                "replicaAutoBalance": "ignored",
-            }
-        }
-        return volume_name, manifest
-
-    def update_manifest_field(self, manifest, field):
-        dict_field = eval(field)
-        for key in dict_field.keys():
-            if key in manifest.keys():
-                for key2 in dict_field[key]:
-                    manifest[key][key2] = dict_field[key][key2]
-            else:
-                manifest.update(dict_field)
-        return manifest
-
-    def create_volume_using_manifest(self,manifest):
-        return self.volume.create_with_manifest(manifest)
-
-    def attach_volume(self, volume_name, node_index=0):
-        node_name = Nodes.get_name_by_index(int(node_index))
-        logging.info(
-            f'attaching the volume {volume_name} to the node {node_name}')
 
+    def attach_volume(self, volume_name, attached_node_index=0):
+        print('attach_volume')
+        node_name = Node().get_by_index(attached_node_index)
         self.volume.attach(volume_name, node_name)
-        return node_name
-
-    def get_non_volume_attached_node(self, attached_node_name):
-        logging.info('getting node without volume attached')
-        nodes = Nodes.all_nodes
-        for node in nodes:
-            node_name = node['name']
-            if node_name != attached_node_name:
-                logging.info(f' volume attached node:{node_name}')
-                return node_name
-        logging.info('cannot find the node without volume attached')
+
 
     def write_volume_random_data(self, volume_name, size_in_mb):
-        logging.info(
-            f'writing {size_in_mb} mb data into volume {volume_name} mount point')
+        print('write_volume_random_data')
         return self.volume.write_random_data(volume_name, size_in_mb)
 
-    def get_volume_end_point(self, volume_name):
-        logging.info(f'gettting volume {volume_name} end point')
-        return self.volume.get_endpoint(volume_name)
 
     def check_data(self, volume_name, checksum):
-        logging.info(f"checking volume {volume_name} data with checksum {checksum}")
+        print(f"check volume {volume_name} data with checksum {checksum}")
         self.volume.check_data(volume_name, checksum)
 
-    def get_volume_state(self, volume_name):
-        logging.info(f"getting the volume {volume_name} state")
-        return self.volume.get_volume_state(volume_name)
+
+    def delete_replica(self, volume_name, replica_index):
+        replica_node_name = Node().get_by_index(int(replica_index))
+        print(f"delete volume {volume_name}'s replica\
+                {replica_index} {replica_node_name}")
+        self.volume.delete_replica(volume_name, replica_node_name)
+
+
+    def wait_for_replica_rebuilding_start(self, volume_name, replica_index):
+        replica_node_name = Node().get_by_index(int(replica_index))
+        print(f"wait for  volume {volume_name}'s replica\
+                {replica_index} {replica_node_name} rebuilding started")
+        self.volume.wait_for_replica_rebuilding_start(
+            volume_name,
+            replica_node_name
+        )
+
+
+    def wait_for_replica_rebuilding_complete(self, volume_name, replica_index):
+        replica_node_name = Node().get_by_index(int(replica_index))
+        print(f"wait for  volume {volume_name}'s replica\
+                {replica_index} {replica_node_name} rebuilding completed")
+        self.volume.wait_for_replica_rebuilding_complete(
+            volume_name,
+            replica_node_name
+        )
+
+
+    def cleanup_resources(self):
+        print('cleanup_resources')
+        self.node_exec.cleanup()
diff --git a/e2e/libs/node/__init__.py b/e2e/libs/node/__init__.py
@@ -1 +1 @@
-from node.node import Nodes
+from node.node import Node
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		from node.node import Nodes
		from node.node import Node