diff --git a/e2e/keywords/common.resource b/e2e/keywords/common.resource index f67a690e0b..dda0983eef 100644 --- a/e2e/keywords/common.resource +++ b/e2e/keywords/common.resource @@ -4,6 +4,7 @@ Documentation Common keywords Library ../libs/keywords/common_keywords.py Library ../libs/keywords/volume_keywords.py Library ../libs/keywords/recurring_job_keywords.py +Library ../libs/keywords/workload_keywords.py *** Variables *** @@ -15,8 +16,14 @@ Set test environment init_node_exec ${TEST NAME} @{volume_list} = Create List Set Test Variable ${volume_list} + @{deployment_list} = Create List + Set Test Variable ${deployment_list} + @{statefulset_list} = Create List + Set Test Variable ${statefulset_list} Cleanup test resources cleanup_node_exec cleanup_recurring_jobs ${volume_list} cleanup_volumes ${volume_list} + cleanup_deployments ${deployment_list} + cleanup_statefulsets ${statefulset_list} diff --git a/e2e/keywords/node.resource b/e2e/keywords/node.resource index ed1e15b0e8..160e6c5920 100644 --- a/e2e/keywords/node.resource +++ b/e2e/keywords/node.resource @@ -3,6 +3,7 @@ Documentation Physical Node Keywords Library ../libs/keywords/volume_keywords.py Library ../libs/keywords/node_keywords.py +Library ../libs/keywords/workload_keywords.py *** Keywords *** During replica rebuilding, reboot volume node @@ -22,3 +23,12 @@ Reboot volume ${idx} replica node FOR ${item} IN @{volume_list} wait for volume_attached ${item} END + +Restart cluster + restart_all_nodes + FOR ${deployment} IN @{deployment_list} + wait_for_workload_pod_stable ${deployment} + END + FOR ${statefulset} IN @{statefulset_list} + wait_for_workload_pod_stable ${statefulset} + END diff --git a/e2e/keywords/volume.resource b/e2e/keywords/volume.resource index 551d93092a..1fe66c8be4 100644 --- a/e2e/keywords/volume.resource +++ b/e2e/keywords/volume.resource @@ -12,7 +12,7 @@ Create a volume with ${size} GB and ${replica_count} replicas Set Test Variable ${volume_name} Append To List ${volume_list} ${volume_name} -Create volume ${idx} with size ${size} GB and ${replica_count} replicas +Create volume ${idx} with ${size} GB and ${replica_count} replicas ${volume_name} = create_volume ${size} ${replica_count} attach_volume ${volume_name} Insert Into List ${volume_list} ${idx} ${volume_name} diff --git a/e2e/keywords/workload.resource b/e2e/keywords/workload.resource new file mode 100644 index 0000000000..0fcefe5d32 --- /dev/null +++ b/e2e/keywords/workload.resource @@ -0,0 +1,33 @@ +*** Settings *** +Documentation Workload Keywords + +Library Collections +Library ../libs/keywords/workload_keywords.py + +*** Keywords *** +Create deployment ${idx} with ${volume_type} volume + ${deployment_name} = create_deployment ${volume_type} + Insert Into List ${deployment_list} ${idx} ${deployment_name} + +Create statefulset ${idx} with ${volume_type} volume + ${statefulset_name} = create_statefulset ${volume_type} + Insert Into List ${statefulset_list} ${idx} ${statefulset_name} + +Keep writing data to deployment ${idx} + ${pod_name} = get_workload_pod_name ${deployment_list}[${idx}] + keep_writing_pod_data ${pod_name} + +Keep writing data to statefulset ${idx} + ${pod_name} = get_workload_pod_name ${statefulset_list}[${idx}] + keep_writing_pod_data ${pod_name} + +Check deployment ${idx} works + ${pod_name} = get_workload_pod_name ${deployment_list}[${idx}] + ${pod_data_checksum} = write_pod_random_data ${pod_name} 1024 + check_pod_data ${pod_name} ${pod_data_checksum} + +Check statefulset ${idx} works + ${pod_name} = get_workload_pod_name ${statefulset_list}[${idx}] + ${pod_data_checksum} = write_pod_random_data ${pod_name} 1024 + check_pod_data ${pod_name} ${pod_data_checksum} + diff --git a/e2e/libs/keywords/node_keywords.py b/e2e/libs/keywords/node_keywords.py index 8dd903e2b0..271d510695 100644 --- a/e2e/libs/keywords/node_keywords.py +++ b/e2e/libs/keywords/node_keywords.py @@ -20,3 +20,6 @@ def reboot_replica_node(self, volume_name): volume_keywords = BuiltIn().get_library_instance('volume_keywords') replica_node = volume_keywords.get_replica_node(volume_name) self.node.reboot_node(test_pod_running_node, replica_node) + + def restart_all_nodes(self): + self.node.restart_all_nodes() diff --git a/e2e/libs/keywords/volume_keywords.py b/e2e/libs/keywords/volume_keywords.py index ac50c6ce4f..c7ce9c0149 100644 --- a/e2e/libs/keywords/volume_keywords.py +++ b/e2e/libs/keywords/volume_keywords.py @@ -87,6 +87,9 @@ def wait_for_replica_rebuilding_complete(self, volume_name, replica_node): def wait_for_volume_attached(self, volume_name): self.volume.wait_for_volume_attached(volume_name) + def wait_for_volume_healthy(self, volume_name): + self.volume.wait_for_volume_healthy(volume_name) + def cleanup_volumes(self, volume_names): logging.warn(f"cleanup volumes {volume_names}") self.volume.cleanup(volume_names) \ No newline at end of file diff --git a/e2e/libs/keywords/workload_keywords.py b/e2e/libs/keywords/workload_keywords.py new file mode 100644 index 0000000000..6145513f24 --- /dev/null +++ b/e2e/libs/keywords/workload_keywords.py @@ -0,0 +1,50 @@ +from workload.workload import * +import logging + +class workload_keywords: + + def __init__(self): + logging.warn("initialize workload_keywords class") + + def create_deployment(self, volume_type="rwo"): + pvc_filepath = f"./templates/workload/{volume_type}_pvc.yaml" + deployment_filepath = f"./templates/workload/deployment_with_{volume_type}_volume.yaml" + pvc_name = create_pvc(pvc_filepath) + deployment_name = create_deployment(deployment_filepath) + return deployment_name + + def create_statefulset(self, volume_type="rwo"): + statefulset_filepath = f"./templates/workload/statefulset_with_{volume_type}_volume.yaml" + statefulset_name = create_statefulset(statefulset_filepath) + return statefulset_name + + def get_workload_pod_name(self, workload_name): + return get_workload_pod_names(workload_name)[0] + + def get_workload_volume_name(self, workload_name): + return get_workload_volume_name(workload_name) + + def keep_writing_pod_data(self, pod_name): + return keep_writing_pod_data(pod_name) + + def write_pod_random_data(self, pod, size_in_mb): + return write_pod_random_data(pod, size_in_mb) + + def check_pod_data(self, pod_name, checksum): + print(f"check pod {pod_name} data with checksum {checksum}") + check_pod_data(pod_name, checksum) + + def cleanup_deployments(self, deployment_names): + for name in deployment_names: + pvc_name = get_workload_pvc_name(name) + delete_deployment(name) + delete_pvc(pvc_name) + + def cleanup_statefulsets(self, statefulset_names): + for name in statefulset_names: + pvc_name = get_workload_pvc_name(name) + delete_statefulset(name) + delete_pvc(pvc_name) + + def wait_for_workload_pod_stable(self, workload_name): + return wait_for_workload_pod_stable(workload_name) diff --git a/e2e/libs/node/node.py b/e2e/libs/node/node.py index 18224c3208..43b018b62f 100644 --- a/e2e/libs/node/node.py +++ b/e2e/libs/node/node.py @@ -3,18 +3,42 @@ import time import logging from utility.utility import apply_cr_from_yaml, get_cr +from utility.utility import wait_for_cluster_ready +import boto3 RETRY_COUNT = 180 RETRY_INTERVAL = 1 class Node: + def __init__(self): + with open('/tmp/instance_mapping', 'r') as f: + self.mapping = yaml.safe_load(f) + self.aws_client = boto3.client('ec2') + #logging.warn(f"describe_instances = {self.aws_client.describe_instances()}") + + def restart_all_nodes(self): + instance_ids = [value for value in self.mapping.values()] + print(instance_ids) + resp = self.aws_client.stop_instances(InstanceIds=instance_ids) + print(resp) + waiter = self.aws_client.get_waiter('instance_stopped') + waiter.wait(InstanceIds=instance_ids) + print(f"all instances stopped") + time.sleep(60) + resp = self.aws_client.start_instances(InstanceIds=instance_ids) + print(resp) + waiter = self.aws_client.get_waiter('instance_running') + waiter.wait(InstanceIds=instance_ids) + wait_for_cluster_ready() + print(f"all instances running") + def reboot_node(self, running_on_node_name, reboot_node_name, shut_down_time_in_sec=10): with open('/tmp/instance_mapping', 'r') as f: mapping = yaml.safe_load(f) reboot_node_id = mapping[reboot_node_name] - filepath = './litmus/reboot-engine.yaml' + filepath = './litmus/reboot-node.yaml' with open(filepath, 'r') as f: data = yaml.safe_load(f) data['spec']['components']['runner']['nodeSelector']['kubernetes.io/hostname'] = running_on_node_name @@ -33,7 +57,7 @@ def reboot_node(self, running_on_node_name, reboot_node_name, shut_down_time_in_ 'v1alpha1', 'default', 'chaosresults', - 'reboot-engine-ec2-terminate-by-id') + 'reboot-node-ec2-terminate-by-id') if results['status']['experimentStatus']['verdict'] == 'Pass': break time.sleep(RETRY_INTERVAL) diff --git a/e2e/libs/utility/utility.py b/e2e/libs/utility/utility.py index 246487835f..cd75506c6c 100644 --- a/e2e/libs/utility/utility.py +++ b/e2e/libs/utility/utility.py @@ -1,5 +1,6 @@ from kubernetes import config, client, dynamic from kubernetes.client.rest import ApiException +from kubernetes.stream import stream from longhorn import from_env import string import random @@ -37,6 +38,24 @@ def list_nodes(): nodes.append(item.metadata.name) return sorted(nodes) +def wait_for_cluster_ready(): + core_api = client.CoreV1Api() + for i in range(RETRY_COUNTS): + try: + resp = core_api.list_node() + ready = True + for item in resp.items: + for condition in item.status.conditions: + if condition.type == 'Ready' and condition.status != 'True': + ready = False + break + if ready: + break + except Exception as e: + logging.warn(f"list node error: {e}") + time.sleep(RETRY_INTERVAL) + assert ready, f"expect cluster's ready but it isn't {resp}" + def get_node(index): nodes = list_nodes() return nodes[int(index)] @@ -123,8 +142,13 @@ def get_longhorn_client(): # manually expose longhorn client # to access longhorn manager in local environment longhorn_client_url = os.getenv('LONGHORN_CLIENT_URL') - longhorn_client = from_env(url=f"{longhorn_client_url}/v1/schemas") - return longhorn_client + for i in range(RETRY_COUNTS): + try: + longhorn_client = from_env(url=f"{longhorn_client_url}/v1/schemas") + return longhorn_client + except Exception as e: + logging.info(f"get longhorn client error: {e}") + time.sleep(RETRY_INTERVAL) else: logging.info(f"initialize longhorn api client from longhorn manager") # for ci, run test in in-cluster environment diff --git a/e2e/libs/volume/volume.py b/e2e/libs/volume/volume.py index 2c714f6e0b..81be48edf8 100644 --- a/e2e/libs/volume/volume.py +++ b/e2e/libs/volume/volume.py @@ -34,6 +34,10 @@ def wait_for_volume_attached(self, volume_name): self.volume.wait_for_volume_state(volume_name, "attached") self.volume.wait_for_volume_robustness_not(volume_name, "unknown") + def wait_for_volume_healthy(self, volume_name): + self.volume.wait_for_volume_state(volume_name, "attached") + self.volume.wait_for_volume_robustness(volume_name, "healthy") + def get_endpoint(self, volume_name): return self.volume.get_endpoint(volume_name) diff --git a/e2e/libs/workload/__init__.py b/e2e/libs/workload/__init__.py new file mode 100644 index 0000000000..00f53801d2 --- /dev/null +++ b/e2e/libs/workload/__init__.py @@ -0,0 +1 @@ +from workload import workload \ No newline at end of file diff --git a/e2e/libs/workload/workload.py b/e2e/libs/workload/workload.py new file mode 100644 index 0000000000..6e72ec00d9 --- /dev/null +++ b/e2e/libs/workload/workload.py @@ -0,0 +1,252 @@ +from kubernetes import client +from kubernetes.client.rest import ApiException +from kubernetes.stream import stream +import time +import yaml +import logging + +RETRY_COUNTS = 150 +RETRY_INTERVAL = 1 +POD_WAIT_INTERVAL = 1 +POD_WAIT_TIMEOUT = 240 +WAIT_FOR_POD_STABLE_MAX_RETRY = 90 + +def create_deployment(filepath): + with open(filepath, 'r') as f: + namespace = 'default' + manifest_dict = yaml.safe_load(f) + api = client.AppsV1Api() + try: + deployment = api.create_namespaced_deployment( + namespace=namespace, + body=manifest_dict) + print(deployment) + + deployment_name = deployment.metadata.name + replicas = deployment.spec.replicas + + for i in range(RETRY_COUNTS): + deployment = api.read_namespaced_deployment( + name=deployment_name, + namespace=namespace) + # deployment is none if deployment is not yet created + if deployment is not None and \ + deployment.status.ready_replicas == replicas: + break + time.sleep(RETRY_INTERVAL) + + assert deployment.status.ready_replicas == replicas + + except Exception as e: + print(f"Exception when create deployment: {e}") + return deployment_name + +def delete_deployment(name, namespace='default'): + api = client.AppsV1Api() + + try: + api.delete_namespaced_deployment( + name=name, + namespace=namespace, + grace_period_seconds=0) + except ApiException as e: + assert e.status == 404 + + for i in range(RETRY_COUNTS): + resp = api.list_namespaced_deployment(namespace=namespace) + deleted = True + for item in resp.items: + if item.metadata.name == name: + deleted = False + break + if deleted: + break + time.sleep(RETRY_INTERVAL) + assert deleted + +def create_statefulset(filepath): + with open(filepath, 'r') as f: + namespace = 'default' + manifest_dict = yaml.safe_load(f) + api = client.AppsV1Api() + try: + statefulset = api.create_namespaced_stateful_set( + body=manifest_dict, + namespace=namespace) + + statefulset_name = statefulset.metadata.name + replicas = statefulset.spec.replicas + + for i in range(RETRY_COUNTS): + statefulset = api.read_namespaced_stateful_set( + name=statefulset_name, + namespace=namespace) + # statefulset is none if statefulset is not yet created + if statefulset is not None and \ + statefulset.status.ready_replicas == replicas: + break + time.sleep(RETRY_INTERVAL) + + assert statefulset.status.ready_replicas == replicas + + except Exception as e: + print(f"Exception when create statefulset: {e}") + return statefulset_name + +def delete_statefulset(name, namespace='default'): + api = client.AppsV1Api() + + try: + api.delete_namespaced_stateful_set( + name=name, + namespace=namespace, + grace_period_seconds=0) + except ApiException as e: + assert e.status == 404 + + for i in range(RETRY_COUNTS): + resp = api.list_namespaced_stateful_set(namespace=namespace) + deleted = True + for item in resp.items: + if item.metadata.name == name: + deleted = False + break + if deleted: + break + time.sleep(RETRY_INTERVAL) + assert deleted + +def create_pvc(filepath): + with open(filepath, 'r') as f: + namespace = 'default' + manifest_dict = yaml.safe_load(f) + api = client.CoreV1Api() + try: + pvc = api.create_namespaced_persistent_volume_claim( + body=manifest_dict, + namespace=namespace) + except Exception as e: + print(f"Exception when create pvc: {e}") + return pvc.metadata.name + + +def delete_pvc(name, namespace='default'): + api = client.CoreV1Api() + + try: + api.delete_namespaced_persistent_volume_claim( + name=name, + namespace=namespace, + grace_period_seconds=0) + except ApiException as e: + assert e.status == 404 + + for i in range(RETRY_COUNTS): + resp = api.list_namespaced_persistent_volume_claim(namespace=namespace) + deleted = True + for item in resp.items: + if item.metadata.name == name: + deleted = False + break + if deleted: + break + time.sleep(RETRY_INTERVAL) + assert deleted + +def get_workload_pod_names(workload_name): + api = client.CoreV1Api() + label_selector = f"app={workload_name}" + pod_list = api.list_namespaced_pod( + namespace="default", + label_selector=label_selector) + pod_names = [] + for pod in pod_list.items: + pod_names.append(pod.metadata.name) + return pod_names + +def get_workload_pods(workload_name): + api = client.CoreV1Api() + label_selector = f"app={workload_name}" + resp = api.list_namespaced_pod( + namespace="default", + label_selector=label_selector) + return resp.items + +def get_workload_volume_name(workload_name): + get_workload_pvc_name(workload_name) + pvc = api.read_namespaced_persistent_volume_claim( + name=pvc_name, namespace='default') + return pvc.spec.volume_name + +def get_workload_pvc_name(workload_name): + api = client.CoreV1Api() + pod = get_workload_pods(workload_name)[0] + print(f"pod = {pod}") + for volume in pod.spec.volumes: + if volume.name == 'pod-data': + pvc_name = volume.persistent_volume_claim.claim_name + break + assert pvc_name + return pvc_name + +def write_pod_random_data(pod_name, size_in_mb, path="/data/random-data"): + api = client.CoreV1Api() + write_cmd = [ + '/bin/sh', + '-c', + f"dd if=/dev/urandom of={path} bs=1M count={size_in_mb} status=none;\ + md5sum {path} | awk \'{{print $1}}\'" + ] + return stream( + api.connect_get_namespaced_pod_exec, pod_name, 'default', + command=write_cmd, stderr=True, stdin=False, stdout=True, + tty=False) + +def keep_writing_pod_data(pod_name, size_in_mb=256, path="/data/overwritten-data"): + api = client.CoreV1Api() + write_cmd = [ + '/bin/sh', + '-c', + f"while true; do dd if=/dev/urandom of={path} bs=1M count={size_in_mb} status=none; done > /dev/null 2> /dev/null &" + ] + logging.warn("before keep_writing_pod_data") + res = stream( + api.connect_get_namespaced_pod_exec, pod_name, 'default', + command=write_cmd, stderr=True, stdin=False, stdout=True, + tty=False) + logging.warn("keep_writing_pod_data return") + return res + +def check_pod_data(pod_name, checksum, path="/data/random-data"): + api = client.CoreV1Api() + cmd = [ + '/bin/sh', + '-c', + f"md5sum {path} | awk \'{{print $1}}\'" + ] + _checksum = stream( + api.connect_get_namespaced_pod_exec, pod_name, 'default', + command=cmd, stderr=True, stdin=False, stdout=True, + tty=False) + print(f"get {path} checksum = {_checksum},\ + expected checksum = {checksum}") + assert _checksum == checksum + +def wait_for_workload_pod_stable(workload_name): + stable_pod = None + wait_for_stable_retry = 0 + for _ in range(POD_WAIT_TIMEOUT): + pods = get_workload_pods(workload_name) + for pod in pods: + if pod.status.phase == "Running": + if stable_pod is None or \ + stable_pod.status.start_time != pod.status.start_time: + stable_pod = pod + wait_for_stable_retry = 0 + break + else: + wait_for_stable_retry += 1 + if wait_for_stable_retry == WAIT_FOR_POD_STABLE_MAX_RETRY: + return stable_pod + time.sleep(POD_WAIT_INTERVAL) + assert False \ No newline at end of file diff --git a/e2e/templates/litmus/reboot-node.yaml b/e2e/templates/litmus/reboot-node.yaml new file mode 100644 index 0000000000..9d42ee3ccb --- /dev/null +++ b/e2e/templates/litmus/reboot-node.yaml @@ -0,0 +1,37 @@ +apiVersion: litmuschaos.io/v1alpha1 +kind: ChaosEngine +metadata: + name: reboot-node + namespace: default +spec: + # It can be delete/retain + jobCleanUpPolicy: 'retain' + engineState: 'active' + chaosServiceAccount: ec2-terminate-by-id-sa + components: + runner: + nodeSelector: + kubernetes.io/hostname: engine-running-node + experiments: + - name: ec2-terminate-by-id + spec: + components: + nodeSelector: + kubernetes.io/hostname: engine-running-node + env: + # set chaos duration (in sec) as desired + - name: TOTAL_CHAOS_DURATION + value: '30' + # set interval duration (in sec) as desired + - name: CHAOS_INTERVAL + value: '180' + # Instance ID of the target ec2 instance + # Multiple IDs can also be provided as comma separated values ex: id1,id2 + - name: EC2_INSTANCE_ID + value: instance-to-be-terminated + # provide the region name of the instance + - name: REGION + value: 'us-east-1' + # enable it if the target instance is a part of self-managed nodegroup. + - name: MANAGED_NODEGROUP + value: 'disable' \ No newline at end of file diff --git a/e2e/templates/workload/deployment_with_rwo_volume.yaml b/e2e/templates/workload/deployment_with_rwo_volume.yaml new file mode 100644 index 0000000000..054200b4f8 --- /dev/null +++ b/e2e/templates/workload/deployment_with_rwo_volume.yaml @@ -0,0 +1,29 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment-rwo + labels: + app: test-deployment-rwo + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: test-deployment-rwo + template: + metadata: + labels: + app: test-deployment-rwo + spec: + containers: + - image: busybox + imagePullPolicy: IfNotPresent + name: sleep + args: ['/bin/sh', '-c', 'while true;do date;sleep 5; done'] + volumeMounts: + - name: pod-data + mountPath: /data + volumes: + - name: pod-data + persistentVolumeClaim: + claimName: test-rwo-pvc \ No newline at end of file diff --git a/e2e/templates/workload/deployment_with_rwx_volume.yaml b/e2e/templates/workload/deployment_with_rwx_volume.yaml new file mode 100644 index 0000000000..d6cb78c479 --- /dev/null +++ b/e2e/templates/workload/deployment_with_rwx_volume.yaml @@ -0,0 +1,29 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment-rwx + labels: + app: test-deployment-rwx + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: test-deployment-rwx + template: + metadata: + labels: + app: test-deployment-rwx + spec: + containers: + - image: busybox + imagePullPolicy: IfNotPresent + name: sleep + args: ['/bin/sh', '-c', 'while true;do date;sleep 5; done'] + volumeMounts: + - name: pod-data + mountPath: /data + volumes: + - name: pod-data + persistentVolumeClaim: + claimName: test-rwx-pvc \ No newline at end of file diff --git a/e2e/templates/workload/rwo_pvc.yaml b/e2e/templates/workload/rwo_pvc.yaml new file mode 100644 index 0000000000..9d7ea5f8ff --- /dev/null +++ b/e2e/templates/workload/rwo_pvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: test-rwo-pvc + namespace: default +spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 3Gi \ No newline at end of file diff --git a/e2e/templates/workload/rwx_pvc.yaml b/e2e/templates/workload/rwx_pvc.yaml new file mode 100644 index 0000000000..6a4601ecbd --- /dev/null +++ b/e2e/templates/workload/rwx_pvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: test-rwx-pvc + namespace: default +spec: + accessModes: + - ReadWriteMany + storageClassName: longhorn + resources: + requests: + storage: 3Gi \ No newline at end of file diff --git a/e2e/templates/workload/statefulset_with_rwo_volume.yaml b/e2e/templates/workload/statefulset_with_rwo_volume.yaml new file mode 100644 index 0000000000..1a64ae43fa --- /dev/null +++ b/e2e/templates/workload/statefulset_with_rwo_volume.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-statefulset + namespace: default +spec: + selector: + matchLabels: + app: test-statefulset + serviceName: test-statefulset + replicas: 1 + template: + metadata: + labels: + app: test-statefulset + spec: + terminationGracePeriodSeconds: 10 + containers: + - image: busybox:1.34.0 + imagePullPolicy: IfNotPresent + name: sleep + args: + - "/bin/sh" + - "-c" + - while true;do date;sleep 5; done + volumeMounts: + - name: pod-data + mountPath: "/data" + volumeClaimTemplates: + - metadata: + name: pod-data + spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn + resources: + requests: + storage: 3Gi diff --git a/e2e/templates/workload/statefulset_with_rwx_volume.yaml b/e2e/templates/workload/statefulset_with_rwx_volume.yaml new file mode 100644 index 0000000000..efdc44d621 --- /dev/null +++ b/e2e/templates/workload/statefulset_with_rwx_volume.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: rwx-test-statefulset + namespace: default +spec: + selector: + matchLabels: + app: rwx-test-statefulset + serviceName: rwx-test-statefulset + replicas: 1 + template: + metadata: + labels: + app: rwx-test-statefulset + spec: + terminationGracePeriodSeconds: 10 + containers: + - image: busybox:1.34.0 + imagePullPolicy: IfNotPresent + name: sleep + args: + - "/bin/sh" + - "-c" + - while true;do date;sleep 5; done + volumeMounts: + - name: pod-data + mountPath: "/data" + volumeClaimTemplates: + - metadata: + name: pod-data + spec: + accessModes: + - ReadWriteMany + storageClassName: longhorn + resources: + requests: + storage: 3Gi diff --git a/e2e/tests/cluster_restart.robot b/e2e/tests/cluster_restart.robot new file mode 100644 index 0000000000..5667e647ad --- /dev/null +++ b/e2e/tests/cluster_restart.robot @@ -0,0 +1,29 @@ +*** Settings *** +Documentation Negative Test Cases +Resource ../keywords/workload.resource +Resource ../keywords/node.resource +Resource ../keywords/common.resource + +Test Setup Set test environment +Test Teardown Cleanup test resources + +*** Variables *** +${LOOP_COUNT} 1 + +*** Test Cases *** +Restart Cluster While Workload Heavy Writing + Create deployment 0 with rwo volume + Create deployment 1 with rwx volume + Create statefulset 0 with rwo volume + Create statefulset 1 with rwx volume + FOR ${i} IN RANGE ${LOOP_COUNT} + Keep writing data to deployment 0 + Keep writing data to deployment 1 + Keep writing data to statefulset 0 + Keep writing data to statefulset 1 + Restart cluster + Check deployment 0 works + Check deployment 1 works + Check statefulset 0 works + Check statefulset 1 works + END \ No newline at end of file diff --git a/e2e/tests/heavy_writing_and_recurring_jobs.robot b/e2e/tests/heavy_writing_and_recurring_jobs.robot index 48826026e5..bf772abaf7 100644 --- a/e2e/tests/heavy_writing_and_recurring_jobs.robot +++ b/e2e/tests/heavy_writing_and_recurring_jobs.robot @@ -13,8 +13,8 @@ ${LOOP_COUNT} 1 *** Test Cases *** Reboot Volume Node While Heavy Writing And Recurring Jobs Exist - Create volume 0 with size 2 GB and 1 replicas - Create volume 1 with size 2 GB and 3 replicas + Create volume 0 with 2 GB and 1 replicas + Create volume 1 with 2 GB and 3 replicas Keep writing data to volume 0 Keep Writing data to volume 1 Create snapshot and backup recurring job for volume 0 @@ -28,8 +28,8 @@ Reboot Volume Node While Heavy Writing And Recurring Jobs Exist END Reboot Replica Node While Heavy Writing And Recurring Jobs Exist - Create volume 0 with size 2 GB and 1 replicas - Create volume 1 with size 2 GB and 3 replicas + Create volume 0 with 2 GB and 1 replicas + Create volume 1 with 2 GB and 3 replicas Keep Writing data to volume 0 Keep Writing data to volume 1 Create snapshot and backup recurring job for volume 0