diff --git a/e2e/keywords/kubelet.resource b/e2e/keywords/kubelet.resource new file mode 100644 index 0000000000..da5abe52ee --- /dev/null +++ b/e2e/keywords/kubelet.resource @@ -0,0 +1,15 @@ +*** Settings *** +Documentation Kubelet keywords + +Library ../libs/keywords/kubelet_keywords.py +Library ../libs/keywords/workload_keywords.py +Library ../libs/keywords/volume_keywords.py + +*** Variables *** + + +*** Keywords *** +Stop volume node kubelet of statefulset ${idx} for ${stop_time_in_sec} seconds + ${volume_name} = get_workload_volume_name ${statefulset_list}[${idx}] + ${node_name} = get_volume_node ${volume_name} + restart_kubelet ${node_name} ${stop_time_in_sec} diff --git a/e2e/libs/keywords/kubelet_keywords.py b/e2e/libs/keywords/kubelet_keywords.py new file mode 100644 index 0000000000..55c8e6cef6 --- /dev/null +++ b/e2e/libs/keywords/kubelet_keywords.py @@ -0,0 +1,6 @@ +from kubelet.kubelet import restart_kubelet + +class kubelet_keywords: + + def restart_kubelet(self, node_name, stop_time_in_sec): + restart_kubelet(node_name, int(stop_time_in_sec)) diff --git a/e2e/libs/kubelet/kubelet.py b/e2e/libs/kubelet/kubelet.py new file mode 100644 index 0000000000..c9c5180050 --- /dev/null +++ b/e2e/libs/kubelet/kubelet.py @@ -0,0 +1,22 @@ +from utility.utility import logging +import time + +from workload.pod import new_pod_manifest +from workload.pod import create_pod +from workload.pod import wait_for_pod_status +from workload.pod import delete_pod +from workload.pod import IMAGE_UBUNTU + +def restart_kubelet(node_name, stop_time_in_sec=10): + manifest = new_pod_manifest( + image=IMAGE_UBUNTU, + command=["/bin/bash"], + args=["-c", f"sleep 10 && systemctl stop k3s-agent && sleep {stop_time_in_sec} && systemctl start k3s-agent"], + node_name=node_name + ) + pod_name = manifest['metadata']['name'] + create_pod(manifest, is_wait_for_pod_running=True) + + time.sleep(stop_time_in_sec) + + delete_pod(pod_name) diff --git a/e2e/libs/node_exec/node_exec.py b/e2e/libs/node_exec/node_exec.py index 8d0ab27ae8..76011c29ef 100644 --- a/e2e/libs/node_exec/node_exec.py +++ b/e2e/libs/node_exec/node_exec.py @@ -4,7 +4,7 @@ from kubernetes.stream import stream from utility.utility import logging -from utility.utility import wait_delete_pod +from workload.pod import wait_delete_pod from utility.utility import wait_delete_ns @@ -52,7 +52,7 @@ def cleanup(self): namespace=self.namespace, body=client.V1DeleteOptions() ) - wait_delete_pod(pod.metadata.uid) + wait_delete_pod(pod.metadata.name) self.core_api.delete_namespace( name=self.namespace ) @@ -61,15 +61,20 @@ def cleanup(self): def issue_cmd(self, node_name, cmd): + logging(f"Issuing command: {cmd} on {node_name}") pod = self.launch_pod(node_name) - exec_command = [ - 'nsenter', - '--mount=/rootfs/proc/1/ns/mnt', - '--', - 'sh', - '-c', - cmd - ] + if isinstance(cmd, list): + exec_command = cmd + else: + exec_command = [ + 'nsenter', + '--mount=/rootfs/proc/1/ns/mnt', + '--net=/rootfs/proc/1/ns/net', + '--', + 'sh', + '-c', + cmd + ] res = stream( self.core_api.connect_get_namespaced_pod_exec, pod.metadata.name, @@ -80,6 +85,7 @@ def issue_cmd(self, node_name, cmd): stdout=True, tty=False ) + logging(f"Issued command: {cmd} with result {res}") return res def launch_pod(self, node_name): @@ -109,7 +115,7 @@ def launch_pod(self, node_name): } }, 'containers': [{ - 'image': 'busybox:1.34.0', + 'image': 'ubuntu:16.04', 'imagePullPolicy': 'IfNotPresent', 'securityContext': { 'privileged': True @@ -120,8 +126,13 @@ def launch_pod(self, node_name): ], "volumeMounts": [{ 'name': 'rootfs', - 'mountPath': '/rootfs', - 'readOnly': True + 'mountPath': '/rootfs' + }, { + 'name': 'bus', + 'mountPath': '/var/run' + }, { + 'name': 'rancher', + 'mountPath': '/var/lib/rancher' }], }], 'volumes': [{ @@ -129,6 +140,16 @@ def launch_pod(self, node_name): 'hostPath': { 'path': '/' } + }, { + 'name': 'bus', + 'hostPath': { + 'path': '/var/run' + } + }, { + 'name': 'rancher', + 'hostPath': { + 'path': '/var/lib/rancher' + } }] } } diff --git a/e2e/libs/utility/utility.py b/e2e/libs/utility/utility.py index f05085f959..0786cf93bd 100644 --- a/e2e/libs/utility/utility.py +++ b/e2e/libs/utility/utility.py @@ -136,21 +136,6 @@ def filter_cr(group, version, namespace, plural, field_selector="", label_select except ApiException as e: logging(f"Listing namespaced custom object: {e}") -def wait_delete_pod(pod_uid, namespace='default'): - api = client.CoreV1Api() - retry_count, retry_interval = get_retry_count_and_interval() - for i in range(retry_count): - ret = api.list_namespaced_pod(namespace=namespace) - found = False - for item in ret.items: - if item.metadata.uid == pod_uid: - found = True - break - if not found: - break - time.sleep(retry_interval) - assert not found - def wait_delete_ns(name): api = client.CoreV1Api() retry_count, retry_interval = get_retry_count_and_interval() diff --git a/e2e/libs/workload/pod.py b/e2e/libs/workload/pod.py index 274067441f..84c2d278cf 100644 --- a/e2e/libs/workload/pod.py +++ b/e2e/libs/workload/pod.py @@ -2,14 +2,14 @@ from kubernetes import client -from robot.libraries.BuiltIn import BuiltIn - from utility.utility import logging from utility.utility import generate_name +from utility.utility import get_retry_count_and_interval IMAGE_BUSYBOX = 'busybox:1.34.0' IMAGE_LITMUX = 'litmuschaos/go-runner:latest' +IMAGE_UBUNTU = 'ubuntu:16.04' def new_pod_manifest(image="", command=[], args=[], claim_name="", node_name="", labels={}): @@ -31,15 +31,35 @@ def new_pod_manifest(image="", command=[], args=[], }, 'spec': { 'nodeName': node_name, + 'restartPolicy': 'Never', 'containers': [{ 'image': image, 'imagePullPolicy': 'IfNotPresent', + 'securityContext': { + 'privileged': True + }, 'name': 'run', 'command': command, 'args': args, - 'volumeMounts': [], + 'volumeMounts': [{ + 'name': 'bus', + 'mountPath': '/var/run' + }, { + 'name': 'rancher', + 'mountPath': '/var/lib/rancher' + }] }], - 'volumes': [] + 'volumes': [{ + 'name': 'bus', + 'hostPath': { + 'path': '/var/run' + } + }, { + 'name': 'rancher', + 'hostPath': { + 'path': '/var/lib/rancher' + } + }] } } @@ -73,15 +93,33 @@ def create_pod(manifest, is_wait_for_pod_running=False): def delete_pod(name, namespace='default'): core_api = client.CoreV1Api() - core_api.delete_namespaced_pod(name=name, namespace=namespace) + try: + core_api.delete_namespaced_pod(name=name, namespace=namespace) + wait_delete_pod(name) + except ApiException as e: + assert e.status == 404 + +def wait_delete_pod(name, namespace='default'): + api = client.CoreV1Api() + retry_count, retry_interval = get_retry_count_and_interval() + for i in range(retry_count): + ret = api.list_namespaced_pod(namespace=namespace) + found = False + for item in ret.items: + if item.metadata.name == name: + found = True + break + if not found: + break + time.sleep(retry_interval) + assert not found def get_pod(name, namespace='default'): core_api = client.CoreV1Api() return core_api.read_namespaced_pod(name=name, namespace=namespace) -def wait_for_pod_status(name, status, namespace='default', - retry_count=int(BuiltIn().get_variable_value("${RETRY_COUNT}")), - retry_interval=int(BuiltIn().get_variable_value("${RETRY_INTERVAL}"))): +def wait_for_pod_status(name, status, namespace='default'): + retry_count, retry_interval = get_retry_count_and_interval() is_running = False for i in range(retry_count): pod = get_pod(name, namespace) diff --git a/e2e/tests/kubelet_restart.robot b/e2e/tests/kubelet_restart.robot new file mode 100644 index 0000000000..feb542b144 --- /dev/null +++ b/e2e/tests/kubelet_restart.robot @@ -0,0 +1,36 @@ +*** Settings *** +Documentation Negative Test Cases +Resource ../keywords/workload.resource +Resource ../keywords/volume.resource +Resource ../keywords/node.resource +Resource ../keywords/common.resource +Resource ../keywords/kubelet.resource + +Test Setup Set test environment +Test Teardown Cleanup test resources + +*** Variables *** +${LOOP_COUNT} 1 +${RETRY_COUNT} 300 +${RETRY_INTERVAL} 1 + +*** Test Cases *** +Restart Volume Node Kubelet While Workload Heavy Writing + Given Create statefulset 0 with rwo volume + FOR ${i} IN RANGE ${LOOP_COUNT} + And Keep writing data to statefulset 0 + When Stop volume node kubelet of statefulset 0 for 10 seconds + And Wait for volume of statefulset 0 healthy + And Wait for statefulset 0 stable + Then Check statefulset 0 works + END + +Stop Volume Node Kubelet For More Than Pod Eviction Timeout While Workload Heavy Writing + Given Create statefulset 0 with rwo volume + FOR ${i} IN RANGE ${LOOP_COUNT} + And Keep writing data to statefulset 0 + When Stop volume node kubelet of statefulset 0 for 360 seconds + And Wait for volume of statefulset 0 healthy + And Wait for statefulset 0 stable + Then Check statefulset 0 works + END