test: implement kubelet restart

currently only support k3s

Signed-off-by: Yang Chiu <[email protected]>

e2e/keywords/kubelet.resource

@@ -0,0 +1,15 @@
+*** Settings ***
+Documentation       Kubelet keywords
+
+Library             ../libs/keywords/kubelet_keywords.py
+Library             ../libs/keywords/workload_keywords.py
+Library             ../libs/keywords/volume_keywords.py
+
+*** Variables ***
+
+
+*** Keywords ***
+Stop volume node kubelet of statefulset ${idx} for ${stop_time_in_sec} seconds
+    ${volume_name} =    get_workload_volume_name    ${statefulset_list}[${idx}]
+    ${node_name} =    get_volume_node    ${volume_name}
+    restart_kubelet    ${node_name}    ${stop_time_in_sec}

e2e/libs/keywords/kubelet_keywords.py

@@ -0,0 +1,6 @@
+from kubelet.kubelet import restart_kubelet
+
+class kubelet_keywords:
+
+    def restart_kubelet(self, node_name, stop_time_in_sec):
+        restart_kubelet(node_name, int(stop_time_in_sec))

e2e/libs/kubelet/kubelet.py

@@ -0,0 +1,22 @@
+from utility.utility import logging
+import time
+
+from workload.pod import new_pod_manifest
+from workload.pod import create_pod
+from workload.pod import wait_for_pod_status
+from workload.pod import delete_pod
+from workload.pod import IMAGE_UBUNTU
+
+def restart_kubelet(node_name, stop_time_in_sec=10):
+    manifest = new_pod_manifest(
+        image=IMAGE_UBUNTU,
+        command=["/bin/bash"],
+        args=["-c", f"sleep 10 && systemctl stop k3s-agent && sleep {stop_time_in_sec} && systemctl start k3s-agent"],
+        node_name=node_name
+    )
+    pod_name = manifest['metadata']['name']
+    create_pod(manifest, is_wait_for_pod_running=True)
+
+    time.sleep(stop_time_in_sec)
+
+    delete_pod(pod_name)

e2e/libs/node_exec/node_exec.py

@@ -4,7 +4,7 @@
 from kubernetes.stream import stream
 
 from utility.utility import logging
-from utility.utility import wait_delete_pod
+from workload.pod import wait_delete_pod
 from utility.utility import wait_delete_ns
 
 
@@ -52,7 +52,7 @@ def cleanup(self):
                 namespace=self.namespace,
                 body=client.V1DeleteOptions()
             )
-            wait_delete_pod(pod.metadata.uid)
+            wait_delete_pod(pod.metadata.name)
         self.core_api.delete_namespace(
             name=self.namespace
         )
@@ -61,15 +61,20 @@ def cleanup(self):
 
 
     def issue_cmd(self, node_name, cmd):
+        logging(f"Issuing command: {cmd} on {node_name}")
         pod = self.launch_pod(node_name)
-        exec_command = [
-            'nsenter',
-            '--mount=/rootfs/proc/1/ns/mnt',
-            '--',
-            'sh',
-            '-c',
-            cmd
-        ]
+        if isinstance(cmd, list):
+            exec_command = cmd
+        else:
+            exec_command = [
+                'nsenter',
+                '--mount=/rootfs/proc/1/ns/mnt',
+                '--net=/rootfs/proc/1/ns/net',
+                '--',
+                'sh',
+                '-c',
+                cmd
+            ]
         res = stream(
             self.core_api.connect_get_namespaced_pod_exec,
             pod.metadata.name,
@@ -80,6 +85,7 @@ def issue_cmd(self, node_name, cmd):
             stdout=True,
             tty=False
         )
+        logging(f"Issued command: {cmd} with result {res}")
         return res
 
     def launch_pod(self, node_name):
@@ -109,7 +115,7 @@ def launch_pod(self, node_name):
                         }
                     },
                     'containers': [{
-                        'image': 'busybox:1.34.0',
+                        'image': 'ubuntu:16.04',
                         'imagePullPolicy': 'IfNotPresent',
                         'securityContext': {
                             'privileged': True
@@ -120,15 +126,30 @@ def launch_pod(self, node_name):
                         ],
                         "volumeMounts": [{
                             'name': 'rootfs',
-                            'mountPath': '/rootfs',
-                            'readOnly': True
+                            'mountPath': '/rootfs'
+                        }, {
+                            'name': 'bus',
+                            'mountPath': '/var/run'
+                        }, {
+                            'name': 'rancher',
+                            'mountPath': '/var/lib/rancher'
                         }],
                     }],
                     'volumes': [{
                         'name': 'rootfs',
                         'hostPath': {
                             'path': '/'
                         }
+                    }, {
+                        'name': 'bus',
+                        'hostPath': {
+                            'path': '/var/run'
+                        }
+                    }, {
+                        'name': 'rancher',
+                        'hostPath': {
+                            'path': '/var/lib/rancher'
+                        }
                     }]
                 }
             }

e2e/libs/utility/utility.py

@@ -136,21 +136,6 @@ def filter_cr(group, version, namespace, plural, field_selector="", label_select
     except ApiException as e:
         logging(f"Listing namespaced custom object: {e}")
 
-def wait_delete_pod(pod_uid, namespace='default'):
-    api = client.CoreV1Api()
-    retry_count, retry_interval = get_retry_count_and_interval()
-    for i in range(retry_count):
-        ret = api.list_namespaced_pod(namespace=namespace)
-        found = False
-        for item in ret.items:
-            if item.metadata.uid == pod_uid:
-                found = True
-                break
-        if not found:
-            break
-        time.sleep(retry_interval)
-    assert not found
-
 def wait_delete_ns(name):
     api = client.CoreV1Api()
     retry_count, retry_interval = get_retry_count_and_interval()

e2e/libs/workload/pod.py

@@ -2,14 +2,14 @@
 
 from kubernetes import client
 
-from robot.libraries.BuiltIn import BuiltIn
-
 from utility.utility import logging
 from utility.utility import generate_name
+from utility.utility import get_retry_count_and_interval
 
 
 IMAGE_BUSYBOX = 'busybox:1.34.0'
 IMAGE_LITMUX = 'litmuschaos/go-runner:latest'
+IMAGE_UBUNTU = 'ubuntu:16.04'
 
 def new_pod_manifest(image="", command=[], args=[],
                      claim_name="", node_name="", labels={}):
@@ -31,15 +31,35 @@ def new_pod_manifest(image="", command=[], args=[],
         },
         'spec': {
             'nodeName': node_name,
+            'restartPolicy': 'Never',
             'containers': [{
                 'image': image,
                 'imagePullPolicy': 'IfNotPresent',
+                'securityContext': {
+                    'privileged': True
+                },
                 'name': 'run',
                 'command': command,
                 'args': args,
-                'volumeMounts': [],
+                'volumeMounts': [{
+                    'name': 'bus',
+                    'mountPath': '/var/run'
+                }, {
+                    'name': 'rancher',
+                    'mountPath': '/var/lib/rancher'
+                }]
             }],
-            'volumes': []
+            'volumes': [{
+                'name': 'bus',
+                'hostPath': {
+                    'path': '/var/run'
+                }
+            }, {
+                'name': 'rancher',
+                'hostPath': {
+                    'path': '/var/lib/rancher'
+                }
+            }]
         }
     }
 
@@ -73,15 +93,33 @@ def create_pod(manifest, is_wait_for_pod_running=False):
 
 def delete_pod(name, namespace='default'):
     core_api = client.CoreV1Api()
-    core_api.delete_namespaced_pod(name=name, namespace=namespace)
+    try:
+        core_api.delete_namespaced_pod(name=name, namespace=namespace)
+        wait_delete_pod(name)
+    except ApiException as e:
+        assert e.status == 404
+
+def wait_delete_pod(name, namespace='default'):
+    api = client.CoreV1Api()
+    retry_count, retry_interval = get_retry_count_and_interval()
+    for i in range(retry_count):
+        ret = api.list_namespaced_pod(namespace=namespace)
+        found = False
+        for item in ret.items:
+            if item.metadata.name == name:
+                found = True
+                break
+        if not found:
+            break
+        time.sleep(retry_interval)
+    assert not found
 
 def get_pod(name, namespace='default'):
     core_api = client.CoreV1Api()
     return core_api.read_namespaced_pod(name=name, namespace=namespace)
 
-def wait_for_pod_status(name, status, namespace='default',
-                        retry_count=int(BuiltIn().get_variable_value("${RETRY_COUNT}")),
-                        retry_interval=int(BuiltIn().get_variable_value("${RETRY_INTERVAL}"))):
+def wait_for_pod_status(name, status, namespace='default'):
+    retry_count, retry_interval = get_retry_count_and_interval()
     is_running = False
     for i in range(retry_count):
         pod = get_pod(name, namespace)

e2e/tests/kubelet_restart.robot

@@ -0,0 +1,36 @@
+*** Settings ***
+Documentation    Negative Test Cases
+Resource    ../keywords/workload.resource
+Resource    ../keywords/volume.resource
+Resource    ../keywords/node.resource
+Resource    ../keywords/common.resource
+Resource    ../keywords/kubelet.resource
+
+Test Setup    Set test environment
+Test Teardown    Cleanup test resources
+
+*** Variables ***
+${LOOP_COUNT}    1
+${RETRY_COUNT}    300
+${RETRY_INTERVAL}    1
+
+*** Test Cases ***
+Restart Volume Node Kubelet While Workload Heavy Writing
+    Given Create statefulset 0 with rwo volume
+    FOR    ${i}    IN RANGE    ${LOOP_COUNT}
+        And Keep writing data to statefulset 0
+        When Stop volume node kubelet of statefulset 0 for 10 seconds
+        And Wait for volume of statefulset 0 healthy
+        And Wait for statefulset 0 stable
+        Then Check statefulset 0 works
+    END
+
+Stop Volume Node Kubelet For More Than Pod Eviction Timeout While Workload Heavy Writing
+    Given Create statefulset 0 with rwo volume
+    FOR    ${i}    IN RANGE    ${LOOP_COUNT}
+        And Keep writing data to statefulset 0
+        When Stop volume node kubelet of statefulset 0 for 360 seconds
+        And Wait for volume of statefulset 0 healthy
+        And Wait for statefulset 0 stable
+        Then Check statefulset 0 works
+    END