template: wait for pod to teardown (if container is present) during d…

…elete

Pod teardown may take longer than 5 mins (default ci-operator timeout).
This commit would ensure the same timeout is applied to the teardown
container - and then applied to the pod again.

This is useful for rehearse jobs, which reuse the namespace when testing
a new commit

pkg/steps/template.go

@@ -421,24 +421,80 @@ func createOrRestartPod(podClient coreclientset.PodInterface, pod *coreapi.Pod)
 	return created, nil
 }
 
-func waitForPodDeletion(podClient coreclientset.PodInterface, name string, uid types.UID) error {
-	timeout := 300
-	for i := 0; i < timeout; i += 2 {
-		pod, err := podClient.Get(name, meta.GetOptions{})
-		if errors.IsNotFound(err) {
-			return nil
-		}
+func waitForTeardownToComplete(podClient coreclientset.PodInterface, name string, uid types.UID) error {
+	timeout := 5*time.Minutes
+
+	log.Printf("Waiting for pod %s to complete teardown ...", name)
+	err = wait.Poll(10*time.Second, timeout, func() (done bool, err error) {
+		pod, err := checkPodExists(podClient, name, uid)
 		if err != nil {
-			return fmt.Errorf("could not retrieve deleting pod: %v", err)
+			// Pod was destroyed
+			return true, nil
 		}
-		if pod.UID != uid {
-			return nil
+
+		for _, status := range append(append([]coreapi.ContainerStatus{}, pod.Status.InitContainerStatuses...), pod.Status.ContainerStatuses...) {
+			if status.Name == "teardown" && status.State.Terminated != nil {
+				// Teardown container finished
+				return true, nil
+			}
 		}
-		log.Printf("Waiting for pod %s to be deleted ... (%ds/%d)", name, i, timeout)
-		time.Sleep(2 * time.Second)
+		return false, nil
+	})
+	return err
+}
+
+func checkPodExists(podClient coreclientset.PodInterface, name string, uid types.UID) (*coreapi.Pod, error) {
+	pod, err := podClient.Get(name, meta.GetOptions{})
+	if errors.IsNotFound(err) {
+		return nil, errors.NewGone("pod was removed")
+	}
+	if err != nil {
+		return nil, fmt.Errorf("could not retrieve deleting pod: %v", err)
+	}
+	if pod.UID != uid {
+		return nil, errors.NewBadRequest("mismatching UID")
+	}
+	return pod, nil
+}
+
+func waitForPodDeletion(podClient coreclientset.PodInterface, name string, uid types.UID) error {
+	timeout := 5 * time.Minutes
+	pod, err := checkPodExists(podClient, name, uid)
+	if err != nil {
+		// Pod with this name and UID is gone, so its successfully deleted
+		return nil
 	}
 
-	return fmt.Errorf("waited for pod %s deletion for %ds, was not deleted", name, timeout)
+	// Attempts to wait for teardown to complete
+	containerNames := podContainerNames(pod)
+	if sort.SearchStrings(containerNames, "teardown") < len(containerNames) {
+		err := waitForTeardownToComplete(podClient, name, uid)
+		if err != nil {
+			return err
+		}
+	}
+
+	watcher, err := podClient.Watch(meta.ListOptions{
+		FieldSelector: fields.Set{"metadata.name": name}.AsSelector().String(),
+		Watch:         true,
+	})
+	if err != nil {
+		return false, fmt.Errorf("could not create watcher for pod: %v", err)
+	}
+	defer watcher.Stop()
+
+	log.Printf("Waiting for pod %s to be deleted in %d seconds", name, timeout)
+	_, err := watch.Until(timeout, watcher, func(event watch.Event) (done bool, err error) {
+		if event.Type == watch.Deleted {
+			return true, nil
+		}
+		return false, nil
+	})
+	if err != nil {
+		return fmt.Errorf("waited for pod %s deletion for %ds, was not deleted", name, timeout)
+	} else {
+		return nil
+	}
 }
 
 func waitForCompletedPodDeletion(podClient coreclientset.PodInterface, name string) error {
@@ -620,6 +676,15 @@ func templateInstanceReady(instance *templateapi.TemplateInstance) (ready bool,
 	return false, nil
 }
 
+func podContainerNames(pod *coreapi.Pod) []string {
+	var names []string
+	for _, status := range append(append([]coreapi.ContainerStatus{}, pod.Status.InitContainerStatuses...), pod.Status.ContainerStatuses...) {
+		names = append(names, status.Name)
+	}
+	sort.Strings(names)
+	return names
+}
+
 func podRunningContainers(pod *coreapi.Pod) []string {
 	var names []string
 	for _, status := range append(append([]coreapi.ContainerStatus{}, pod.Status.InitContainerStatuses...), pod.Status.ContainerStatuses...) {
@@ -628,6 +693,7 @@ func podRunningContainers(pod *coreapi.Pod) []string {
 		}
 		names = append(names, status.Name)
 	}
+	sort.Strings(names)
 	return names
 }