template: wait for pod to teardown (if container is present) during d…

…elete

Pod teardown may take longer than 5 mins (default ci-operator timeout). 
This commit would ensure the same timeout is applied to the teardown 
container - and then applied to the pod again.

This is useful for rehearse jobs, which reuse the namespace when testing 
a new commit

pkg/steps/template.go

@@ -434,6 +434,15 @@ func waitForPodDeletion(podClient coreclientset.PodInterface, name string, uid t
 		if pod.UID != uid {
 			return nil
 		}
+		for j := 0; j < timeout; j += 2 {
+			runningContainers := podRunningContainers(pod)
+			if sort.SearchStrings(runningContainers, "teardown") < len(runningContainers) {
+				break
+			} else {
+				log.Printf("Waiting for pod %s to complete teardown ... (%ds/%d)", name, j, timeout)
+				time.Sleep(2 * time.Second)
+			}
+		}
 		log.Printf("Waiting for pod %s to be deleted ... (%ds/%d)", name, i, timeout)
 		time.Sleep(2 * time.Second)
 	}
@@ -628,6 +637,7 @@ func podRunningContainers(pod *coreapi.Pod) []string {
 		}
 		names = append(names, status.Name)
 	}
+	sort.Strings(names)
 	return names
 }