Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

e2e: wait for port-forwarder pods #850

Merged
merged 2 commits into from
Sep 4, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions e2e/internal/contrasttest/contrasttest.go
Original file line number Diff line number Diff line change
Expand Up @@ -321,6 +321,9 @@ func (ct *ContrastTest) runAgainstCoordinator(ctx context.Context, cmd *cobra.Co
if err := ct.Kubeclient.WaitFor(ctx, kubeclient.StatefulSet{}, ct.Namespace, "coordinator"); err != nil {
return fmt.Errorf("waiting for coordinator: %w", err)
}
if err := ct.Kubeclient.WaitFor(ctx, kubeclient.Pod{}, ct.Namespace, "port-forwarder-coordinator"); err != nil {
return fmt.Errorf("waiting for port-forwarder-coordinator: %w", err)
}

// Make the subcommand aware of the persistent flag.
// Do it outside the closure because declaring a flag twice panics.
Expand Down
27 changes: 26 additions & 1 deletion e2e/internal/kubeclient/deploy.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,29 @@ type ResourceWaiter interface {
getPods(context.Context, *Kubeclient, string, string) ([]corev1.Pod, error)
}

// Pod implements ResourceWaiter.
type Pod struct{}

func (p Pod) kind() string {
return "Pod"
}

func (p Pod) watcher(ctx context.Context, client *kubernetes.Clientset, namespace, name string) (watch.Interface, error) {
return client.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{FieldSelector: "metadata.name=" + name})
}

func (p Pod) numDesiredPods(_ any) (int, error) {
return 1, nil
}

func (p Pod) getPods(ctx context.Context, client *Kubeclient, namespace, name string) ([]corev1.Pod, error) {
pod, err := client.Client.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{})
if err != nil {
return nil, err
}
return []corev1.Pod{*pod}, nil
}

// Deployment implements ResourceWaiter.
type Deployment struct{}

Expand Down Expand Up @@ -141,6 +164,9 @@ func (c *Kubeclient) WaitForPod(ctx context.Context, namespace, name string) err
// WaitFor watches the given resource kind and blocks until the desired number of pods are
// ready or the context expires (is cancelled or times out).
func (c *Kubeclient) WaitFor(ctx context.Context, resource ResourceWaiter, namespace, name string) error {
logger := c.log.With("namespace", namespace)
logger.Info(fmt.Sprintf("Waiting for %s %s/%s to become ready", resource.kind(), namespace, name))

// When the node-installer restarts K3s, the watcher fails. The watcher has
// a retry loop internally, but it only retries starting the request, once
// it has established a request and that request dies spuriously, the
Expand Down Expand Up @@ -175,7 +201,6 @@ retryLoop:
}
return fmt.Errorf("watcher for %s %s/%s unexpectedly closed", resource.kind(), namespace, name)
}
logger := c.log.With("namespace", namespace)
logger.Error("resource did not become ready", "kind", resource, "name", name, "contextErr", ctx.Err())
if ctx.Err() != context.DeadlineExceeded {
return ctx.Err()
Expand Down
1 change: 1 addition & 0 deletions e2e/openssl/openssl_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ func TestOpenSSL(t *testing.T) {
require := require.New(t)

require.NoError(ct.Kubeclient.WaitFor(ctx, kubeclient.Deployment{}, ct.Namespace, opensslFrontend))
require.NoError(ct.Kubeclient.WaitFor(ctx, kubeclient.Pod{}, ct.Namespace, "port-forwarder-openssl-frontend"))

require.NoError(ct.Kubeclient.WithForwardedPort(ctx, ct.Namespace, "port-forwarder-openssl-frontend", "443", func(addr string) error {
dialer := &tls.Dialer{Config: &tls.Config{RootCAs: pool}}
Expand Down
1 change: 1 addition & 0 deletions e2e/servicemesh/servicemesh_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ func TestIngressEgress(t *testing.T) {
require.NoError(ct.Kubeclient.WaitFor(ctx, kubeclient.Deployment{}, ct.Namespace, "emoji"))
require.NoError(ct.Kubeclient.WaitFor(ctx, kubeclient.Deployment{}, ct.Namespace, "voting"))
require.NoError(ct.Kubeclient.WaitFor(ctx, kubeclient.Deployment{}, ct.Namespace, "web"))
require.NoError(ct.Kubeclient.WaitFor(ctx, kubeclient.Pod{}, ct.Namespace, "port-forwarder-web-svc"))
}), "deployments need to be ready for subsequent tests")

certs := map[string]*x509.CertPool{
Expand Down
4 changes: 3 additions & 1 deletion justfile
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,16 @@ default_platform := "AKS-CLH-SNP"
workspace_dir := "workspace"

# Build the node-installer, containerize and push it.
node-installer platform=default_platform: tardev-snapshotter nydus-snapshotter
node-installer platform=default_platform:
#!/usr/bin/env bash
set -euo pipefail
case {{ platform }} in
"AKS-CLH-SNP")
just push "tardev-snapshotter"
just push "node-installer-microsoft"
;;
"K3s-QEMU-SNP"|"K3s-QEMU-TDX"|"RKE2-QEMU-TDX")
just push "nydus-snapshotter"
just push "node-installer-kata"
;;
*)
Expand Down