From 4eeafc1ef8611c48c608495ca7ef41cd4e84687a Mon Sep 17 00:00:00 2001 From: Nick Santos Date: Tue, 28 Nov 2023 18:35:40 -0500 Subject: [PATCH] cluster: unify minikube and kind registry configs (#322) Signed-off-by: Nick Santos --- pkg/cluster/admin_helpers.go | 40 ++++++++++++++++++++++ pkg/cluster/admin_kind.go | 25 +------------- pkg/cluster/admin_minikube.go | 63 +++++++++-------------------------- 3 files changed, 57 insertions(+), 71 deletions(-) create mode 100644 pkg/cluster/admin_helpers.go diff --git a/pkg/cluster/admin_helpers.go b/pkg/cluster/admin_helpers.go new file mode 100644 index 0000000..6f3181b --- /dev/null +++ b/pkg/cluster/admin_helpers.go @@ -0,0 +1,40 @@ +package cluster + +import ( + "context" + "fmt" + "strings" + + "github.com/pkg/errors" + "github.com/tilt-dev/ctlptl/internal/exec" + "github.com/tilt-dev/ctlptl/pkg/api" + "k8s.io/cli-runtime/pkg/genericclioptions" +) + +func applyContainerdPatchRegistryApiV2( + ctx context.Context, runner exec.CmdRunner, iostreams genericclioptions.IOStreams, + nodes []string, desired *api.Cluster, registry *api.Registry) error { + for _, node := range nodes { + contents := fmt.Sprintf(`[host."http://%s:%d"] +`, registry.Name, registry.Status.ContainerPort) + + localRegistryDir := fmt.Sprintf("/etc/containerd/certs.d/localhost:%d", registry.Status.HostPort) + err := runner.RunIO(ctx, + genericclioptions.IOStreams{In: strings.NewReader(contents), Out: iostreams.Out, ErrOut: iostreams.ErrOut}, + "docker", "exec", "-i", node, "sh", "-c", + fmt.Sprintf("mkdir -p %s && cp /dev/stdin %s/hosts.toml", localRegistryDir, localRegistryDir)) + if err != nil { + return errors.Wrap(err, "configuring registry") + } + + networkRegistryDir := fmt.Sprintf("/etc/containerd/certs.d/%s:%d", registry.Name, registry.Status.ContainerPort) + err = runner.RunIO(ctx, + genericclioptions.IOStreams{In: strings.NewReader(contents), Out: iostreams.Out, ErrOut: iostreams.ErrOut}, + "docker", "exec", "-i", node, "sh", "-c", + fmt.Sprintf("mkdir -p %s && cp /dev/stdin %s/hosts.toml", networkRegistryDir, networkRegistryDir)) + if err != nil { + return errors.Wrap(err, "configuring registry") + } + } + return nil +} diff --git a/pkg/cluster/admin_kind.go b/pkg/cluster/admin_kind.go index 0013209..f20683b 100644 --- a/pkg/cluster/admin_kind.go +++ b/pkg/cluster/admin_kind.go @@ -187,30 +187,7 @@ func (a *kindAdmin) applyContainerdPatchRegistryApiV2(ctx context.Context, desir if err != nil { return errors.Wrap(err, "configuring registry") } - - for _, node := range nodes { - contents := fmt.Sprintf(`[host."http://%s:%d"] -`, registry.Name, registry.Status.ContainerPort) - - localRegistryDir := fmt.Sprintf("/etc/containerd/certs.d/localhost:%d", registry.Status.HostPort) - err := a.runner.RunIO(ctx, - genericclioptions.IOStreams{In: strings.NewReader(contents), Out: a.iostreams.Out, ErrOut: a.iostreams.ErrOut}, - "docker", "exec", "-i", node, "sh", "-c", - fmt.Sprintf("mkdir -p %s && cp /dev/stdin %s/hosts.toml", localRegistryDir, localRegistryDir)) - if err != nil { - return errors.Wrap(err, "configuring registry") - } - - networkRegistryDir := fmt.Sprintf("/etc/containerd/certs.d/%s:%d", registry.Name, registry.Status.ContainerPort) - err = a.runner.RunIO(ctx, - genericclioptions.IOStreams{In: strings.NewReader(contents), Out: a.iostreams.Out, ErrOut: a.iostreams.ErrOut}, - "docker", "exec", "-i", node, "sh", "-c", - fmt.Sprintf("mkdir -p %s && cp /dev/stdin %s/hosts.toml", networkRegistryDir, networkRegistryDir)) - if err != nil { - return errors.Wrap(err, "configuring registry") - } - } - return nil + return applyContainerdPatchRegistryApiV2(ctx, a.runner, a.iostreams, nodes, desired, registry) } func (a *kindAdmin) getNodes(ctx context.Context, cluster string) ([]string, error) { diff --git a/pkg/cluster/admin_minikube.go b/pkg/cluster/admin_minikube.go index 8b70c97..ec84335 100644 --- a/pkg/cluster/admin_minikube.go +++ b/pkg/cluster/admin_minikube.go @@ -224,17 +224,13 @@ func (a *minikubeAdmin) ensureRegistryDisconnected(ctx context.Context, registry return nil } -// We want to make sure that the image is pullable from either: -// localhost:[registry-port] or -// [registry-name]:5000 -// by cloning the registry config created by minikube's --insecure-registry. -func (a *minikubeAdmin) applyContainerdPatchRegistryApiV2(ctx context.Context, desired *api.Cluster, registry *api.Registry, networkMode container.NetworkMode) error { +func (a *minikubeAdmin) getNodes(ctx context.Context, name string) ([]string, error) { nodeOutput := bytes.NewBuffer(nil) err := a.runner.RunIO(ctx, genericclioptions.IOStreams{Out: nodeOutput, ErrOut: a.iostreams.ErrOut}, - "minikube", "-p", desired.Name, "node", "list") + "minikube", "-p", name, "node", "list") if err != nil { - return errors.Wrap(err, "configuring minikube registry") + return nil, errors.Wrap(err, "configuring minikube registry") } nodes := []string{} @@ -250,30 +246,20 @@ func (a *minikubeAdmin) applyContainerdPatchRegistryApiV2(ctx context.Context, d } nodes = append(nodes, node) } + return nodes, nil +} - for _, node := range nodes { - networkHost := registry.Status.IPAddress - if networkMode.IsUserDefined() { - networkHost = registry.Name - } - - err := a.runner.RunIO(ctx, - a.iostreams, - "minikube", "-p", desired.Name, "--node", node, - "ssh", "sudo", "cp", `\-R`, - fmt.Sprintf(`/etc/containerd/certs.d/%s\:%d`, networkHost, registry.Status.ContainerPort), - fmt.Sprintf(`/etc/containerd/certs.d/localhost\:%d`, registry.Status.HostPort)) - if err != nil { - return errors.Wrap(err, "configuring minikube registry") - } - - err = a.runner.RunIO(ctx, a.iostreams, "minikube", "-p", desired.Name, "--node", node, - "ssh", "sudo", "systemctl", "restart", "containerd") - if err != nil { - return errors.Wrap(err, "configuring minikube registry") - } +// We want to make sure that the image is pullable from either: +// localhost:[registry-port] or +// [registry-name]:5000 +// by cloning the registry config created by minikube's --insecure-registry. +func (a *minikubeAdmin) applyContainerdPatchRegistryApiV2(ctx context.Context, desired *api.Cluster, registry *api.Registry, networkMode container.NetworkMode) error { + nodes, err := a.getNodes(ctx, desired.Name) + if err != nil { + return errors.Wrap(err, "configuring minikube registry") } - return nil + + return applyContainerdPatchRegistryApiV2(ctx, a.runner, a.iostreams, nodes, desired, registry) } // We still patch containerd so that the user can push/pull from localhost. @@ -282,28 +268,11 @@ func (a *minikubeAdmin) applyContainerdPatchRegistryApiV2(ctx context.Context, d func (a *minikubeAdmin) applyContainerdPatchRegistryApiV1(ctx context.Context, desired *api.Cluster, registry *api.Registry, networkMode container.NetworkMode) error { configPath := "/etc/containerd/config.toml" - nodeOutput := bytes.NewBuffer(nil) - err := a.runner.RunIO(ctx, - genericclioptions.IOStreams{Out: nodeOutput, ErrOut: a.iostreams.ErrOut}, - "minikube", "-p", desired.Name, "node", "list") + nodes, err := a.getNodes(ctx, desired.Name) if err != nil { return errors.Wrap(err, "configuring minikube registry") } - nodes := []string{} - nodeOutputSplit := strings.Split(nodeOutput.String(), "\n") - for _, line := range nodeOutputSplit { - fields := strings.Fields(line) - if len(fields) == 0 { - continue - } - node := strings.TrimSpace(fields[0]) - if node == "" { - continue - } - nodes = append(nodes, node) - } - for _, node := range nodes { networkHost := registry.Status.IPAddress if networkMode.IsUserDefined() {