From c6b239bfa58c0217035e734f198a1cabeba3088a Mon Sep 17 00:00:00 2001 From: Derek Brown <6845676+DerekTBrown@users.noreply.github.com> Date: Mon, 6 Jan 2025 07:16:57 -0800 Subject: [PATCH] [feat] pull-through caching/proxying for images (#370) * [feat] add pull-through registry support Signed-off-by: Derek Brown <6845676+DerekTBrown@users.noreply.github.com> * address comments Signed-off-by: Derek Brown <6845676+DerekTBrown@users.noreply.github.comn> * nit Signed-off-by: Derek Brown <6845676+DerekTBrown@users.noreply.github.comn> * address comments Signed-off-by: Derek Brown <6845676+DerekTBrown@users.noreply.github.comn> --------- Signed-off-by: Derek Brown <6845676+DerekTBrown@users.noreply.github.com> Signed-off-by: Derek Brown <6845676+DerekTBrown@users.noreply.github.comn> Co-authored-by: Derek Brown <6845676+DerekTBrown@users.noreply.github.comn> --- examples/kind_registry_auth.yaml | 15 +++++++++++ pkg/api/types.go | 16 +++++++++++ pkg/cluster/admin_docker_desktop.go | 3 +++ pkg/cluster/admin_k3d.go | 3 +++ pkg/cluster/admin_kind.go | 42 +++++++++++++++++++++++++---- pkg/cluster/admin_kind_test.go | 37 +++++++++++++++++++++++++ pkg/cluster/admin_minikube.go | 3 +++ 7 files changed, 114 insertions(+), 5 deletions(-) create mode 100644 examples/kind_registry_auth.yaml diff --git a/examples/kind_registry_auth.yaml b/examples/kind_registry_auth.yaml new file mode 100644 index 0000000..08ca490 --- /dev/null +++ b/examples/kind_registry_auth.yaml @@ -0,0 +1,15 @@ +# Creates a kind cluster with Kind's custom cluster config +# +apiVersion: ctlptl.dev/v1alpha1 +kind: Cluster +product: kind +registry: ctlptl-registry +registryAuths: +- host: docker.io + endpoint: https://registry-1.docker.io + username: + password: +kindV1Alpha4Cluster: + name: my-cluster + nodes: + - role: control-plane diff --git a/pkg/api/types.go b/pkg/api/types.go index e146e33..3ce00f9 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -16,6 +16,17 @@ type TypeMeta struct { APIVersion string `json:"apiVersion,omitempty" yaml:"apiVersion,omitempty"` } +// RegistryAuth contains configuration for pull-through registries +type RegistryAuth struct { + // The FQDN of the registry (i.e. docker.io) + Host string `json:"host,omitempty" yaml:"host,omitempty"` + + // The Endpoint of the registry (i.e. https://registry-1.docker.io) + Endpoint string `json:"endpoint,omitempty" yaml:"endpoint,omitempty"` + Username string `json:"username,omitempty" yaml:"username,omitempty"` + Password string `json:"password,omitempty" yaml:"password,omitempty"` +} + // Cluster contains cluster configuration. // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object type Cluster struct { @@ -42,6 +53,11 @@ type Cluster struct { // Not supported on all cluster products. Registry string `json:"registry,omitempty" yaml:"registry,omitempty"` + // A list of pull-through registries to configure on the cluster. + // + // Not supported on all cluster products. + RegistryAuths []RegistryAuth `json:"registryAuths,omitempty" yaml:"registryAuths,omitempty"` + // The desired version of Kubernetes to run. // // Examples: diff --git a/pkg/cluster/admin_docker_desktop.go b/pkg/cluster/admin_docker_desktop.go index 3603ff1..263ad34 100644 --- a/pkg/cluster/admin_docker_desktop.go +++ b/pkg/cluster/admin_docker_desktop.go @@ -27,6 +27,9 @@ func (a *dockerDesktopAdmin) Create(ctx context.Context, desired *api.Cluster, r if registry != nil { return fmt.Errorf("ctlptl currently does not support connecting a registry to docker-desktop") } + if len(desired.RegistryAuths) > 0 { + return fmt.Errorf("ctlptl currently does not support connecting pull-through registries to docker-desktop") + } isLocalDockerDesktop := docker.IsLocalDockerDesktop(a.host, a.os) if !isLocalDockerDesktop { diff --git a/pkg/cluster/admin_k3d.go b/pkg/cluster/admin_k3d.go index 631abed..aaa0b8b 100644 --- a/pkg/cluster/admin_k3d.go +++ b/pkg/cluster/admin_k3d.go @@ -53,6 +53,9 @@ func (a *k3dAdmin) Create(ctx context.Context, desired *api.Cluster, registry *a if registry != nil { klog.V(3).Infof("Initializing cluster with registry config:\n%+v\n---\n", registry) } + if len(desired.RegistryAuths) > 0 { + return fmt.Errorf("ctlptl currently does not support connecting pull-through registries to k3d") + } k3dV, err := a.version(ctx) if err != nil { diff --git a/pkg/cluster/admin_kind.go b/pkg/cluster/admin_kind.go index a70c375..bcf442b 100644 --- a/pkg/cluster/admin_kind.go +++ b/pkg/cluster/admin_kind.go @@ -5,6 +5,7 @@ import ( "bytes" "context" "fmt" + "net/url" "os" "os/exec" "strings" @@ -56,7 +57,7 @@ func (a *kindAdmin) EnsureInstalled(ctx context.Context) error { return nil } -func (a *kindAdmin) kindClusterConfig(desired *api.Cluster, registry *api.Registry, registryAPI containerdRegistryAPI) *v1alpha4.Cluster { +func (a *kindAdmin) kindClusterConfig(desired *api.Cluster, registry *api.Registry, registryAPI containerdRegistryAPI) (*v1alpha4.Cluster, error) { kindConfig := desired.KindV1Alpha4Cluster if kindConfig == nil { kindConfig = &v1alpha4.Cluster{} @@ -67,7 +68,7 @@ func (a *kindAdmin) kindClusterConfig(desired *api.Cluster, registry *api.Regist kindConfig.APIVersion = "kind.x-k8s.io/v1alpha4" if registry != nil { - if registryAPI == containerdRegistryV2 { + if registryAPI == containerdRegistryV2 && len(desired.RegistryAuths) == 0 { // Point to the registry config path. // We'll add these files post-creation. patch := `[plugins."io.containerd.grpc.v1.cri".registry] @@ -84,7 +85,34 @@ func (a *kindAdmin) kindClusterConfig(desired *api.Cluster, registry *api.Regist kindConfig.ContainerdConfigPatches = append(kindConfig.ContainerdConfigPatches, patch) } } - return kindConfig + + for _, reg := range desired.RegistryAuths { + // Parse the endpoint + parsedEndpoint, err := url.Parse(reg.Endpoint) + if err != nil { + return nil, errors.Wrapf(err, "Error parsing registry endpoint: %s", reg.Endpoint) + } + + // Add the registry to the list of mirrors. + patch := fmt.Sprintf(`[plugins."io.containerd.grpc.v1.cri".registry.mirrors."%s"] + endpoint = ["%s"] +`, reg.Host, reg.Endpoint) + kindConfig.ContainerdConfigPatches = append(kindConfig.ContainerdConfigPatches, patch) + + // Specify the auth for the registry, if provided. + if reg.Username != "" || reg.Password != "" { + usernameValue := os.ExpandEnv(reg.Username) + passwordValue := os.ExpandEnv(reg.Password) + + patch := fmt.Sprintf(`[plugins."io.containerd.grpc.v1.cri".registry.configs."%s".auth] + username = "%s" + password = "%s" +`, parsedEndpoint.Host, usernameValue, passwordValue) + kindConfig.ContainerdConfigPatches = append(kindConfig.ContainerdConfigPatches, patch) + } + } + + return kindConfig, nil } func (a *kindAdmin) Create(ctx context.Context, desired *api.Cluster, registry *api.Registry) error { @@ -139,7 +167,11 @@ func (a *kindAdmin) Create(ctx context.Context, desired *api.Cluster, registry * args = append(args, "--image", node) } - kindConfig := a.kindClusterConfig(desired, registry, registryAPI) + kindConfig, err := a.kindClusterConfig(desired, registry, registryAPI) + if err != nil { + return errors.Wrap(err, "generating kind config") + } + buf := bytes.NewBuffer(nil) encoder := yaml.NewEncoder(buf) err = encoder.Encode(kindConfig) @@ -168,7 +200,7 @@ func (a *kindAdmin) Create(ctx context.Context, desired *api.Cluster, registry * } } - if registryAPI == containerdRegistryV2 { + if registryAPI == containerdRegistryV2 && len(desired.RegistryAuths) == 0 { err = a.applyContainerdPatchRegistryApiV2(ctx, desired, registry) if err != nil { return err diff --git a/pkg/cluster/admin_kind_test.go b/pkg/cluster/admin_kind_test.go index fdecd58..1302e3e 100644 --- a/pkg/cluster/admin_kind_test.go +++ b/pkg/cluster/admin_kind_test.go @@ -82,3 +82,40 @@ kind-control-plane2 "kind-control-plane2", }, nodeExec) } + +func TestKindClusterConfigWithPullThroughRegistries(t *testing.T) { + iostreams := genericclioptions.IOStreams{ + In: os.Stdin, + Out: os.Stdout, + ErrOut: os.Stderr, + } + runner := exec.NewFakeCmdRunner(func(argv []string) string { + return "" + }) + a := newKindAdmin(iostreams, runner, &fakeDockerClient{}) + + desired := &api.Cluster{ + RegistryAuths: []api.RegistryAuth{ + { + Host: "example.com", + Endpoint: "http://example.com:5000", + Username: "user", + Password: "pass", + }, + }, + } + + kindConfig, err := a.kindClusterConfig(desired, nil, containerdRegistryV2) + assert.NoError(t, err) + + expectedMirror := `[plugins."io.containerd.grpc.v1.cri".registry.mirrors."example.com"] + endpoint = ["http://example.com:5000"] +` + expectedAuth := `[plugins."io.containerd.grpc.v1.cri".registry.configs."example.com:5000".auth] + username = "user" + password = "pass" +` + + assert.Contains(t, kindConfig.ContainerdConfigPatches, expectedMirror) + assert.Contains(t, kindConfig.ContainerdConfigPatches, expectedAuth) +} diff --git a/pkg/cluster/admin_minikube.go b/pkg/cluster/admin_minikube.go index ec84335..d37fda7 100644 --- a/pkg/cluster/admin_minikube.go +++ b/pkg/cluster/admin_minikube.go @@ -83,6 +83,9 @@ func (a *minikubeAdmin) Create(ctx context.Context, desired *api.Cluster, regist if registry != nil { klog.V(3).Infof("Initializing cluster with registry config:\n%+v\n---\n", registry) } + if len(desired.RegistryAuths) > 0 { + return fmt.Errorf("ctlptl currently does not support connecting pull-through registries to minikube") + } v, err := a.version(ctx) if err != nil {