[feat] pull-through caching/proxying for images (#370)

* [feat] add pull-through registry support Signed-off-by: Derek Brown <[email protected]> * address comments Signed-off-by: Derek Brown <[email protected]> * nit Signed-off-by: Derek Brown <[email protected]> * address comments Signed-off-by: Derek Brown <[email protected]> --------- Signed-off-by: Derek Brown <[email protected]> Signed-off-by: Derek Brown <[email protected]> Co-authored-by: Derek Brown <[email protected]>
tilt-dev · Jan 6, 2025 · c6b239b · c6b239b
1 parent 835f4a6
commit c6b239b
Show file tree

Hide file tree

Showing 7 changed files with 114 additions and 5 deletions.
diff --git a/examples/kind_registry_auth.yaml b/examples/kind_registry_auth.yaml
@@ -0,0 +1,15 @@
+# Creates a kind cluster with Kind's custom cluster config
+#
+apiVersion: ctlptl.dev/v1alpha1
+kind: Cluster
+product: kind
+registry: ctlptl-registry
+registryAuths:
+- host: docker.io
+  endpoint: https://registry-1.docker.io
+  username: <docker hub username>
+  password: <docker hub token>
+kindV1Alpha4Cluster:
+  name: my-cluster
+  nodes:
+  - role: control-plane
diff --git a/pkg/api/types.go b/pkg/api/types.go
@@ -16,6 +16,17 @@ type TypeMeta struct {
 	APIVersion string `json:"apiVersion,omitempty" yaml:"apiVersion,omitempty"`
 }
 
+// RegistryAuth contains configuration for pull-through registries
+type RegistryAuth struct {
+	// The FQDN of the registry (i.e. docker.io)
+	Host string `json:"host,omitempty" yaml:"host,omitempty"`
+
+	// The Endpoint of the registry (i.e. https://registry-1.docker.io)
+	Endpoint string `json:"endpoint,omitempty" yaml:"endpoint,omitempty"`
+	Username string `json:"username,omitempty" yaml:"username,omitempty"`
+	Password string `json:"password,omitempty" yaml:"password,omitempty"`
+}
+
 // Cluster contains cluster configuration.
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 type Cluster struct {
@@ -42,6 +53,11 @@ type Cluster struct {
 	// Not supported on all cluster products.
 	Registry string `json:"registry,omitempty" yaml:"registry,omitempty"`
 
+	// A list of pull-through registries to configure on the cluster.
+	//
+	// Not supported on all cluster products.
+	RegistryAuths []RegistryAuth `json:"registryAuths,omitempty" yaml:"registryAuths,omitempty"`
+
 	// The desired version of Kubernetes to run.
 	//
 	// Examples:

diff --git a/pkg/cluster/admin_docker_desktop.go b/pkg/cluster/admin_docker_desktop.go
@@ -27,6 +27,9 @@ func (a *dockerDesktopAdmin) Create(ctx context.Context, desired *api.Cluster, r
 	if registry != nil {
 		return fmt.Errorf("ctlptl currently does not support connecting a registry to docker-desktop")
 	}
+	if len(desired.RegistryAuths) > 0 {
+		return fmt.Errorf("ctlptl currently does not support connecting pull-through registries to docker-desktop")
+	}
 
 	isLocalDockerDesktop := docker.IsLocalDockerDesktop(a.host, a.os)
 	if !isLocalDockerDesktop {

diff --git a/pkg/cluster/admin_k3d.go b/pkg/cluster/admin_k3d.go
@@ -53,6 +53,9 @@ func (a *k3dAdmin) Create(ctx context.Context, desired *api.Cluster, registry *a
 	if registry != nil {
 		klog.V(3).Infof("Initializing cluster with registry config:\n%+v\n---\n", registry)
 	}
+	if len(desired.RegistryAuths) > 0 {
+		return fmt.Errorf("ctlptl currently does not support connecting pull-through registries to k3d")
+	}
 
 	k3dV, err := a.version(ctx)
 	if err != nil {

diff --git a/pkg/cluster/admin_kind.go b/pkg/cluster/admin_kind.go
@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"net/url"
 	"os"
 	"os/exec"
 	"strings"
@@ -56,7 +57,7 @@ func (a *kindAdmin) EnsureInstalled(ctx context.Context) error {
 	return nil
 }
 
-func (a *kindAdmin) kindClusterConfig(desired *api.Cluster, registry *api.Registry, registryAPI containerdRegistryAPI) *v1alpha4.Cluster {
+func (a *kindAdmin) kindClusterConfig(desired *api.Cluster, registry *api.Registry, registryAPI containerdRegistryAPI) (*v1alpha4.Cluster, error) {
 	kindConfig := desired.KindV1Alpha4Cluster
 	if kindConfig == nil {
 		kindConfig = &v1alpha4.Cluster{}
@@ -67,7 +68,7 @@ func (a *kindAdmin) kindClusterConfig(desired *api.Cluster, registry *api.Regist
 	kindConfig.APIVersion = "kind.x-k8s.io/v1alpha4"
 
 	if registry != nil {
-		if registryAPI == containerdRegistryV2 {
+		if registryAPI == containerdRegistryV2 && len(desired.RegistryAuths) == 0 {
 			// Point to the registry config path.
 			// We'll add these files post-creation.
 			patch := `[plugins."io.containerd.grpc.v1.cri".registry]
@@ -84,7 +85,34 @@ func (a *kindAdmin) kindClusterConfig(desired *api.Cluster, registry *api.Regist
 			kindConfig.ContainerdConfigPatches = append(kindConfig.ContainerdConfigPatches, patch)
 		}
 	}
-	return kindConfig
+
+	for _, reg := range desired.RegistryAuths {
+		// Parse the endpoint
+		parsedEndpoint, err := url.Parse(reg.Endpoint)
+		if err != nil {
+			return nil, errors.Wrapf(err, "Error parsing registry endpoint: %s", reg.Endpoint)
+		}
+
+		// Add the registry to the list of mirrors.
+		patch := fmt.Sprintf(`[plugins."io.containerd.grpc.v1.cri".registry.mirrors."%s"]
+  endpoint = ["%s"]
+`, reg.Host, reg.Endpoint)
+		kindConfig.ContainerdConfigPatches = append(kindConfig.ContainerdConfigPatches, patch)
+
+		// Specify the auth for the registry, if provided.
+		if reg.Username != "" || reg.Password != "" {
+			usernameValue := os.ExpandEnv(reg.Username)
+			passwordValue := os.ExpandEnv(reg.Password)
+
+			patch := fmt.Sprintf(`[plugins."io.containerd.grpc.v1.cri".registry.configs."%s".auth]
+  username = "%s"
+  password = "%s"
+`, parsedEndpoint.Host, usernameValue, passwordValue)
+			kindConfig.ContainerdConfigPatches = append(kindConfig.ContainerdConfigPatches, patch)
+		}
+	}
+
+	return kindConfig, nil
 }
 
 func (a *kindAdmin) Create(ctx context.Context, desired *api.Cluster, registry *api.Registry) error {
@@ -139,7 +167,11 @@ func (a *kindAdmin) Create(ctx context.Context, desired *api.Cluster, registry *
 		args = append(args, "--image", node)
 	}
 
-	kindConfig := a.kindClusterConfig(desired, registry, registryAPI)
+	kindConfig, err := a.kindClusterConfig(desired, registry, registryAPI)
+	if err != nil {
+		return errors.Wrap(err, "generating kind config")
+	}
+
 	buf := bytes.NewBuffer(nil)
 	encoder := yaml.NewEncoder(buf)
 	err = encoder.Encode(kindConfig)
@@ -168,7 +200,7 @@ func (a *kindAdmin) Create(ctx context.Context, desired *api.Cluster, registry *
 			}
 		}
 
-		if registryAPI == containerdRegistryV2 {
+		if registryAPI == containerdRegistryV2 && len(desired.RegistryAuths) == 0 {
 			err = a.applyContainerdPatchRegistryApiV2(ctx, desired, registry)
 			if err != nil {
 				return err

diff --git a/pkg/cluster/admin_kind_test.go b/pkg/cluster/admin_kind_test.go
@@ -82,3 +82,40 @@ kind-control-plane2
 		"kind-control-plane2",
 	}, nodeExec)
 }
+
+func TestKindClusterConfigWithPullThroughRegistries(t *testing.T) {
+	iostreams := genericclioptions.IOStreams{
+		In:     os.Stdin,
+		Out:    os.Stdout,
+		ErrOut: os.Stderr,
+	}
+	runner := exec.NewFakeCmdRunner(func(argv []string) string {
+		return ""
+	})
+	a := newKindAdmin(iostreams, runner, &fakeDockerClient{})
+
+	desired := &api.Cluster{
+		RegistryAuths: []api.RegistryAuth{
+			{
+				Host:     "example.com",
+				Endpoint: "http://example.com:5000",
+				Username: "user",
+				Password: "pass",
+			},
+		},
+	}
+
+	kindConfig, err := a.kindClusterConfig(desired, nil, containerdRegistryV2)
+	assert.NoError(t, err)
+
+	expectedMirror := `[plugins."io.containerd.grpc.v1.cri".registry.mirrors."example.com"]
+  endpoint = ["http://example.com:5000"]
+`
+	expectedAuth := `[plugins."io.containerd.grpc.v1.cri".registry.configs."example.com:5000".auth]
+  username = "user"
+  password = "pass"
+`
+
+	assert.Contains(t, kindConfig.ContainerdConfigPatches, expectedMirror)
+	assert.Contains(t, kindConfig.ContainerdConfigPatches, expectedAuth)
+}
diff --git a/pkg/cluster/admin_minikube.go b/pkg/cluster/admin_minikube.go
@@ -83,6 +83,9 @@ func (a *minikubeAdmin) Create(ctx context.Context, desired *api.Cluster, regist
 	if registry != nil {
 		klog.V(3).Infof("Initializing cluster with registry config:\n%+v\n---\n", registry)
 	}
+	if len(desired.RegistryAuths) > 0 {
+		return fmt.Errorf("ctlptl currently does not support connecting pull-through registries to minikube")
+	}
 
 	v, err := a.version(ctx)
 	if err != nil {