[feat] implement pull-through caching

Signed-off-by: Derek Brown <[email protected]>

docs/ctlptl_create_registry.md

@@ -12,6 +12,7 @@ ctlptl create registry [name] [flags]
   ctlptl create registry ctlptl-registry
   ctlptl create registry ctlptl-registry --port=5000
   ctlptl create registry ctlptl-registry --port=5000 --listen-address 0.0.0.0
+  ctlptl create registry ctlptl-pull-through-registry --proxy-remote-url=https://registry-1.docker.io
 ```
 
 ### Options
@@ -27,6 +28,15 @@ ctlptl create registry [name] [flags]
       --template string               Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].
 ```
 
+### Remote proxy options
+If `--remote-proxy-url` is specified, the registry is configured as a pull-through cache:
+```
+      --proxy-remote-url string       The remote URL for the pull-through proxy
+      --proxy-username string         The username for the pull-through proxy authentication
+      --proxy-password string         The password for the pull-through proxy authentication. 
+      --proxy-ttl string              The TTL for the pull-through proxy cache
+```
+
 ### SEE ALSO
 
 * [ctlptl create](ctlptl_create.md)	 - Create a cluster or registry

examples/pull-through-registry.yaml

@@ -0,0 +1,25 @@
+# Creates a registry called ctlptl-registry available on 127.0.0.1:5002
+apiVersion: ctlptl.dev/v1alpha1
+kind: Registry
+name: ctlptl-registry
+port: 5002
+listenAddress: 127.0.0.1
+---
+# Creates a pull-through registry called ctlptl-pull-through-registry available on 127.0.0.1:5003
+apiVersion: ctlptl.dev/v1alpha1
+kind: Registry
+name: ctlptl-pull-through-registry
+port: 5003
+listenAddress: 127.0.0.1
+proxy:
+  remoteURL: "https://registry-1.docker.io"
+  username: "my-username"
+  password: "$MY_PASSWORD_ENV"
+---
+# Create a Kind cluster with the pull-through registry
+apiVersion: ctlptl.dev/v1alpha1
+kind: Cluster
+product: kind
+registry: ctlptl-registry
+pullThroughRegistries:
+  - ctlptl-pull-through-registry

internal/dctr/run.go

@@ -152,6 +152,7 @@ func Run(ctx context.Context, cli CLI, name string, config *container.Config, ho
 
 	id := resp.ID
 	err = c.ContainerStart(ctx, id, container.StartOptions{})
+
 	if err != nil {
 		return fmt.Errorf("starting %s: %v", name, err)
 	}

pkg/api/types.go

@@ -42,6 +42,13 @@ type Cluster struct {
 	// Not supported on all cluster products.
 	Registry string `json:"registry,omitempty" yaml:"registry,omitempty"`
 
+	// The name of pull-through registries to add to the cluster.
+	//
+	// These registries must already exist. `ctlptl` will fail if they don't.
+	//
+	// Not supported on all cluster products.
+	PullThroughRegistries []string `json:"pullThroughRegistries,omitempty" yaml:"pullThroughRegistries,omitempty"`
+
 	// The desired version of Kubernetes to run.
 	//
 	// Examples:
@@ -160,6 +167,15 @@ type ClusterList struct {
 	Items []Cluster `json:"items" protobuf:"bytes,2,rep,name=items"`
 }
 
+// RegistryProxySpec describes the configuration for a pull-through registry.
+// See: https://distribution.github.io/distribution/recipes/mirror/#run-a-registry-as-a-pull-through-cache
+type RegistryProxySpec struct {
+	RemoteURL string `json:"remoteURL,omitempty" yaml:"remoteURL,omitempty"`
+	Username  string `json:"username,omitempty" yaml:"username,omitempty"`
+	Password  string `json:"password,omitempty" yaml:"password,omitempty"`
+	TTL       string `json:"ttl,omitempty" yaml:"ttl,omitempty"`
+}
+
 // Cluster contains registry configuration.
 //
 // Currently designed for local registries on the host machine, but
@@ -201,6 +217,10 @@ type Registry struct {
 	// Defaults to `docker.io/library/registry:2`.
 	Image string `json:"image,omitempty" yaml:"image,omitempty"`
 
+	// Proxy configuration for a pull-through registry.
+	// If provided, the registry will be configured as a pull-through registry.
+	Proxy *RegistryProxySpec `json:"proxy,omitempty" yaml:"proxy,omitempty"`
+
 	// Most recently observed status of the registry.
 	// Populated by the system.
 	// Read-only.

pkg/cluster/admin.go

@@ -18,7 +18,7 @@ type Admin interface {
 	//
 	// Make a best effort attempt to delete any resources that might block creation
 	// of the cluster.
-	Create(ctx context.Context, desired *api.Cluster, registry *api.Registry) error
+	Create(ctx context.Context, desired *api.Cluster, registry *api.Registry, pullThroughRegistries []*api.Registry) error
 
 	// Infers the LocalRegistryHosting that this admin will try to configure.
 	LocalRegistryHosting(ctx context.Context, desired *api.Cluster, registry *api.Registry) (*localregistry.LocalRegistryHostingV1, error)

pkg/cluster/admin_docker_desktop.go

@@ -23,10 +23,13 @@ func newDockerDesktopAdmin(host string, os string, d4m d4mClient) *dockerDesktop
 }
 
 func (a *dockerDesktopAdmin) EnsureInstalled(ctx context.Context) error { return nil }
-func (a *dockerDesktopAdmin) Create(ctx context.Context, desired *api.Cluster, registry *api.Registry) error {
+func (a *dockerDesktopAdmin) Create(ctx context.Context, desired *api.Cluster, registry *api.Registry, pullThroughRegistries []*api.Registry) error {
 	if registry != nil {
 		return fmt.Errorf("ctlptl currently does not support connecting a registry to docker-desktop")
 	}
+	if len(pullThroughRegistries) > 0 {
+		return fmt.Errorf("ctlptl currently does not support connecting pull-through registries to docker-desktop")
+	}
 
 	isLocalDockerDesktop := docker.IsLocalDockerDesktop(a.host, a.os)
 	if !isLocalDockerDesktop {

pkg/cluster/admin_k3d.go

@@ -48,11 +48,14 @@ func (a *k3dAdmin) EnsureInstalled(ctx context.Context) error {
 	return nil
 }
 
-func (a *k3dAdmin) Create(ctx context.Context, desired *api.Cluster, registry *api.Registry) error {
+func (a *k3dAdmin) Create(ctx context.Context, desired *api.Cluster, registry *api.Registry, pullThroughRegistries []*api.Registry) error {
 	klog.V(3).Infof("Creating cluster with config:\n%+v\n---\n", desired)
 	if registry != nil {
 		klog.V(3).Infof("Initializing cluster with registry config:\n%+v\n---\n", registry)
 	}
+	if len(pullThroughRegistries) > 0 {
+		return fmt.Errorf("ctlptl currently does not support connecting pull-through registries to docker-desktop")
+	}
 
 	k3dV, err := a.version(ctx)
 	if err != nil {

pkg/cluster/admin_k3d_test.go

@@ -27,7 +27,7 @@ func TestK3DStartFlagsV4(t *testing.T) {
 
 	err = f.a.Create(ctx, &api.Cluster{
 		Name: "k3d-my-cluster",
-	}, &api.Registry{Name: "my-reg"})
+	}, &api.Registry{Name: "my-reg"}, []*api.Registry{})
 	assert.NoError(t, err)
 	assert.Equal(t, []string{
 		"k3d", "cluster", "create", "my-cluster",
@@ -50,7 +50,7 @@ func TestK3DStartFlagsV5(t *testing.T) {
 				Network: "bar",
 			},
 		},
-	}, &api.Registry{Name: "my-reg"})
+	}, &api.Registry{Name: "my-reg"}, []*api.Registry{})
 	require.NoError(t, err)
 	assert.Equal(t, []string{
 		"k3d", "cluster", "create", "my-cluster",
@@ -82,7 +82,7 @@ func TestK3DV1alpha5File(t *testing.T) {
 				Network: "bar",
 			},
 		},
-	}, &api.Registry{Name: "my-reg"})
+	}, &api.Registry{Name: "my-reg"}, []*api.Registry{})
 	require.NoError(t, err)
 	assert.Equal(t, []string{
 		"k3d", "cluster", "create", "my-cluster",
@@ -106,7 +106,7 @@ func TestK3DV1alpha4FileOnOldVersions(t *testing.T) {
 	ctx := context.Background()
 	err := f.a.Create(ctx, &api.Cluster{
 		Name: "k3d-my-cluster",
-	}, &api.Registry{Name: "my-reg"})
+	}, &api.Registry{Name: "my-reg"}, []*api.Registry{})
 	require.NoError(t, err)
 	assert.Equal(t, []string{
 		"k3d", "cluster", "create", "my-cluster",

pkg/cluster/admin_kind.go

@@ -56,7 +56,7 @@ func (a *kindAdmin) EnsureInstalled(ctx context.Context) error {
 	return nil
 }
 
-func (a *kindAdmin) kindClusterConfig(desired *api.Cluster, registry *api.Registry, registryAPI containerdRegistryAPI) *v1alpha4.Cluster {
+func (a *kindAdmin) kindClusterConfig(ctx context.Context, desired *api.Cluster, registry *api.Registry, pullThroughRegistries []*api.Registry, registryAPI containerdRegistryAPI) (*v1alpha4.Cluster, error) {
 	kindConfig := desired.KindV1Alpha4Cluster
 	if kindConfig == nil {
 		kindConfig = &v1alpha4.Cluster{}
@@ -84,10 +84,20 @@ func (a *kindAdmin) kindClusterConfig(desired *api.Cluster, registry *api.Regist
 			kindConfig.ContainerdConfigPatches = append(kindConfig.ContainerdConfigPatches, patch)
 		}
 	}
-	return kindConfig
+
+	for _, reg := range pullThroughRegistries {
+		if reg.Proxy != nil && reg.Proxy.RemoteURL != "" {
+			patch := fmt.Sprintf(`[plugins."io.containerd.grpc.v1.cri".registry.mirrors."%s"]
+  endpoint = ["http://%s:%d"]
+`, reg.Proxy.RemoteURL, reg.Name, reg.Status.ContainerPort)
+			kindConfig.ContainerdConfigPatches = append(kindConfig.ContainerdConfigPatches, patch)
+		}
+	}
+
+	return kindConfig, nil
 }
 
-func (a *kindAdmin) Create(ctx context.Context, desired *api.Cluster, registry *api.Registry) error {
+func (a *kindAdmin) Create(ctx context.Context, desired *api.Cluster, registry *api.Registry, pullThroughRegistries []*api.Registry) error {
 	klog.V(3).Infof("Creating cluster with config:\n%+v\n---\n", desired)
 	if registry != nil {
 		klog.V(3).Infof("Initializing cluster with registry config:\n%+v\n---\n", registry)
@@ -139,7 +149,10 @@ func (a *kindAdmin) Create(ctx context.Context, desired *api.Cluster, registry *
 		args = append(args, "--image", node)
 	}
 
-	kindConfig := a.kindClusterConfig(desired, registry, registryAPI)
+	kindConfig, err := a.kindClusterConfig(ctx, desired, registry, pullThroughRegistries, registryAPI)
+	if err != nil {
+		return errors.Wrap(err, "creating kind cluster")
+	}
 	buf := bytes.NewBuffer(nil)
 	encoder := yaml.NewEncoder(buf)
 	err = encoder.Encode(kindConfig)

pkg/cluster/admin_kind_test.go

@@ -82,3 +82,36 @@ kind-control-plane2
 		"kind-control-plane2",
 	}, nodeExec)
 }
+
+func TestKindClusterConfigWithPullThroughRegistries(t *testing.T) {
+	runner := exec.NewFakeCmdRunner(func(argv []string) string {
+		return ""
+	})
+	iostreams := genericclioptions.IOStreams{
+		In:     os.Stdin,
+		Out:    os.Stdout,
+		ErrOut: os.Stderr,
+	}
+	a := newKindAdmin(iostreams, runner, &fakeDockerClient{})
+	ctx := context.Background()
+
+	pullThroughRegistries := []*api.Registry{
+		{
+			Name: "test-registry",
+			Proxy: &api.RegistryProxySpec{
+				RemoteURL: "remote.registry",
+			},
+			Status: api.RegistryStatus{
+				ContainerPort: 5000,
+			},
+		},
+	}
+
+	kindConfig, err := a.kindClusterConfig(ctx, &api.Cluster{}, nil, pullThroughRegistries, containerdRegistryV2)
+	assert.NoError(t, err)
+
+	expectedPatch := `[plugins."io.containerd.grpc.v1.cri".registry.mirrors."remote.registry"]
+  endpoint = ["http://test-registry:5000"]
+`
+	assert.Contains(t, kindConfig.ContainerdConfigPatches, expectedPatch)
+}

pkg/cluster/admin_minikube.go

@@ -78,11 +78,14 @@ func (a *minikubeAdmin) version(ctx context.Context) (semver.Version, error) {
 	return result, nil
 }
 
-func (a *minikubeAdmin) Create(ctx context.Context, desired *api.Cluster, registry *api.Registry) error {
+func (a *minikubeAdmin) Create(ctx context.Context, desired *api.Cluster, registry *api.Registry, pullThroughRegistries []*api.Registry) error {
 	klog.V(3).Infof("Creating cluster with config:\n%+v\n---\n", desired)
 	if registry != nil {
 		klog.V(3).Infof("Initializing cluster with registry config:\n%+v\n---\n", registry)
 	}
+	if len(pullThroughRegistries) > 0 {
+		return fmt.Errorf("ctlptl currently does not support connecting pull-through registries to docker-desktop")
+	}
 
 	v, err := a.version(ctx)
 	if err != nil {

pkg/cluster/admin_minikube_test.go

@@ -16,7 +16,7 @@ import (
 func TestMinikubeStartFlags(t *testing.T) {
 	f := newMinikubeFixture()
 	ctx := context.Background()
-	err := f.a.Create(ctx, &api.Cluster{Name: "minikube", Minikube: &api.MinikubeCluster{StartFlags: []string{"--foo"}}}, nil)
+	err := f.a.Create(ctx, &api.Cluster{Name: "minikube", Minikube: &api.MinikubeCluster{StartFlags: []string{"--foo"}}}, nil, []*api.Registry{})
 	require.NoError(t, err)
 	assert.Equal(t, []string{
 		"minikube", "start",

pkg/cluster/cluster.go

@@ -668,6 +668,30 @@ func (c *Controller) ensureRegistryExistsForCluster(ctx context.Context, desired
 	})
 }
 
+// Get the pull-through registries for the given cluster.
+func (c *Controller) pullThroughRegistries(ctx context.Context, desired *api.Cluster) ([]*api.Registry, error) {
+	regCtl, err := c.registryController(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	// Filter for all the registries that are in the desired.PullThroughRegistries list.
+	pullThroughRegistries := make([]*api.Registry, 0)
+	allRegistries, err := regCtl.List(ctx, registry.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+	for _, reg := range allRegistries.Items {
+		for _, desiredRegName := range desired.PullThroughRegistries {
+			if reg.Name == desiredRegName {
+				pullThroughRegistries = append(pullThroughRegistries, &reg)
+			}
+		}
+	}
+
+	return pullThroughRegistries, nil
+}
+
 // Compare the desired cluster against the existing cluster, and reconcile
 // the two to match.
 func (c *Controller) Apply(ctx context.Context, desired *api.Cluster) (*api.Cluster, error) {
@@ -748,17 +772,24 @@ func (c *Controller) Apply(ctx context.Context, desired *api.Cluster) (*api.Clus
 		}
 	}
 
+	// Ensure the registry exists.
 	reg, err := c.ensureRegistryExistsForCluster(ctx, desired)
 	if err != nil {
 		return nil, err
 	}
 
+	// Fetch the pull-through registries.
+	pullThroughRegistries, err := c.pullThroughRegistries(ctx, desired)
+	if err != nil {
+		return nil, err
+	}
+
 	// Configure the cluster to match what we want.
 	needsCreate := existingStatus.CreationTimestamp.Time.IsZero() ||
 		desired.Name != existingCluster.Name ||
 		desired.Product != existingCluster.Product
 	if needsCreate {
-		err := admin.Create(ctx, desired, reg)
+		err := admin.Create(ctx, desired, reg, pullThroughRegistries)
 		if err != nil {
 			return nil, err
 		}

pkg/cluster/cluster_test.go

@@ -731,7 +731,7 @@ func newFakeAdmin(config *clientcmdapi.Config, fakeK8s *fake.Clientset) *fakeAdm
 
 func (a *fakeAdmin) EnsureInstalled(ctx context.Context) error { return nil }
 
-func (a *fakeAdmin) Create(ctx context.Context, config *api.Cluster, registry *api.Registry) error {
+func (a *fakeAdmin) Create(ctx context.Context, config *api.Cluster, registry *api.Registry, registries []*api.Registry) error {
 	a.created = config.DeepCopy()
 	a.createdRegistry = registry.DeepCopy()
 	a.config.Contexts[config.Name] = &clientcmdapi.Context{Cluster: config.Name}

pkg/cmd/create_registry.go

@@ -38,7 +38,8 @@ func (o *CreateRegistryOptions) Command() *cobra.Command {
 		Short: "Create a registry with the given name",
 		Example: "  ctlptl create registry ctlptl-registry\n" +
 			"  ctlptl create registry ctlptl-registry --port=5000\n" +
-			"  ctlptl create registry ctlptl-registry --port=5000 --listen-address 0.0.0.0",
+			"  ctlptl create registry ctlptl-registry --port=5000 --listen-address 0.0.0.0\n" +
+			"  ctlptl create registry ctlptl-pull-through-registry --proxy-remote-url=https://registry-1.docker.io",
 		Run:  o.Run,
 		Args: cobra.ExactArgs(1),
 	}
@@ -53,6 +54,28 @@ func (o *CreateRegistryOptions) Command() *cobra.Command {
 	cmd.Flags().StringVar(&o.Registry.Image, "image", registry.DefaultRegistryImageRef,
 		"Registry image to use")
 
+	// Initialize Proxy only if any proxy-related flag is set
+	var proxyRemoteURL, proxyUsername, proxyPassword, proxyTTL string
+	cmd.Flags().StringVar(&proxyRemoteURL, "proxy-remote-url", "",
+		"The remote URL for the pull-through proxy")
+	cmd.Flags().StringVar(&proxyUsername, "proxy-username", "",
+		"The username for the pull-through proxy authentication")
+	cmd.Flags().StringVar(&proxyPassword, "proxy-password", "",
+		"The password for the pull-through proxy authentication")
+	cmd.Flags().StringVar(&proxyTTL, "proxy-ttl", "",
+		"The TTL for the pull-through proxy cache")
+
+	cmd.PreRun = func(cmd *cobra.Command, args []string) {
+		if proxyRemoteURL != "" {
+			o.Registry.Proxy = &api.RegistryProxySpec{
+				RemoteURL: proxyRemoteURL,
+				Username:  proxyUsername,
+				Password:  proxyPassword,
+				TTL:       proxyTTL,
+			}
+		}
+	}
+
 	return cmd
 }