[feat] implement pull-through caching

Signed-off-by: Derek Brown <[email protected]>

docs/ctlptl_create_registry.md

@@ -12,6 +12,7 @@ ctlptl create registry [name] [flags]
   ctlptl create registry ctlptl-registry
   ctlptl create registry ctlptl-registry --port=5000
   ctlptl create registry ctlptl-registry --port=5000 --listen-address 0.0.0.0
+  ctlptl create registry ctlptl-pull-through-registry --proxy-remote-url=https://registry-1.docker.io
 ```
 
 ### Options
@@ -27,6 +28,15 @@ ctlptl create registry [name] [flags]
       --template string               Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].
 ```
 
+### Remote proxy options
+If `--remote-proxy-url` is specified, the registry is configured as a pull-through cache:
+```
+      --proxy-remote-url string       The remote URL for the pull-through proxy
+      --proxy-username string         The username for the pull-through proxy authentication
+      --proxy-password string         The password for the pull-through proxy authentication. 
+      --proxy-ttl string              The TTL for the pull-through proxy cache
+```
+
 ### SEE ALSO
 
 * [ctlptl create](ctlptl_create.md)	 - Create a cluster or registry

examples/pull-through-registry.yaml

@@ -0,0 +1,25 @@
+# Creates a registry called ctlptl-registry available on 127.0.0.1:5002
+apiVersion: ctlptl.dev/v1alpha1
+kind: Registry
+name: ctlptl-registry
+port: 5002
+listenAddress: 127.0.0.1
+---
+# Creates a pull-through registry called ctlptl-pull-through-registry available on 127.0.0.1:5003
+apiVersion: ctlptl.dev/v1alpha1
+kind: Registry
+name: ctlptl-pull-through-registry
+port: 5003
+listenAddress: 127.0.0.1
+proxy:
+  remoteURL: "https://registry-1.docker.io"
+  username: "my-username"
+  password: "$MY_PASSWORD_ENV"
+---
+# Create a Kind cluster with the pull-through registry
+apiVersion: ctlptl.dev/v1alpha1
+kind: Cluster
+product: kind
+registry: ctlptl-registry
+pullThroughRegistries:
+  - ctlptl-pull-through-registry

internal/dctr/run.go

@@ -152,6 +152,7 @@ func Run(ctx context.Context, cli CLI, name string, config *container.Config, ho
 
 	id := resp.ID
 	err = c.ContainerStart(ctx, id, container.StartOptions{})
+
 	if err != nil {
 		return fmt.Errorf("starting %s: %v", name, err)
 	}

pkg/api/types.go

@@ -42,6 +42,13 @@ type Cluster struct {
 	// Not supported on all cluster products.
 	Registry string `json:"registry,omitempty" yaml:"registry,omitempty"`
 
+	// The name of pull-through registries to add to the cluster.
+	//
+	// These registries must already exist. `ctlptl` will fail if they don't.
+	//
+	// Not supported on all cluster products.
+	PullThroughRegistries []string `json:"pullThroughRegistries,omitempty" yaml:"pullThroughRegistries,omitempty"`
+
 	// The desired version of Kubernetes to run.
 	//
 	// Examples:
@@ -160,6 +167,15 @@ type ClusterList struct {
 	Items []Cluster `json:"items" protobuf:"bytes,2,rep,name=items"`
 }
 
+// RegistryProxySpec describes the configuration for a pull-through registry.
+// See: https://distribution.github.io/distribution/recipes/mirror/#run-a-registry-as-a-pull-through-cache
+type RegistryProxySpec struct {
+	RemoteURL string `json:"remoteURL,omitempty" yaml:"remoteURL,omitempty"`
+	Username  string `json:"username,omitempty" yaml:"username,omitempty"`
+	Password  string `json:"password,omitempty" yaml:"password,omitempty"`
+	TTL       string `json:"ttl,omitempty" yaml:"ttl,omitempty"`
+}
+
 // Cluster contains registry configuration.
 //
 // Currently designed for local registries on the host machine, but
@@ -201,6 +217,10 @@ type Registry struct {
 	// Defaults to `docker.io/library/registry:2`.
 	Image string `json:"image,omitempty" yaml:"image,omitempty"`
 
+	// Proxy configuration for a pull-through registry.
+	// If provided, the registry will be configured as a pull-through registry.
+	Proxy *RegistryProxySpec `json:"proxy,omitempty" yaml:"proxy,omitempty"`
+
 	// Most recently observed status of the registry.
 	// Populated by the system.
 	// Read-only.

pkg/cluster/admin.go

@@ -18,7 +18,7 @@ type Admin interface {
 	//
 	// Make a best effort attempt to delete any resources that might block creation
 	// of the cluster.
-	Create(ctx context.Context, desired *api.Cluster, registry *api.Registry) error
+	Create(ctx context.Context, desired *api.Cluster, registry *api.Registry, pullThroughRegistries []*api.Registry) error
 
 	// Infers the LocalRegistryHosting that this admin will try to configure.
 	LocalRegistryHosting(ctx context.Context, desired *api.Cluster, registry *api.Registry) (*localregistry.LocalRegistryHostingV1, error)

pkg/cluster/admin_docker_desktop.go

@@ -23,10 +23,13 @@ func newDockerDesktopAdmin(host string, os string, d4m d4mClient) *dockerDesktop
 }
 
 func (a *dockerDesktopAdmin) EnsureInstalled(ctx context.Context) error { return nil }
-func (a *dockerDesktopAdmin) Create(ctx context.Context, desired *api.Cluster, registry *api.Registry) error {
+func (a *dockerDesktopAdmin) Create(ctx context.Context, desired *api.Cluster, registry *api.Registry, pullThroughRegistries []*api.Registry) error {
 	if registry != nil {
 		return fmt.Errorf("ctlptl currently does not support connecting a registry to docker-desktop")
 	}
+	if len(pullThroughRegistries) > 0 {
+		return fmt.Errorf("ctlptl currently does not support connecting pull-through registries to docker-desktop")
+	}
 
 	isLocalDockerDesktop := docker.IsLocalDockerDesktop(a.host, a.os)
 	if !isLocalDockerDesktop {

pkg/cluster/admin_k3d.go

@@ -48,11 +48,14 @@ func (a *k3dAdmin) EnsureInstalled(ctx context.Context) error {
 	return nil
 }
 
-func (a *k3dAdmin) Create(ctx context.Context, desired *api.Cluster, registry *api.Registry) error {
+func (a *k3dAdmin) Create(ctx context.Context, desired *api.Cluster, registry *api.Registry, pullThroughRegistries []*api.Registry) error {
 	klog.V(3).Infof("Creating cluster with config:\n%+v\n---\n", desired)
 	if registry != nil {
 		klog.V(3).Infof("Initializing cluster with registry config:\n%+v\n---\n", registry)
 	}
+	if len(pullThroughRegistries) > 0 {
+		return fmt.Errorf("ctlptl currently does not support connecting pull-through registries to docker-desktop")
+	}
 
 	k3dV, err := a.version(ctx)
 	if err != nil {

pkg/cluster/admin_kind.go

@@ -56,7 +56,7 @@ func (a *kindAdmin) EnsureInstalled(ctx context.Context) error {
 	return nil
 }
 
-func (a *kindAdmin) kindClusterConfig(desired *api.Cluster, registry *api.Registry, registryAPI containerdRegistryAPI) *v1alpha4.Cluster {
+func (a *kindAdmin) kindClusterConfig(ctx context.Context, desired *api.Cluster, registry *api.Registry, pullThroughRegistries []*api.Registry, registryAPI containerdRegistryAPI) (*v1alpha4.Cluster, error) {
 	kindConfig := desired.KindV1Alpha4Cluster
 	if kindConfig == nil {
 		kindConfig = &v1alpha4.Cluster{}
@@ -84,10 +84,20 @@ func (a *kindAdmin) kindClusterConfig(desired *api.Cluster, registry *api.Regist
 			kindConfig.ContainerdConfigPatches = append(kindConfig.ContainerdConfigPatches, patch)
 		}
 	}
-	return kindConfig
+
+	for _, reg := range pullThroughRegistries {
+		if reg.Proxy != nil && reg.Proxy.RemoteURL != "" {
+			patch := fmt.Sprintf(`[plugins."io.containerd.grpc.v1.cri".registry.mirrors."%s"]
+  endpoint = ["http://%s:%d"]
+`, reg.Proxy.RemoteURL, reg.Name, reg.Status.ContainerPort)
+			kindConfig.ContainerdConfigPatches = append(kindConfig.ContainerdConfigPatches, patch)
+		}
+	}
+
+	return kindConfig, nil
 }
 
-func (a *kindAdmin) Create(ctx context.Context, desired *api.Cluster, registry *api.Registry) error {
+func (a *kindAdmin) Create(ctx context.Context, desired *api.Cluster, registry *api.Registry, pullThroughRegistries []*api.Registry) error {
 	klog.V(3).Infof("Creating cluster with config:\n%+v\n---\n", desired)
 	if registry != nil {
 		klog.V(3).Infof("Initializing cluster with registry config:\n%+v\n---\n", registry)
@@ -139,7 +149,10 @@ func (a *kindAdmin) Create(ctx context.Context, desired *api.Cluster, registry *
 		args = append(args, "--image", node)
 	}
 
-	kindConfig := a.kindClusterConfig(desired, registry, registryAPI)
+	kindConfig, err := a.kindClusterConfig(ctx, desired, registry, pullThroughRegistries, registryAPI)
+	if err != nil {
+		return errors.Wrap(err, "creating kind cluster")
+	}
 	buf := bytes.NewBuffer(nil)
 	encoder := yaml.NewEncoder(buf)
 	err = encoder.Encode(kindConfig)

pkg/cluster/admin_minikube.go

@@ -78,11 +78,14 @@ func (a *minikubeAdmin) version(ctx context.Context) (semver.Version, error) {
 	return result, nil
 }
 
-func (a *minikubeAdmin) Create(ctx context.Context, desired *api.Cluster, registry *api.Registry) error {
+func (a *minikubeAdmin) Create(ctx context.Context, desired *api.Cluster, registry *api.Registry, pullThroughRegistries []*api.Registry) error {
 	klog.V(3).Infof("Creating cluster with config:\n%+v\n---\n", desired)
 	if registry != nil {
 		klog.V(3).Infof("Initializing cluster with registry config:\n%+v\n---\n", registry)
 	}
+	if len(pullThroughRegistries) > 0 {
+		return fmt.Errorf("ctlptl currently does not support connecting pull-through registries to docker-desktop")
+	}
 
 	v, err := a.version(ctx)
 	if err != nil {

pkg/cluster/cluster.go

@@ -668,6 +668,30 @@ func (c *Controller) ensureRegistryExistsForCluster(ctx context.Context, desired
 	})
 }
 
+// Get the pull-through registries for the given cluster.
+func (c *Controller) pullThroughRegistries(ctx context.Context, desired *api.Cluster) ([]*api.Registry, error) {
+	regCtl, err := c.registryController(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	// Filter for all the registries that are in the desired.PullThroughRegistries list.
+	pullThroughRegistries := make([]*api.Registry, 0)
+	allRegistries, err := regCtl.List(ctx, registry.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+	for _, reg := range allRegistries.Items {
+		for _, desiredRegName := range desired.PullThroughRegistries {
+			if reg.Name == desiredRegName {
+				pullThroughRegistries = append(pullThroughRegistries, &reg)
+			}
+		}
+	}
+
+	return pullThroughRegistries, nil
+}
+
 // Compare the desired cluster against the existing cluster, and reconcile
 // the two to match.
 func (c *Controller) Apply(ctx context.Context, desired *api.Cluster) (*api.Cluster, error) {
@@ -748,17 +772,24 @@ func (c *Controller) Apply(ctx context.Context, desired *api.Cluster) (*api.Clus
 		}
 	}
 
+	// Ensure the registry exists.
 	reg, err := c.ensureRegistryExistsForCluster(ctx, desired)
 	if err != nil {
 		return nil, err
 	}
 
+	// Fetch the pull-through registries.
+	pullThroughRegistries, err := c.pullThroughRegistries(ctx, desired)
+	if err != nil {
+		return nil, err
+	}
+
 	// Configure the cluster to match what we want.
 	needsCreate := existingStatus.CreationTimestamp.Time.IsZero() ||
 		desired.Name != existingCluster.Name ||
 		desired.Product != existingCluster.Product
 	if needsCreate {
-		err := admin.Create(ctx, desired, reg)
+		err := admin.Create(ctx, desired, reg, pullThroughRegistries)
 		if err != nil {
 			return nil, err
 		}

pkg/cmd/create_registry.go

@@ -38,7 +38,8 @@ func (o *CreateRegistryOptions) Command() *cobra.Command {
 		Short: "Create a registry with the given name",
 		Example: "  ctlptl create registry ctlptl-registry\n" +
 			"  ctlptl create registry ctlptl-registry --port=5000\n" +
-			"  ctlptl create registry ctlptl-registry --port=5000 --listen-address 0.0.0.0",
+			"  ctlptl create registry ctlptl-registry --port=5000 --listen-address 0.0.0.0\n" +
+			"  ctlptl create registry ctlptl-pull-through-registry --proxy-remote-url=https://registry-1.docker.io",
 		Run:  o.Run,
 		Args: cobra.ExactArgs(1),
 	}
@@ -53,6 +54,28 @@ func (o *CreateRegistryOptions) Command() *cobra.Command {
 	cmd.Flags().StringVar(&o.Registry.Image, "image", registry.DefaultRegistryImageRef,
 		"Registry image to use")
 
+	// Initialize Proxy only if any proxy-related flag is set
+	var proxyRemoteURL, proxyUsername, proxyPassword, proxyTTL string
+	cmd.Flags().StringVar(&proxyRemoteURL, "proxy-remote-url", "",
+		"The remote URL for the pull-through proxy")
+	cmd.Flags().StringVar(&proxyUsername, "proxy-username", "",
+		"The username for the pull-through proxy authentication")
+	cmd.Flags().StringVar(&proxyPassword, "proxy-password", "",
+		"The password for the pull-through proxy authentication")
+	cmd.Flags().StringVar(&proxyTTL, "proxy-ttl", "",
+		"The TTL for the pull-through proxy cache")
+
+	cmd.PreRun = func(cmd *cobra.Command, args []string) {
+		if proxyRemoteURL != "" {
+			o.Registry.Proxy = &api.RegistryProxySpec{
+				RemoteURL: proxyRemoteURL,
+				Username:  proxyUsername,
+				Password:  proxyPassword,
+				TTL:       proxyTTL,
+			}
+		}
+	}
+
 	return cmd
 }
 

pkg/registry/registry.go

@@ -3,6 +3,7 @@ package registry
 import (
 	"context"
 	"fmt"
+	"os"
 	"reflect"
 	"regexp"
 	"sort"
@@ -155,10 +156,37 @@ func (c *Controller) List(ctx context.Context, options ListOptions) (*api.Regist
 			warnings = append(warnings, fmt.Sprintf("Unexpected registry ports: %+v", container.Ports))
 		}
 
+		// Extract proxy settings from environment variables
+		var proxy *api.RegistryProxySpec
+		for _, envVar := range env {
+			if strings.HasPrefix(envVar, "REGISTRY_PROXY_REMOTEURL=") {
+				if proxy == nil {
+					proxy = &api.RegistryProxySpec{}
+				}
+				proxy.RemoteURL = strings.TrimPrefix(envVar, "REGISTRY_PROXY_REMOTEURL=")
+			} else if strings.HasPrefix(envVar, "REGISTRY_PROXY_USERNAME=") {
+				if proxy == nil {
+					proxy = &api.RegistryProxySpec{}
+				}
+				proxy.Username = strings.TrimPrefix(envVar, "REGISTRY_PROXY_USERNAME=")
+			} else if strings.HasPrefix(envVar, "REGISTRY_PROXY_PASSWORD=") {
+				if proxy == nil {
+					proxy = &api.RegistryProxySpec{}
+				}
+				proxy.Password = strings.TrimPrefix(envVar, "REGISTRY_PROXY_PASSWORD=")
+			} else if strings.HasPrefix(envVar, "REGISTRY_PROXY_TTL=") {
+				if proxy == nil {
+					proxy = &api.RegistryProxySpec{}
+				}
+				proxy.TTL = strings.TrimPrefix(envVar, "REGISTRY_PROXY_TTL=")
+			}
+		}
+
 		registry := &api.Registry{
 			TypeMeta: typeMeta,
 			Name:     name,
 			Port:     hostPort,
+			Proxy:    proxy,
 			Status: api.RegistryStatus{
 				CreationTimestamp: metav1.Time{Time: created},
 				ContainerID:       container.ID,
@@ -259,6 +287,29 @@ func (c *Controller) Apply(ctx context.Context, desired *api.Registry) (*api.Reg
 		desiredEnvs["REGISTRY_STORAGE_DELETE_ENABLED"] = "true"
 		desired.Env = append(desired.Env, "REGISTRY_STORAGE_DELETE_ENABLED=true")
 	}
+
+	// Set proxy environment variables if Proxy is configured
+	if desired.Proxy != nil {
+		if desired.Proxy.RemoteURL != "" {
+			desired.Env = append(desired.Env, fmt.Sprintf("REGISTRY_PROXY_REMOTEURL=%s", desired.Proxy.RemoteURL))
+			desiredEnvs["REGISTRY_PROXY_REMOTEURL"] = desired.Proxy.RemoteURL
+		}
+		if desired.Proxy.Username != "" {
+			desired.Env = append(desired.Env, fmt.Sprintf("REGISTRY_PROXY_USERNAME=%s", desired.Proxy.Username))
+			desiredEnvs["REGISTRY_PROXY_USERNAME"] = desired.Proxy.Username
+		}
+		if desired.Proxy.Password != "" {
+			// Use the new method for environment variable replacement
+			password := expandEnvVariables(desired.Proxy.Password)
+			desired.Env = append(desired.Env, fmt.Sprintf("REGISTRY_PROXY_PASSWORD=%s", password))
+			desiredEnvs["REGISTRY_PROXY_PASSWORD"] = password
+		}
+		if desired.Proxy.TTL != "" {
+			desired.Env = append(desired.Env, fmt.Sprintf("REGISTRY_PROXY_TTL=%s", desired.Proxy.TTL))
+			desiredEnvs["REGISTRY_PROXY_TTL"] = desired.Proxy.TTL
+		}
+	}
+
 	if eq := reflect.DeepEqual(desiredEnvs, existingEnvs); !eq {
 		needsDelete = true
 	}
@@ -480,3 +531,11 @@ func imagesRefsEqual(a, b string) bool {
 
 	return aRef.String() == bRef.String()
 }
+
+// expandEnvVariables replaces both ${VAR_NAME} and $VAR_NAME with their environment variable values.
+func expandEnvVariables(input string) string {
+	// Use os.Expand to handle both ${VAR_NAME} and $VAR_NAME
+	return os.Expand(input, func(varName string) string {
+		return os.Getenv(varName)
+	})
+}