From 26acb4a40f668a9c965a2c3a44f171c415d967ec Mon Sep 17 00:00:00 2001 From: justinsb Date: Tue, 15 Oct 2024 18:30:04 -0400 Subject: [PATCH] operator: support remapping of images If the IMAGE_PREFIX env var or the --image-prefix flag is provided to the operator, remap images to pull from that registry/mirror. For example, with IMAGE_PREFIX=gcr.io/foo, an image like gcr.io/example/cnrm/manager:latest will be rewritten to gcr.io/foo/manager:latest. --- operator/cmd/manager/main.go | 21 +++- .../configconnector_controller.go | 24 +++-- .../controllers/configconnector/e2e_test.go | 6 +- .../configconnectorcontext_controller.go | 24 +++-- .../configconnectorcontext/e2e_test.go | 6 +- operator/pkg/controllers/imagetransform.go | 102 ++++++++++++++++++ .../pkg/controllers/imagetransform_test.go | 61 +++++++++++ 7 files changed, 226 insertions(+), 18 deletions(-) create mode 100644 operator/pkg/controllers/imagetransform.go create mode 100644 operator/pkg/controllers/imagetransform_test.go diff --git a/operator/cmd/manager/main.go b/operator/cmd/manager/main.go index 9953b54bfb..6b11dfd16c 100644 --- a/operator/cmd/manager/main.go +++ b/operator/cmd/manager/main.go @@ -53,6 +53,10 @@ func main() { "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&enablePprof, "enable-pprof", false, "Enable the pprof server.") flag.IntVar(&pprofPort, "pprof-port", 6060, "The port that the pprof server binds to if enabled.") + + imagePrefix := os.Getenv("IMAGE_PREFIX") + flag.StringVar(&imagePrefix, "image-prefix", imagePrefix, "Remap container images to pull from the specified registry or mirror.") + flag.Parse() ctrl.SetLogger(logging.BuildLogger(os.Stderr)) @@ -92,12 +96,25 @@ func main() { os.Exit(1) } - if err := configconnector.Add(mgr, repoPath); err != nil { + var imageTransform *controllers.ImageTransform + if imagePrefix != "" { + imageTransform = controllers.NewImageTransform(imagePrefix) + } + + ccOptions := &configconnector.ReconcilerOptions{ + RepoPath: repoPath, + ImageTransform: imageTransform, + } + if err := configconnector.Add(mgr, ccOptions); err != nil { setupLog.Error(err, "unable to create controller", "controller", "ConfigConnector") os.Exit(1) } - if err = configconnectorcontext.Add(mgr, repoPath); err != nil { + cccOptions := &configconnectorcontext.ReconcilerOptions{ + RepoPath: repoPath, + ImageTransform: imageTransform, + } + if err = configconnectorcontext.Add(mgr, cccOptions); err != nil { setupLog.Error(err, "unable to create controller", "controller", "ConfigConnectorContext") os.Exit(1) } diff --git a/operator/pkg/controllers/configconnector/configconnector_controller.go b/operator/pkg/controllers/configconnector/configconnector_controller.go index c2a9b46952..ec3bc81057 100644 --- a/operator/pkg/controllers/configconnector/configconnector_controller.go +++ b/operator/pkg/controllers/configconnector/configconnector_controller.go @@ -57,6 +57,12 @@ import ( const controllerName = "configconnector-controller" +// ReconcilerOptions holds configuration options for the reconciler +type ReconcilerOptions struct { + RepoPath string + ImageTransform *controllers.ImageTransform +} + // Reconciler reconciles a ConfigConnector object. // Reconciler watches 'ConfigConnector' kind and is responsible for managing the lifecycle of KCC resource CRDs and other shared components like webhook, deletion defender, recorder. @@ -73,8 +79,8 @@ type Reconciler struct { customizationWatcher *controllers.CustomizationWatcher } -func Add(mgr ctrl.Manager, repoPath string) error { - r, err := newReconciler(mgr, repoPath) +func Add(mgr ctrl.Manager, opt *ReconcilerOptions) error { + r, err := newReconciler(mgr, opt) if err != nil { return err } @@ -95,8 +101,8 @@ func Add(mgr ctrl.Manager, repoPath string) error { return nil } -func newReconciler(mgr ctrl.Manager, repoPath string) (*Reconciler, error) { - repo := cnrmmanifest.NewLocalRepository(repoPath) +func newReconciler(mgr ctrl.Manager, opt *ReconcilerOptions) (*Reconciler, error) { + repo := cnrmmanifest.NewLocalRepository(opt.RepoPath) manifestLoader := cnrmmanifest.NewLoader(repo) preflight := preflight.NewCompositePreflight([]declarative.Preflight{ preflight.NewNameChecker(mgr.GetClient(), k8s.ConfigConnectorAllowedName), @@ -118,7 +124,7 @@ func newReconciler(mgr ctrl.Manager, repoPath string) (*Reconciler, error) { Log: r.log, }) - err := r.reconciler.Init(mgr, &corev1beta1.ConfigConnector{}, + options := []declarative.ReconcilerOption{ declarative.WithLabels(r.labelMaker), declarative.WithPreserveNamespace(), declarative.WithManifestController(manifestLoader), @@ -130,7 +136,13 @@ func newReconciler(mgr ctrl.Manager, repoPath string) (*Reconciler, error) { declarative.WithStatus(&declarative.StatusBuilder{ PreflightImpl: preflight, }), - ) + } + + if opt.ImageTransform != nil { + options = append(options, declarative.WithObjectTransform(opt.ImageTransform.RemapImages)) + } + + err := r.reconciler.Init(mgr, &corev1beta1.ConfigConnector{}, options...) return r, err } diff --git a/operator/pkg/controllers/configconnector/e2e_test.go b/operator/pkg/controllers/configconnector/e2e_test.go index 1671c0c179..7712a0fc07 100644 --- a/operator/pkg/controllers/configconnector/e2e_test.go +++ b/operator/pkg/controllers/configconnector/e2e_test.go @@ -31,8 +31,10 @@ func TestConfigConnectorE2E(t *testing.T) { mgr, stop := testmain.StartTestManagerFromNewTestEnv() defer stop() - repoPath := "../../../channels" - if err := Add(mgr, repoPath); err != nil { + opt := &ReconcilerOptions{ + RepoPath: "../../../channels", + } + if err := Add(mgr, opt); err != nil { t.Fatalf("error from Add: %v", err) } diff --git a/operator/pkg/controllers/configconnectorcontext/configconnectorcontext_controller.go b/operator/pkg/controllers/configconnectorcontext/configconnectorcontext_controller.go index d8f5c216f6..644bb3cc5a 100644 --- a/operator/pkg/controllers/configconnectorcontext/configconnectorcontext_controller.go +++ b/operator/pkg/controllers/configconnectorcontext/configconnectorcontext_controller.go @@ -54,6 +54,12 @@ import ( const controllerName = "configconnectorcontext-controller" +// ReconcilerOptions holds configuration options for the reconciler +type ReconcilerOptions struct { + RepoPath string + ImageTransform *controllers.ImageTransform +} + // Reconciler reconciles a ConfigConnectorContext object. // // From the high level, the Reconciler watches `ConfigConnectorContext` kind @@ -71,8 +77,8 @@ type Reconciler struct { jitterGen jitter.Generator } -func Add(mgr ctrl.Manager, repoPath string) error { - r, err := newReconciler(mgr, repoPath) +func Add(mgr ctrl.Manager, opt *ReconcilerOptions) error { + r, err := newReconciler(mgr, opt) if err != nil { return err } @@ -93,8 +99,8 @@ func Add(mgr ctrl.Manager, repoPath string) error { return nil } -func newReconciler(mgr ctrl.Manager, repoPath string) (*Reconciler, error) { - repo := cnrmmanifest.NewLocalRepository(repoPath) +func newReconciler(mgr ctrl.Manager, opt *ReconcilerOptions) (*Reconciler, error) { + repo := cnrmmanifest.NewLocalRepository(opt.RepoPath) manifestLoader := cnrmmanifest.NewPerNamespaceManifestLoader(repo) preflight := preflight.NewCompositePreflight([]declarative.Preflight{ preflight.NewNameChecker(mgr.GetClient(), k8s.ConfigConnectorContextAllowedName), @@ -118,7 +124,7 @@ func newReconciler(mgr ctrl.Manager, repoPath string) (*Reconciler, error) { Log: r.log, }) - err := r.reconciler.Init(mgr, &corev1beta1.ConfigConnectorContext{}, + options := []declarative.ReconcilerOption{ declarative.WithPreserveNamespace(), declarative.WithManifestController(manifestLoader), declarative.WithObjectTransform(r.transformNamespacedComponents()), @@ -128,7 +134,13 @@ func newReconciler(mgr ctrl.Manager, repoPath string) (*Reconciler, error) { declarative.WithStatus(&declarative.StatusBuilder{ PreflightImpl: preflight, }), - ) + } + + if opt.ImageTransform != nil { + options = append(options, declarative.WithObjectTransform(opt.ImageTransform.RemapImages)) + } + + err := r.reconciler.Init(mgr, &corev1beta1.ConfigConnectorContext{}, options...) return r, err } diff --git a/operator/pkg/controllers/configconnectorcontext/e2e_test.go b/operator/pkg/controllers/configconnectorcontext/e2e_test.go index 5bbda9c7bc..3db74138d3 100644 --- a/operator/pkg/controllers/configconnectorcontext/e2e_test.go +++ b/operator/pkg/controllers/configconnectorcontext/e2e_test.go @@ -33,8 +33,10 @@ func TestConfigConnectorContextE2E(t *testing.T) { mgr, stop := testmain.StartTestManagerFromNewTestEnv() defer stop() - repoPath := "../../../channels" - if err := Add(mgr, repoPath); err != nil { + opt := &ReconcilerOptions{ + RepoPath: "../../../channels", + } + if err := Add(mgr, opt); err != nil { t.Fatalf("error from Add: %v", err) } diff --git a/operator/pkg/controllers/imagetransform.go b/operator/pkg/controllers/imagetransform.go new file mode 100644 index 0000000000..10478432d1 --- /dev/null +++ b/operator/pkg/controllers/imagetransform.go @@ -0,0 +1,102 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllers + +import ( + "context" + "strings" + + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/kubebuilder-declarative-pattern/pkg/patterns/declarative" + "sigs.k8s.io/kubebuilder-declarative-pattern/pkg/patterns/declarative/pkg/manifest" +) + +// ImageTranform remaps container images in a manifest. +type ImageTransform struct { + // ImagePrefix changes the image registry to a different registry, keeping the name. + // We strip off all but the last component of the image name, and then add the prefix. + // The tag is unchanged. + // + // gcr.io/gke-release/cnrm/deletiondefender:1.0 => ${IMAGE_PREFIX}/deletiondefender:1.0 + // deletiondefender:1.0 => ${IMAGE_PREFIX}/deletiondefender:1.0 + ImagePrefix string +} + +// NewImageTransform builds an ImageTransform +func NewImageTransform(imagePrefix string) *ImageTransform { + imagePrefix = strings.TrimSuffix(imagePrefix, "/") + "/" + if imagePrefix == "/" { + // Special case: empty image prefix should remain empty + imagePrefix = "" + } + return &ImageTransform{ImagePrefix: imagePrefix} +} + +// Remap images to the specified mirror / alternative location. +// This function can be used as an object transfomration. +func (x *ImageTransform) RemapImages(ctx context.Context, o declarative.DeclarativeObject, manifest *manifest.Objects) error { + for _, obj := range manifest.Items { + if err := x.remapImages(obj); err != nil { + return err + } + } + return nil +} + +func (x *ImageTransform) remapImages(obj *manifest.Object) error { + // Check that this object has images (Deployment, StatefulSet) + switch obj.GroupKind() { + case schema.GroupKind{Group: "apps", Kind: "Deployment"}: + case schema.GroupKind{Group: "apps", Kind: "StatefulSet"}: + case schema.GroupKind{Group: "apps", Kind: "DaemonSet"}: + + default: + return nil + } + + if err := obj.MutateContainers(func(container map[string]any) error { + image, ok := container["image"].(string) + if !ok { + return nil + } + + newImage := x.remapImage(image) + container["image"] = newImage + return nil + }); err != nil { + return err + } + return nil +} + +func (x *ImageTransform) remapImage(image string) string { + lastColon := strings.LastIndex(image, ":") + name := image + tag := "" + if lastColon != -1 { + name = image[:lastColon] + tag = image[lastColon+1:] + } + + nameTokens := strings.Split(name, "/") + newName := x.ImagePrefix + nameTokens[len(nameTokens)-1] + + newImage := newName + if tag != "" { + newImage += ":" + tag + } + + return newImage +} diff --git a/operator/pkg/controllers/imagetransform_test.go b/operator/pkg/controllers/imagetransform_test.go new file mode 100644 index 0000000000..666d846fef --- /dev/null +++ b/operator/pkg/controllers/imagetransform_test.go @@ -0,0 +1,61 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllers + +import ( + "testing" +) + +func TestImageTransform(t *testing.T) { + grid := []struct { + ImagePrefix string + Image string + Want string + }{ + { + ImagePrefix: "foo/bar", + Image: "gcr.io/gke-release/cnrm/deletiondefender:1.124.0-rc.1", + Want: "foo/bar/deletiondefender:1.124.0-rc.1", + }, + { + ImagePrefix: "foo/bar", + Image: "gcr.io/gke-release/cnrm/deletiondefender:latest", + Want: "foo/bar/deletiondefender:latest", + }, + { + ImagePrefix: "foo/bar", + Image: "gcr.io/gke-release/cnrm/deletiondefender", + Want: "foo/bar/deletiondefender", + }, + { + ImagePrefix: "foo/bar/", + Image: "gcr.io/gke-release/cnrm/deletiondefender", + Want: "foo/bar/deletiondefender", + }, + { + ImagePrefix: "", + Image: "gcr.io/gke-release/cnrm/deletiondefender", + Want: "deletiondefender", + }, + } + + for _, g := range grid { + x := NewImageTransform(g.ImagePrefix) + got := x.remapImage(g.Image) + if got != g.Want { + t.Errorf("unexpected remapping with ImagePrefix=%q from %q; got %q, want %q", g.ImagePrefix, g.Image, got, g.Want) + } + } +}