Skip to content

Commit

Permalink
Add logic to check prefix of image url and set an imagePullSecret (#1083
Browse files Browse the repository at this point in the history
)

add instructions to install guide

ensure pull secret is reconciled when changed

Signed-off-by: craig <[email protected]>

rh-pre-commit.version: 2.2.0
rh-pre-commit.check-secrets: ENABLED
  • Loading branch information
maleck13 authored Dec 19, 2024
1 parent 6195492 commit d7d1e9f
Show file tree
Hide file tree
Showing 6 changed files with 294 additions and 12 deletions.
5 changes: 5 additions & 0 deletions controllers/data_plane_policies_workflow.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,11 @@ const (

var (
WASMFilterImageURL = env.GetString("RELATED_IMAGE_WASMSHIM", "oci://quay.io/kuadrant/wasm-shim:latest")
// protectedRegistry this defines a default protected registry. If this is in the wasm image URL we add a pull secret name to the WASMPLugin resource
ProtectedRegistry = env.GetString("PROTECTED_REGISTRY", "registry.redhat.io")

// registryPullSecretName this is the pull secret name we will add to the WASMPlugin if the URL for he image is from the defined PROTECTED_REGISTRY
RegistryPullSecretName = "wasm-plugin-pull-secret"

StateIstioExtensionsModified = "IstioExtensionsModified"
StateEnvoyGatewayExtensionsModified = "EnvoyGatewayExtensionsModified"
Expand Down
24 changes: 18 additions & 6 deletions controllers/envoy_gateway_extension_reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"strings"
"sync"

envoygatewayv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1"
Expand All @@ -15,7 +16,9 @@ import (
k8stypes "k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/dynamic"
"k8s.io/utils/ptr"
v1 "sigs.k8s.io/gateway-api/apis/v1"
gatewayapiv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1"

kuadrantv1 "github.com/kuadrant/kuadrant-operator/api/v1"
kuadrantv1beta1 "github.com/kuadrant/kuadrant-operator/api/v1beta1"
Expand Down Expand Up @@ -52,7 +55,7 @@ func (r *EnvoyGatewayExtensionReconciler) Subscription() controller.Subscription
func (r *EnvoyGatewayExtensionReconciler) Reconcile(ctx context.Context, _ []controller.ResourceEvent, topology *machinery.Topology, _ error, state *sync.Map) error {
logger := controller.LoggerFromContext(ctx).WithName("EnvoyGatewayExtensionReconciler")

logger.V(1).Info("building envoy gateway extension")
logger.V(1).Info("building envoy gateway extension", "image url", WASMFilterImageURL)
defer logger.V(1).Info("finished building envoy gateway extension")

// build wasm plugin configs for each gateway
Expand All @@ -76,8 +79,7 @@ func (r *EnvoyGatewayExtensionReconciler) Reconcile(ctx context.Context, _ []con

for _, gateway := range gateways {
gatewayKey := k8stypes.NamespacedName{Name: gateway.GetName(), Namespace: gateway.GetNamespace()}

desiredEnvoyExtensionPolicy := buildEnvoyExtensionPolicyForGateway(gateway, wasmConfigs[gateway.GetLocator()])
desiredEnvoyExtensionPolicy := buildEnvoyExtensionPolicyForGateway(gateway, wasmConfigs[gateway.GetLocator()], ProtectedRegistry, WASMFilterImageURL)

resource := r.client.Resource(kuadrantenvoygateway.EnvoyExtensionPoliciesResource).Namespace(desiredEnvoyExtensionPolicy.GetNamespace())

Expand Down Expand Up @@ -216,7 +218,7 @@ func (r *EnvoyGatewayExtensionReconciler) buildWasmConfigs(ctx context.Context,
}

// buildEnvoyExtensionPolicyForGateway builds a desired EnvoyExtensionPolicy custom resource for a given gateway and corresponding wasm config
func buildEnvoyExtensionPolicyForGateway(gateway *machinery.Gateway, wasmConfig wasm.Config) *envoygatewayv1alpha1.EnvoyExtensionPolicy {
func buildEnvoyExtensionPolicyForGateway(gateway *machinery.Gateway, wasmConfig wasm.Config, protectedRegistry, imageURL string) *envoygatewayv1alpha1.EnvoyExtensionPolicy {
envoyPolicy := &envoygatewayv1alpha1.EnvoyExtensionPolicy{
TypeMeta: metav1.TypeMeta{
Kind: kuadrantenvoygateway.EnvoyExtensionPolicyGroupKind.Kind,
Expand Down Expand Up @@ -256,7 +258,7 @@ func buildEnvoyExtensionPolicyForGateway(gateway *machinery.Gateway, wasmConfig
Code: envoygatewayv1alpha1.WasmCodeSource{
Type: envoygatewayv1alpha1.ImageWasmCodeSourceType,
Image: &envoygatewayv1alpha1.ImageWasmCodeSource{
URL: WASMFilterImageURL,
URL: imageURL,
},
},
Config: nil,
Expand All @@ -268,6 +270,16 @@ func buildEnvoyExtensionPolicyForGateway(gateway *machinery.Gateway, wasmConfig
},
},
}
for _, wasm := range envoyPolicy.Spec.Wasm {
if wasm.Code.Image.PullSecretRef != nil {
//reset it to empty this will remove it if the image is now public registry
wasm.Code.Image.PullSecretRef = nil
}
// if we are in a protected registry set the object
if protectedRegistry != "" && strings.Contains(imageURL, protectedRegistry) {
wasm.Code.Image.PullSecretRef = &gwapiv1b1.SecretObjectReference{Name: v1.ObjectName(RegistryPullSecretName)}
}
}

if len(wasmConfig.ActionSets) == 0 {
utils.TagObjectToDelete(envoyPolicy)
Expand All @@ -292,7 +304,7 @@ func equalEnvoyExtensionPolicies(a, b *envoygatewayv1alpha1.EnvoyExtensionPolicy

return len(aWasms) == len(bWasms) && lo.EveryBy(aWasms, func(aWasm envoygatewayv1alpha1.Wasm) bool {
return lo.SomeBy(bWasms, func(bWasm envoygatewayv1alpha1.Wasm) bool {
if ptr.Deref(aWasm.Name, "") != ptr.Deref(bWasm.Name, "") || ptr.Deref(aWasm.RootID, "") != ptr.Deref(bWasm.RootID, "") || ptr.Deref(aWasm.FailOpen, false) != ptr.Deref(bWasm.FailOpen, false) || aWasm.Code.Type != bWasm.Code.Type || aWasm.Code.Image.URL != bWasm.Code.Image.URL {
if ptr.Deref(aWasm.Name, "") != ptr.Deref(bWasm.Name, "") || ptr.Deref(aWasm.RootID, "") != ptr.Deref(bWasm.RootID, "") || ptr.Deref(aWasm.FailOpen, false) != ptr.Deref(bWasm.FailOpen, false) || aWasm.Code.Type != bWasm.Code.Type || aWasm.Code.Image.URL != bWasm.Code.Image.URL || ptr.Deref(aWasm.Code.Image.PullSecretRef, gwapiv1b1.SecretObjectReference{}) != ptr.Deref(bWasm.Code.Image.PullSecretRef, gwapiv1b1.SecretObjectReference{}) {
return false
}
aConfig, err := wasm.ConfigFromJSON(aWasm.Config)
Expand Down
178 changes: 178 additions & 0 deletions controllers/extenstion_reconciler_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@
//go:build unit

package controllers

import (
"fmt"
"testing"

envoygatewayv1alpha1 "github.com/envoyproxy/gateway/api/v1alpha1"
"github.com/kuadrant/kuadrant-operator/pkg/wasm"
"github.com/kuadrant/policy-machinery/machinery"
istioclientgoextensionv1alpha1 "istio.io/client-go/pkg/apis/extensions/v1alpha1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
v1 "sigs.k8s.io/gateway-api/apis/v1"
)

var (
defaultWasmImage = WASMFilterImageURL
registry = "protected.registry.io"
protectedRegImage = fmt.Sprintf("oci://%s/kuadrant/wasm-shim:latest", registry)
testGateway = &machinery.Gateway{
Gateway: &v1.Gateway{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "test",
},
},
}
testWasmConfig = wasm.Config{
ActionSets: []wasm.ActionSet{
{
Name: "test",
},
},
}
)

func Test_buildIstioWasmPluginForGateway(t *testing.T) {
testCases := []struct {
Name string
WASMImageURLS func() []string
ProtectedRegistryPrefix string
Assert func(t *testing.T, plugin *istioclientgoextensionv1alpha1.WasmPlugin)
}{
{
Name: "ensure image pull secret is set in wasmPlugin for protected registry",
WASMImageURLS: func() []string {
// note currently this is a package global
return []string{protectedRegImage}
},
ProtectedRegistryPrefix: registry,
Assert: func(t *testing.T, plugin *istioclientgoextensionv1alpha1.WasmPlugin) {
if plugin == nil {
t.Fatalf("Expected a wasmplugin")
}
if plugin.Spec.ImagePullSecret != RegistryPullSecretName {
t.Fatalf("Expected wasm plugin to have imagePullSecret %s but got %s", RegistryPullSecretName, plugin.Spec.ImagePullSecret)
}
},
},
{
Name: "ensure image pull secret is NOT set in wasmPlugin for unprotected registry",
WASMImageURLS: func() []string {
return []string{WASMFilterImageURL}
},
Assert: func(t *testing.T, plugin *istioclientgoextensionv1alpha1.WasmPlugin) {
if plugin == nil {
t.Fatalf("Expected a wasmplugin")
}
if plugin.Spec.ImagePullSecret != "" {
t.Fatalf("Expected wasm plugin to NOT have imagePullSecret %v", plugin.Spec.ImagePullSecret)
}
},
},
{
Name: "ensure image pull secret is set in wasmPlugin for protected registry and unset for unprotected registry",
WASMImageURLS: func() []string {
return []string{ProtectedRegistry, WASMFilterImageURL}
},
Assert: func(t *testing.T, plugin *istioclientgoextensionv1alpha1.WasmPlugin) {
if plugin == nil {
t.Fatalf("Expected a wasmplugin")
}
if plugin.Spec.Url == protectedRegImage && plugin.Spec.ImagePullSecret == "" {
t.Fatalf("Expected wasm plugin to have imagePullSecret set but got none")
}
if plugin.Spec.Url == WASMFilterImageURL && plugin.Spec.ImagePullSecret != "" {
t.Fatalf("Expected wasm plugin to not have imagePullSecret set but got %v", plugin.Spec.ImagePullSecret)
}
},
},
}

for _, testCase := range testCases {
t.Run(testCase.Name, func(t *testing.T) {
images := testCase.WASMImageURLS()
for _, image := range images {
plugin := buildIstioWasmPluginForGateway(testGateway, testWasmConfig, testCase.ProtectedRegistryPrefix, image)
testCase.Assert(t, plugin)
}
})
}

}

func Test_buildEnvoyExtensionPolicyForGateway(t *testing.T) {
testCases := []struct {
Name string
WASMImageURLS func() []string
ProtectedRegistryPrefix string
Assert func(t *testing.T, policy *envoygatewayv1alpha1.EnvoyExtensionPolicy)
}{
{
Name: "ensure image pull secret is set in ExtensionPolicy for protected registry",
WASMImageURLS: func() []string {
return []string{protectedRegImage}
},
ProtectedRegistryPrefix: registry,
Assert: func(t *testing.T, policy *envoygatewayv1alpha1.EnvoyExtensionPolicy) {
if policy == nil {
t.Fatalf("Expected a wasmplugin")
}
for _, w := range policy.Spec.Wasm {
if w.Code.Image.PullSecretRef == nil {
t.Fatalf("Expected extension to have imagePullSecret %v but no pullSecretRef", RegistryPullSecretName)
}
if w.Code.Image.PullSecretRef.Name != v1.ObjectName(RegistryPullSecretName) {
t.Fatalf("expected the pull secret name to be %s but got %v", RegistryPullSecretName, w.Code.Image.PullSecretRef.Name)
}
}
},
},
{
Name: "ensure image pull secret is NOT set in wasmPlugin for unprotected registry",
WASMImageURLS: func() []string {
return []string{defaultWasmImage}
},
Assert: func(t *testing.T, policy *envoygatewayv1alpha1.EnvoyExtensionPolicy) {
if policy == nil {
t.Fatalf("Expected a wasmplugin")
}
for _, w := range policy.Spec.Wasm {
if w.Code.Image.PullSecretRef != nil {
t.Fatalf("Expected extension to have not imagePullSecret but got %v", w.Code.Image.PullSecretRef)
}
}
},
},
{
Name: "ensure image pull secret is set in extension for protected registry and unset for unprotected registry",
WASMImageURLS: func() []string {
return []string{ProtectedRegistry, WASMFilterImageURL}
},
Assert: func(t *testing.T, policy *envoygatewayv1alpha1.EnvoyExtensionPolicy) {
if policy == nil {
t.Fatalf("Expected a wasmplugin")
}
for _, w := range policy.Spec.Wasm {
if w.Code.Image.PullSecretRef == nil && w.Code.Image.URL == protectedRegImage {
t.Fatalf("Expected policy to have imagePullSecret set but got none")
}
if w.Code.Image.PullSecretRef != nil && w.Code.Image.URL == WASMFilterImageURL {
t.Fatalf("Expected policy to not have imagePullSecret set but got %v", w.Code.Image.PullSecretRef)
}
}

},
},
}

for _, testCase := range testCases {
images := testCase.WASMImageURLS()
for _, image := range images {
policy := buildEnvoyExtensionPolicyForGateway(testGateway, testWasmConfig, testCase.ProtectedRegistryPrefix, image)
testCase.Assert(t, policy)
}
}
}
20 changes: 14 additions & 6 deletions controllers/istio_extension_reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"strings"
"sync"

"github.com/kuadrant/policy-machinery/controller"
Expand Down Expand Up @@ -53,7 +54,7 @@ func (r *IstioExtensionReconciler) Subscription() controller.Subscription {
func (r *IstioExtensionReconciler) Reconcile(ctx context.Context, _ []controller.ResourceEvent, topology *machinery.Topology, _ error, state *sync.Map) error {
logger := controller.LoggerFromContext(ctx).WithName("IstioExtensionReconciler")

logger.V(1).Info("building istio extension")
logger.V(1).Info("building istio extension ", "image url", WASMFilterImageURL)
defer logger.V(1).Info("finished building istio extension")

// build wasm plugin configs for each gateway
Expand All @@ -78,7 +79,7 @@ func (r *IstioExtensionReconciler) Reconcile(ctx context.Context, _ []controller
for _, gateway := range gateways {
gatewayKey := k8stypes.NamespacedName{Name: gateway.GetName(), Namespace: gateway.GetNamespace()}

desiredWasmPlugin := buildIstioWasmPluginForGateway(gateway, wasmConfigs[gateway.GetLocator()])
desiredWasmPlugin := buildIstioWasmPluginForGateway(gateway, wasmConfigs[gateway.GetLocator()], ProtectedRegistry, WASMFilterImageURL)

resource := r.client.Resource(kuadrantistio.WasmPluginsResource).Namespace(desiredWasmPlugin.GetNamespace())

Expand Down Expand Up @@ -114,7 +115,7 @@ func (r *IstioExtensionReconciler) Reconcile(ctx context.Context, _ []controller
}
continue
}

logger.V(1).Info("wasmplugin object ", "desired", desiredWasmPlugin)
if equalWasmPlugins(existingWasmPlugin, desiredWasmPlugin) {
logger.V(1).Info("wasmplugin object is up to date, nothing to do")
continue
Expand All @@ -125,6 +126,7 @@ func (r *IstioExtensionReconciler) Reconcile(ctx context.Context, _ []controller
existingWasmPlugin.Spec.Phase = desiredWasmPlugin.Spec.Phase
existingWasmPlugin.Spec.TargetRefs = desiredWasmPlugin.Spec.TargetRefs
existingWasmPlugin.Spec.PluginConfig = desiredWasmPlugin.Spec.PluginConfig
existingWasmPlugin.Spec.ImagePullSecret = desiredWasmPlugin.Spec.ImagePullSecret

existingWasmPluginUnstructured, err := controller.Destruct(existingWasmPlugin)
if err != nil {
Expand Down Expand Up @@ -228,7 +230,7 @@ func hasAuthAccess(actionSet []wasm.Action) bool {
}

// buildIstioWasmPluginForGateway builds a desired WasmPlugin custom resource for a given gateway and corresponding wasm config
func buildIstioWasmPluginForGateway(gateway *machinery.Gateway, wasmConfig wasm.Config) *istioclientgoextensionv1alpha1.WasmPlugin {
func buildIstioWasmPluginForGateway(gateway *machinery.Gateway, wasmConfig wasm.Config, protectedRegistry, imageURL string) *istioclientgoextensionv1alpha1.WasmPlugin {
wasmPlugin := &istioclientgoextensionv1alpha1.WasmPlugin{
TypeMeta: metav1.TypeMeta{
Kind: kuadrantistio.WasmPluginGroupKind.Kind,
Expand Down Expand Up @@ -257,11 +259,17 @@ func buildIstioWasmPluginForGateway(gateway *machinery.Gateway, wasmConfig wasm.
Name: gateway.GetName(),
},
},
Url: WASMFilterImageURL,
Url: imageURL,
PluginConfig: nil,
Phase: istioextensionsv1alpha1.PluginPhase_STATS, // insert the plugin before Istio stats filters and after Istio authorization filters.
},
}
// reset to empty to allow fo the image having moved to a public registry
wasmPlugin.Spec.ImagePullSecret = ""
// only set to pull secret if we are in a protected registry
if protectedRegistry != "" && strings.Contains(imageURL, protectedRegistry) {
wasmPlugin.Spec.ImagePullSecret = RegistryPullSecretName
}

if len(wasmConfig.ActionSets) == 0 {
utils.TagObjectToDelete(wasmPlugin)
Expand All @@ -277,7 +285,7 @@ func buildIstioWasmPluginForGateway(gateway *machinery.Gateway, wasmConfig wasm.
}

func equalWasmPlugins(a, b *istioclientgoextensionv1alpha1.WasmPlugin) bool {
if a.Spec.Url != b.Spec.Url || a.Spec.Phase != b.Spec.Phase || !kuadrantistio.EqualTargetRefs(a.Spec.TargetRefs, b.Spec.TargetRefs) {
if a.Spec.ImagePullSecret != b.Spec.ImagePullSecret || a.Spec.Url != b.Spec.Url || a.Spec.Phase != b.Spec.Phase || !kuadrantistio.EqualTargetRefs(a.Spec.TargetRefs, b.Spec.TargetRefs) {
return false
}

Expand Down
13 changes: 13 additions & 0 deletions doc/install/install-openshift.md
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,19 @@ spec:
upgradeStrategy: Default
EOF
```
**Authenticated Registry**

!!! note

If you need to use a wasm image from a protected registry (such as the redhat registry), you will need to configure an image pull secret to use. This secret must exist in each namespace where there is a Gateway resource defined that you intend to target with `AuthPolicy` or `RatelimitPolicy` and it must be named `wasm-plugin-pull-secret`.

Example Creating Pull Secret:

```bash
kubectl create secret docker-registry wasm-plugin-pull-secret -n ${GATEWAY_NAMESPACE} \ --docker-server=my.registry.io \ --docker-username=your-registry-service-account-username \ --docker-password=your-registry-service-account-password
```

The configuration for the gateway will expect this secret to exist if the registry name begins with `registry.redhat.com` by default. This can be changed via the env var `PROTECTED_REGISTRY` set in the kuadrant-operator.

Wait for the Kuadrant Operators to be installed as follows:

Expand Down
Loading

0 comments on commit d7d1e9f

Please sign in to comment.