add feature online plugins filter flag (#1458)

* add feature online plugins filter flag * list refactor fix reflection add test fix lint fix lint add valid entities fux lint Updated go-apiops (#1452) * chore: updated go-apiops to v0.1.40 * chore: release prep for v1.41.4 fux lint fix lint fix lint * fix test * fix err msg * add tests * add tests
Kong · Dec 6, 2024 · 4c52f38 · 4c52f38
1 parent 4da4246
commit 4c52f38
Show file tree

Hide file tree

Showing 5 changed files with 156 additions and 85 deletions.
diff --git a/cmd/gateway_validate.go b/cmd/gateway_validate.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"sort"
 
 	"github.com/blang/semver/v4"
 	"github.com/kong/deck/utils"
@@ -17,12 +18,13 @@ import (
 )
 
 var (
-	validateCmdKongStateFile     []string
-	validateCmdRBACResourcesOnly bool
-	validateOnline               bool
-	validateWorkspace            string
-	validateParallelism          int
-	validateKonnectCompatibility bool
+	validateCmdKongStateFile        []string
+	validateCmdOnlineEntitiesFilter []string
+	validateCmdRBACResourcesOnly    bool
+	validateOnline                  bool
+	validateKonnectCompatibility    bool
+	validateWorkspace               string
+	validateParallelism             int
 )
 
 func executeValidate(cmd *cobra.Command, _ []string) error {
@@ -210,6 +212,26 @@ this command unless --online flag is used.
 			if len(validateCmdKongStateFile) == 0 {
 				validateCmdKongStateFile = []string{"-"}
 			}
+
+			// Iterate over the input values and validate them against the keys in entityMap
+			for _, value := range validateCmdOnlineEntitiesFilter {
+				// Check if the value is valid by comparing it with keys in EntityMap
+				if _, exists := validate.EntityMap[value]; !exists {
+					// Generate an error message with the list of valid keys
+					listOfKeys := make([]string, 0, len(validate.EntityMap))
+					for key := range validate.EntityMap {
+						listOfKeys = append(listOfKeys, key)
+					}
+					// Sort the keys alphabetically
+					sort.Strings(listOfKeys)
+
+					return fmt.Errorf(
+						"invalid value '%s' for --online-entities-list; it should be a valid Kong entity (case-sensitive). "+
+							"Valid entities: %v",
+						value, listOfKeys,
+					)
+				}
+			}
 			return preRunSilenceEventsFlag()
 		}
 
@@ -237,6 +259,8 @@ this command unless --online flag is used.
 
 	validateCmd.Flags().BoolVar(&validateCmdRBACResourcesOnly, "rbac-resources-only",
 		false, "indicate that the state file(s) contains RBAC resources only (Kong Enterprise only).")
+	validateCmd.Flags().StringSliceVarP(&validateCmdOnlineEntitiesFilter, "online-entities-list",
+		"", []string{}, "indicate the list of entities that should be validated online validation.")
 	if deprecated {
 		validateCmd.Flags().StringSliceVarP(&validateCmdKongStateFile,
 			"state", "s", []string{"kong.yaml"}, "file(s) containing Kong's configuration.\n"+
@@ -279,11 +303,12 @@ func validateWithKong(
 		return []error{fmt.Errorf("parsing Kong version: %w", err)}
 	}
 	opts := validate.ValidatorOpts{
-		Ctx:               ctx,
-		State:             ks,
-		Client:            kongClient,
-		Parallelism:       validateParallelism,
-		RBACResourcesOnly: validateCmdRBACResourcesOnly,
+		Ctx:                  ctx,
+		State:                ks,
+		Client:               kongClient,
+		Parallelism:          validateParallelism,
+		RBACResourcesOnly:    validateCmdRBACResourcesOnly,
+		OnlineEntitiesFilter: validateCmdOnlineEntitiesFilter,
 	}
 	validator := validate.NewValidator(opts)
 	return validator.Validate(parsedFormatVersion)

diff --git a/tests/integration/sync_test.go b/tests/integration/sync_test.go
@@ -1397,6 +1397,7 @@ var (
 				ID: kong.String("77e6691d-67c0-446a-9401-27be2b141aae"),
 			},
 			Config: kong.Configuration{
+				"compound_identifier":     nil,
 				"consumer_groups":         nil,
 				"dictionary_name":         string("kong_rate_limiting_counters"),
 				"disable_penalty":         bool(false),
@@ -1447,6 +1448,7 @@ var (
 				ID: kong.String("5bcbd3a7-030b-4310-bd1d-2721ff85d236"),
 			},
 			Config: kong.Configuration{
+				"compound_identifier":     nil,
 				"consumer_groups":         nil,
 				"dictionary_name":         string("kong_rate_limiting_counters"),
 				"disable_penalty":         bool(false),
@@ -1494,6 +1496,7 @@ var (
 		{
 			Name: kong.String("rate-limiting-advanced"),
 			Config: kong.Configuration{
+				"compound_identifier":     nil,
 				"consumer_groups":         nil,
 				"dictionary_name":         string("kong_rate_limiting_counters"),
 				"disable_penalty":         bool(false),

diff --git a/tests/integration/testdata/dump/002-skip-consumers/expected-no-skip_konnect.yaml b/tests/integration/testdata/dump/002-skip-consumers/expected-no-skip_konnect.yaml
@@ -5,6 +5,7 @@ consumer_groups:
 - name: basic
   plugins:
   - config:
+      compound_identifier: null
       consumer_groups: null
       dictionary_name: kong_rate_limiting_counters
       disable_penalty: false
@@ -32,6 +33,7 @@ consumer_groups:
         password: null
         port: 6379
         read_timeout: 2000
+        redis_proxy_type: null
         send_timeout: 2000
         sentinel_addresses: null
         sentinel_master: null

diff --git a/tests/integration/validate_test.go b/tests/integration/validate_test.go
@@ -76,6 +76,22 @@ func Test_Validate_Konnect(t *testing.T) {
 			additionalArgs: []string{"--konnect-runtime-group-name=default"},
 			errorExpected:  false,
 		},
+		{
+			name:           "validate with wrong online list, passed via --online-entities-list cli flag",
+			stateFile:      "testdata/validate/kong3x.yaml",
+			additionalArgs: []string{"--online-entities-list=services,Routes,Plugins"},
+			errorExpected:  true,
+			errorString: "invalid value 'services' for --online-entities-list; it should be a valid " +
+				"Kong entity (case-sensitive). Valid entities: [ACLGroups BasicAuths CACertificates Certificates Consumers " +
+				"Documents FilterChains HMACAuths JWTAuths KeyAuths Oauth2Creds Plugins RBACEndpointPermissions RBACRoles " +
+				"Routes SNIs Services Targets Upstreams Vaults]",
+		},
+		{
+			name:           "validate with correct online list, passed via --online-entities-list cli flag",
+			stateFile:      "testdata/validate/kong3x.yaml",
+			additionalArgs: []string{"--online-entities-list=Services,Routes,Plugins"},
+			errorExpected:  false,
+		},
 	}
 
 	for _, tc := range tests {
@@ -173,6 +189,16 @@ func Test_Validate_Gateway(t *testing.T) {
 			stateFile:      "testdata/validate/konnect.yaml",
 			additionalArgs: []string{"--konnect-compatibility"},
 		},
+		{
+			name:           "validate format version 3.0 with --online-entities-list",
+			stateFile:      "testdata/validate/kong3x.yaml",
+			additionalArgs: []string{"--online-entities-list=Services,Routes,Plugins"},
+		},
+		{
+			name:           "validate with konnect and --online-entities-list",
+			stateFile:      "testdata/validate/konnect.yaml",
+			additionalArgs: []string{"--online-entities-list=Services,Routes,Plugins"},
+		},
 	}
 
 	for _, tc := range tests {
@@ -218,6 +244,16 @@ func Test_Validate_Gateway_EE(t *testing.T) {
 			stateFile:      "testdata/validate/kong-ee.yaml",
 			additionalArgs: []string{"--workspace=default"},
 		},
+		{
+			name:           "validate format version 3.0 with --online-entities-list",
+			stateFile:      "testdata/validate/kong-ee.yaml",
+			additionalArgs: []string{"--online-entities-list=Services,Routes,Plugins"},
+		},
+		{
+			name:           "validate with konnect and --online-entities-list",
+			stateFile:      "testdata/validate/konnect.yaml",
+			additionalArgs: []string{"--online-entities-list=Services,Routes,Plugins"},
+		},
 		// TODO: Add a rbac flag test, once the behaviour is fixed
 	}
 

diff --git a/validate/validate.go b/validate/validate.go
@@ -16,28 +16,55 @@ import (
 )
 
 type Validator struct {
-	ctx               context.Context
-	state             *state.KongState
-	client            *kong.Client
-	parallelism       int
-	rbacResourcesOnly bool
+	ctx                  context.Context
+	state                *state.KongState
+	client               *kong.Client
+	parallelism          int
+	rbacResourcesOnly    bool
+	onlineEntitiesFilter []string
 }
 
 type ValidatorOpts struct {
-	Ctx               context.Context
-	State             *state.KongState
-	Client            *kong.Client
-	Parallelism       int
-	RBACResourcesOnly bool
+	Ctx                  context.Context
+	State                *state.KongState
+	Client               *kong.Client
+	Parallelism          int
+	RBACResourcesOnly    bool
+	OnlineEntitiesFilter []string
+}
+
+// Define a map of entity object field names and their corresponding string names
+var EntityMap = map[string]string{
+	"ACLGroups":               "acls",
+	"BasicAuths":              "basicauth_credentials",
+	"CACertificates":          "ca_certificates",
+	"Certificates":            "certificates",
+	"Consumers":               "consumers",
+	"Documents":               "documents",
+	"FilterChains":            "filter_chains",
+	"HMACAuths":               "hmacauth_credentials",
+	"JWTAuths":                "jwt_secrets",
+	"KeyAuths":                "keyauth_credentials",
+	"Oauth2Creds":             "oauth2_credentials",
+	"Plugins":                 "plugins",
+	"RBACEndpointPermissions": "rbac-endpointpermission",
+	"RBACRoles":               "rbac-role",
+	"Routes":                  "routes",
+	"SNIs":                    "snis",
+	"Services":                "services",
+	"Targets":                 "targets",
+	"Upstreams":               "upstreams",
+	"Vaults":                  "vaults",
 }
 
 func NewValidator(opt ValidatorOpts) *Validator {
 	return &Validator{
-		ctx:               opt.Ctx,
-		state:             opt.State,
-		client:            opt.Client,
-		parallelism:       opt.Parallelism,
-		rbacResourcesOnly: opt.RBACResourcesOnly,
+		ctx:                  opt.Ctx,
+		state:                opt.State,
+		client:               opt.Client,
+		parallelism:          opt.Parallelism,
+		rbacResourcesOnly:    opt.RBACResourcesOnly,
+		onlineEntitiesFilter: opt.OnlineEntitiesFilter,
 	}
 }
 
@@ -119,70 +146,48 @@ func (v *Validator) entities(obj interface{}, entityType string) []error {
 func (v *Validator) Validate(formatVersion semver.Version) []error {
 	allErr := []error{}
 
-	// validate RBAC resources first.
-	if err := v.entities(v.state.RBACEndpointPermissions, "rbac-endpointpermission"); err != nil {
-		allErr = append(allErr, err...)
-	}
-	if err := v.entities(v.state.RBACRoles, "rbac-role"); err != nil {
-		allErr = append(allErr, err...)
-	}
 	if v.rbacResourcesOnly {
+		// validate RBAC resources first.
+		if err := v.entities(v.state.RBACEndpointPermissions, "rbac-endpointpermission"); err != nil {
+			allErr = append(allErr, err...)
+		}
+		if err := v.entities(v.state.RBACRoles, "rbac-role"); err != nil {
+			allErr = append(allErr, err...)
+		}
 		return allErr
 	}
 
-	if err := v.entities(v.state.Services, "services"); err != nil {
-		allErr = append(allErr, err...)
-	}
-	if err := v.entities(v.state.ACLGroups, "acls"); err != nil {
-		allErr = append(allErr, err...)
-	}
-	if err := v.entities(v.state.BasicAuths, "basicauth_credentials"); err != nil {
-		allErr = append(allErr, err...)
-	}
-	if err := v.entities(v.state.CACertificates, "ca_certificates"); err != nil {
-		allErr = append(allErr, err...)
-	}
-	if err := v.entities(v.state.Certificates, "certificates"); err != nil {
-		allErr = append(allErr, err...)
-	}
-	if err := v.entities(v.state.Consumers, "consumers"); err != nil {
-		allErr = append(allErr, err...)
-	}
-	if err := v.entities(v.state.Documents, "documents"); err != nil {
-		allErr = append(allErr, err...)
-	}
-	if err := v.entities(v.state.HMACAuths, "hmacauth_credentials"); err != nil {
-		allErr = append(allErr, err...)
-	}
-	if err := v.entities(v.state.JWTAuths, "jwt_secrets"); err != nil {
-		allErr = append(allErr, err...)
-	}
-	if err := v.entities(v.state.KeyAuths, "keyauth_credentials"); err != nil {
-		allErr = append(allErr, err...)
-	}
-	if err := v.entities(v.state.Oauth2Creds, "oauth2_credentials"); err != nil {
-		allErr = append(allErr, err...)
-	}
-	if err := v.entities(v.state.Plugins, "plugins"); err != nil {
-		allErr = append(allErr, err...)
-	}
-	if err := v.entities(v.state.Routes, "routes"); err != nil {
-		allErr = append(allErr, err...)
-	}
-	if err := v.entities(v.state.SNIs, "snis"); err != nil {
-		allErr = append(allErr, err...)
-	}
-	if err := v.entities(v.state.Targets, "targets"); err != nil {
-		allErr = append(allErr, err...)
-	}
-	if err := v.entities(v.state.Upstreams, "upstreams"); err != nil {
-		allErr = append(allErr, err...)
-	}
-	if err := v.entities(v.state.FilterChains, "filter_chains"); err != nil {
-		allErr = append(allErr, err...)
+	// Create a copy of entityMap with only the specififed resources to check online.
+	filteredEntityMap := make(map[string]string)
+	if len(v.onlineEntitiesFilter) > 0 {
+		for _, value := range v.onlineEntitiesFilter {
+			for key, entityName := range EntityMap {
+				if value == key {
+					filteredEntityMap[key] = entityName
+				}
+			}
+		}
+	} else {
+		// If no filter is specified, use the original entityMap.
+		filteredEntityMap = EntityMap
 	}
-	if err := v.entities(v.state.Vaults, "vaults"); err != nil {
-		allErr = append(allErr, err...)
+
+	// Validate each entity using the filtered entityMap
+	for fieldName, entityName := range filteredEntityMap {
+		// Use reflection to get the value of the field from v.state
+		valueOfState := reflect.ValueOf(v.state)
+		if valueOfState.Kind() == reflect.Ptr {
+			valueOfState = valueOfState.Elem() // Dereference if it's a pointer
+		}
+
+		fieldValue := valueOfState.FieldByName(fieldName)
+		if fieldValue.IsValid() && fieldValue.CanInterface() {
+			if err := v.entities(fieldValue.Interface(), entityName); err != nil {
+				allErr = append(allErr, err...)
+			}
+		} else {
+			allErr = append(allErr, fmt.Errorf("invalid field '%s' in state", fieldName))
+		}
 	}
 
 	// validate routes format with Kong 3.x