feat/default-lookup-services-consumerGroups (#130)

This PR is adding some methods require for the default-lookup-services-consumerGroups 
It allow to retrieve consumerGroups and Service using a different tag
It also add a fix to skip Consumers Plugins when using the skip-consumers flag

---------

Co-authored-by: Prashansa Kulshrestha <[email protected]>
Co-authored-by: Michael Heap <[email protected]>

pkg/dump/dump.go

@@ -39,8 +39,10 @@ type Config struct {
 	// LookUpSelectorTags* can be used to ensure state lookup for entities using
 	// these tags. This functionality is essential when using a plugin that references
 	// consumers or routes associated with tags different from those in the sync command.
-	LookUpSelectorTagsConsumers []string
-	LookUpSelectorTagsRoutes    []string
+	LookUpSelectorTagsConsumerGroups []string
+	LookUpSelectorTagsConsumers      []string
+	LookUpSelectorTagsRoutes         []string
+	LookUpSelectorTagsServices       []string
 
 	// KonnectControlPlane
 	KonnectControlPlane string
@@ -94,6 +96,25 @@ func getConsumerGroupsConfiguration(ctx context.Context, group *errgroup.Group,
 			}
 			return fmt.Errorf("consumer_groups: %w", err)
 		}
+		if config.LookUpSelectorTagsConsumerGroups != nil {
+			globalConsumerGroups, err := GetAllConsumerGroups(ctx, client, config.LookUpSelectorTagsConsumerGroups)
+			if err != nil {
+				return fmt.Errorf("error retrieving global consumer groups: %w", err)
+			}
+			// if globalConsumers are not present, add them.
+			for _, globalConsumerGroup := range globalConsumerGroups {
+				found := false
+				for _, consumerGroup := range consumerGroups {
+					if *globalConsumerGroup.ConsumerGroup.ID == *consumerGroup.ConsumerGroup.ID {
+						found = true
+						break
+					}
+				}
+				if !found {
+					consumerGroups = append(consumerGroups, globalConsumerGroup)
+				}
+			}
+		}
 		state.ConsumerGroups = consumerGroups
 		return nil
 	})
@@ -214,6 +235,25 @@ func getProxyConfiguration(ctx context.Context, group *errgroup.Group,
 		if err != nil {
 			return fmt.Errorf("services: %w", err)
 		}
+		if config.LookUpSelectorTagsServices != nil {
+			globalServices, err := GetAllServices(ctx, client, config.LookUpSelectorTagsServices)
+			if err != nil {
+				return fmt.Errorf("error retrieving global services: %w", err)
+			}
+			// if globalServices are not present, add them.
+			for _, globalService := range globalServices {
+				found := false
+				for _, service := range services {
+					if *globalService.ID == *service.ID {
+						found = true
+						break
+					}
+				}
+				if !found {
+					services = append(services, globalService)
+				}
+			}
+		}
 		state.Services = services
 		return nil
 	})
@@ -251,7 +291,9 @@ func getProxyConfiguration(ctx context.Context, group *errgroup.Group,
 		if err != nil {
 			return fmt.Errorf("plugins: %w", err)
 		}
+
 		plugins = excludeKonnectManagedPlugins(plugins)
+
 		if config.SkipConsumers {
 			plugins = excludeConsumersPlugins(plugins)
 			plugins = excludeConsumerGroupsPlugins(plugins)

pkg/file/builder.go

@@ -27,12 +27,14 @@ type stateBuilder struct {
 	defaulter       *utils.Defaulter
 	kongVersion     semver.Version
 
-	selectTags          []string
-	lookupTagsConsumers []string
-	lookupTagsRoutes    []string
-	skipCACerts         bool
-	includeLicenses     bool
-	intermediate        *state.KongState
+	selectTags               []string
+	lookupTagsConsumerGroups []string
+	lookupTagsConsumers      []string
+	lookupTagsRoutes         []string
+	lookupTagsServices       []string
+	skipCACerts              bool
+	includeLicenses          bool
+	intermediate             *state.KongState
 
 	client *kong.Client
 	ctx    context.Context
@@ -177,7 +179,21 @@ func (b *stateBuilder) consumerGroups() {
 				cg.ID = kong.String(*current.ID)
 			}
 		}
-		utils.MustMergeTags(&cg.ConsumerGroup, b.selectTags)
+
+		stringTags := make([]string, len(cg.Tags))
+		for i, tag := range cg.Tags {
+			if tag != nil {
+				stringTags[i] = *tag
+			}
+		}
+		sort.Strings(stringTags)
+		sort.Strings(b.lookupTagsConsumerGroups)
+		// if the consumer group tags and the lookup tags are the same, it means
+		// that the consumer group is a global consumer group retrieved from upstream,
+		// therefore we don't want to merge its tags with the selected tags.
+		if !reflect.DeepEqual(stringTags, b.lookupTagsConsumerGroups) {
+			utils.MustMergeTags(&cg.ConsumerGroup, b.selectTags)
+		}
 
 		cgo := kong.ConsumerGroupObject{
 			ConsumerGroup: &cg.ConsumerGroup,
@@ -886,7 +902,22 @@ func (b *stateBuilder) ingestService(s *FService) error {
 			s.ID = kong.String(*svc.ID)
 		}
 	}
-	utils.MustMergeTags(&s.Service, b.selectTags)
+
+	stringTags := make([]string, len(s.Tags))
+	for i, tag := range s.Tags {
+		if tag != nil {
+			stringTags[i] = *tag
+		}
+	}
+	sort.Strings(stringTags)
+	sort.Strings(b.lookupTagsServices)
+	// if the service tags and the lookup tags are the same, it means
+	// that the service is a global service retrieved from upstream,
+	// therefore we don't want to merge its tags with the selected tags.
+	if !reflect.DeepEqual(stringTags, b.lookupTagsServices) {
+		utils.MustMergeTags(&s.Service, b.selectTags)
+	}
+
 	b.defaulter.MustSet(&s.Service)
 	if svc != nil {
 		s.Service.CreatedAt = svc.CreatedAt

pkg/file/kong_json_schema.json

@@ -1477,6 +1477,18 @@
             "type": "string"
           },
           "type": "array"
+        },
+        "services": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "consumer_groups": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
         }
       },
       "additionalProperties": false,

pkg/file/reader.go

@@ -94,6 +94,14 @@ func Get(ctx context.Context, fileContent *Content, opt RenderConfig, dumpConfig
 		builder.lookupTagsRoutes = dumpConfig.LookUpSelectorTagsRoutes
 	}
 
+	if len(dumpConfig.LookUpSelectorTagsServices) > 0 {
+		builder.lookupTagsServices = dumpConfig.LookUpSelectorTagsServices
+	}
+
+	if len(dumpConfig.LookUpSelectorTagsConsumerGroups) > 0 {
+		builder.lookupTagsConsumerGroups = dumpConfig.LookUpSelectorTagsConsumerGroups
+	}
+
 	if fileContent.Transform != nil && !*fileContent.Transform {
 		return nil, ErrorTransformFalseNotSupported
 	}

pkg/file/readfile.go

@@ -67,7 +67,7 @@ func getContent(filenames []string, mockEnvVars bool) (*Content, error) {
 func getReaders(fileOrDir string) (map[string]io.Reader, error) {
 	// special case where `-` means stdin
 	if fileOrDir == "-" {
-		if term.IsTerminal(int(os.Stdin.Fd())) && term.IsTerminal(int(os.Stderr.Fd())) { //nolint:gosec
+		if term.IsTerminal(int(os.Stdin.Fd())) && term.IsTerminal(int(os.Stderr.Fd())) {
 			fmt.Fprintf(os.Stderr, "reading input from stdin...\n")
 		}
 		return map[string]io.Reader{"STDIN": os.Stdin}, nil

pkg/file/types.go

@@ -778,8 +778,10 @@ type Info struct {
 // for corresponding entities already in Kong.
 // +k8s:deepcopy-gen=true
 type LookUpSelectorTags struct {
-	Consumers []string `json:"consumers,omitempty" yaml:"consumers,omitempty"`
-	Routes    []string `json:"routes,omitempty" yaml:"routes,omitempty"`
+	ConsumerGroups []string `json:"consumer_groups,omitempty" yaml:"consumer_groups,omitempty"`
+	Consumers      []string `json:"consumers,omitempty" yaml:"consumers,omitempty"`
+	Routes         []string `json:"routes,omitempty" yaml:"routes,omitempty"`
+	Services       []string `json:"services,omitempty" yaml:"services,omitempty"`
 }
 
 // Konnect contains configuration specific to Konnect.