From 2d9921fc681a8f12a07f4b3239aee14ae4ca0f7e Mon Sep 17 00:00:00 2001 From: Antoine Jacquemin Date: Wed, 21 Aug 2024 17:08:06 +0200 Subject: [PATCH 01/12] init --- pkg/dump/dump.go | 42 ++++++++++++++++++++++++++- pkg/file/builder.go | 48 +++++++++++++++++++++++++------ pkg/file/kong_json_schema.json | 12 ++++++++ pkg/file/reader.go | 9 ++++++ pkg/file/types.go | 2 ++ pkg/file/zz_generated.deepcopy.go | 5 ++++ 6 files changed, 109 insertions(+), 9 deletions(-) diff --git a/pkg/dump/dump.go b/pkg/dump/dump.go index 3e9ddbd..272470d 100644 --- a/pkg/dump/dump.go +++ b/pkg/dump/dump.go @@ -39,8 +39,10 @@ type Config struct { // LookUpSelectorTags* can be used to ensure state lookup for entities using // these tags. This functionality is essential when using a plugin that references // consumers or routes associated with tags different from those in the sync command. + LookUpSelectorTagsConsumerGroups []string LookUpSelectorTagsConsumers []string LookUpSelectorTagsRoutes []string + LookUpSelectorTagsServices []string // KonnectControlPlane KonnectControlPlane string @@ -94,6 +96,25 @@ func getConsumerGroupsConfiguration(ctx context.Context, group *errgroup.Group, } return fmt.Errorf("consumer_groups: %w", err) } + if config.LookUpSelectorTagsConsumerGroups != nil { + globalConsumerGroups, err := GetAllConsumerGroups(ctx, client, config.LookUpSelectorTagsConsumerGroups) + if err != nil { + return fmt.Errorf("error retrieving global consumer groups: %w", err) + } + // if globalConsumers are not present, add them. + for _, globalConsumerGroup := range globalConsumerGroups { + found := false + for _, consumerGroup := range consumerGroups { + if *globalConsumerGroup.ConsumerGroup.ID == *consumerGroup.ConsumerGroup.ID { + found = true + break + } + } + if !found { + consumerGroups = append(consumerGroups, globalConsumerGroup) + } + } + } state.ConsumerGroups = consumerGroups return nil }) @@ -214,6 +235,25 @@ func getProxyConfiguration(ctx context.Context, group *errgroup.Group, if err != nil { return fmt.Errorf("services: %w", err) } + if config.LookUpSelectorTagsServices != nil { + globalServices, err := GetAllServices(ctx, client, config.LookUpSelectorTagsServices) + if err != nil { + return fmt.Errorf("error retrieving global services: %w", err) + } + // if globalServices are not present, add them. + for _, globalService := range globalServices { + found := false + for _, service := range services { + if *globalService.ID == *service.ID { + found = true + break + } + } + if !found { + services = append(services, globalService) + } + } + } state.Services = services return nil }) @@ -1098,4 +1138,4 @@ func excludeConsumerGroupsPlugins(plugins []*kong.Plugin) []*kong.Plugin { filtered = append(filtered, p) } return filtered -} +} \ No newline at end of file diff --git a/pkg/file/builder.go b/pkg/file/builder.go index c0bf27f..782aa4a 100644 --- a/pkg/file/builder.go +++ b/pkg/file/builder.go @@ -27,12 +27,14 @@ type stateBuilder struct { defaulter *utils.Defaulter kongVersion semver.Version - selectTags []string - lookupTagsConsumers []string - lookupTagsRoutes []string - skipCACerts bool - includeLicenses bool - intermediate *state.KongState + selectTags []string + lookupTagsConsumerGroups []string + lookupTagsConsumers []string + lookupTagsRoutes []string + lookupTagsServices []string + skipCACerts bool + includeLicenses bool + intermediate *state.KongState client *kong.Client ctx context.Context @@ -177,7 +179,21 @@ func (b *stateBuilder) consumerGroups() { cg.ID = kong.String(*current.ID) } } - utils.MustMergeTags(&cg.ConsumerGroup, b.selectTags) + + stringTags := make([]string, len(cg.Tags)) + for i, tag := range cg.Tags { + if tag != nil { + stringTags[i] = *tag + } + } + sort.Strings(stringTags) + sort.Strings(b.lookupTagsConsumers) + // if the consumer tags and the lookup tags are the same, it means + // that the consumer is a global consumer retrieved from upstream, + // therefore we don't want to merge its tags with the selected tags. + if !reflect.DeepEqual(stringTags, b.lookupTagsConsumers) { + utils.MustMergeTags(&cg.ConsumerGroup, b.selectTags) + } cgo := kong.ConsumerGroupObject{ ConsumerGroup: &cg.ConsumerGroup, @@ -886,7 +902,23 @@ func (b *stateBuilder) ingestService(s *FService) error { s.ID = kong.String(*svc.ID) } } - utils.MustMergeTags(&s.Service, b.selectTags) + + stringTags := make([]string, len(s.Tags)) + for i, tag := range s.Tags { + if tag != nil { + stringTags[i] = *tag + } + } + sort.Strings(stringTags) + sort.Strings(b.lookupTagsServices) + // if the service tags and the lookup tags are the same, it means + // that the service is a global service retrieved from upstream, + // therefore we don't want to merge its tags with the selected tags. + if !reflect.DeepEqual(stringTags, b.lookupTagsServices) { + utils.MustMergeTags(&s.Service, b.selectTags) + } + + // utils.MustMergeTags(&s, b.selectTags) b.defaulter.MustSet(&s.Service) if svc != nil { s.Service.CreatedAt = svc.CreatedAt diff --git a/pkg/file/kong_json_schema.json b/pkg/file/kong_json_schema.json index c68dfb0..5063371 100644 --- a/pkg/file/kong_json_schema.json +++ b/pkg/file/kong_json_schema.json @@ -1477,6 +1477,18 @@ "type": "string" }, "type": "array" + }, + "services": { + "items": { + "type": "string" + }, + "type": "array" + }, + "consumer_groups": { + "items": { + "type": "string" + }, + "type": "array" } }, "additionalProperties": false, diff --git a/pkg/file/reader.go b/pkg/file/reader.go index 717b292..8f1663d 100644 --- a/pkg/file/reader.go +++ b/pkg/file/reader.go @@ -94,6 +94,15 @@ func Get(ctx context.Context, fileContent *Content, opt RenderConfig, dumpConfig builder.lookupTagsRoutes = dumpConfig.LookUpSelectorTagsRoutes } + if len(dumpConfig.LookUpSelectorTagsServices) > 0 { + builder.lookupTagsServices = dumpConfig.LookUpSelectorTagsServices + } + + if len(dumpConfig.LookUpSelectorTagsConsumerGroups) > 0 { + builder.lookupTagsConsumerGroups = dumpConfig.LookUpSelectorTagsConsumerGroups + } + + if fileContent.Transform != nil && !*fileContent.Transform { return nil, ErrorTransformFalseNotSupported } diff --git a/pkg/file/types.go b/pkg/file/types.go index ea112f3..ceb0010 100644 --- a/pkg/file/types.go +++ b/pkg/file/types.go @@ -778,8 +778,10 @@ type Info struct { // for corresponding entities already in Kong. // +k8s:deepcopy-gen=true type LookUpSelectorTags struct { + ConsumerGroups []string `json:"consumer_groups,omitempty" yaml:"consumer_groups,omitempty"` Consumers []string `json:"consumers,omitempty" yaml:"consumers,omitempty"` Routes []string `json:"routes,omitempty" yaml:"routes,omitempty"` + Services []string `json:"services,omitempty" yaml:"services,omitempty"` } // Konnect contains configuration specific to Konnect. diff --git a/pkg/file/zz_generated.deepcopy.go b/pkg/file/zz_generated.deepcopy.go index 29b74ab..511e018 100644 --- a/pkg/file/zz_generated.deepcopy.go +++ b/pkg/file/zz_generated.deepcopy.go @@ -886,6 +886,11 @@ func (in *LookUpSelectorTags) DeepCopyInto(out *LookUpSelectorTags) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.Services != nil { + in, out := &in.Services, &out.Services + *out = make([]string, len(*in)) + copy(*out, *in) + } return } From 399d2dbb997627e4eb479b328afe43fb5b1fb480 Mon Sep 17 00:00:00 2001 From: Antoine Jacquemin Date: Wed, 21 Aug 2024 17:36:11 +0200 Subject: [PATCH 02/12] init --- pkg/file/builder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/file/builder.go b/pkg/file/builder.go index 782aa4a..a6a8359 100644 --- a/pkg/file/builder.go +++ b/pkg/file/builder.go @@ -187,11 +187,11 @@ func (b *stateBuilder) consumerGroups() { } } sort.Strings(stringTags) - sort.Strings(b.lookupTagsConsumers) + sort.Strings(b.lookupTagsConsumerGroups) // if the consumer tags and the lookup tags are the same, it means // that the consumer is a global consumer retrieved from upstream, // therefore we don't want to merge its tags with the selected tags. - if !reflect.DeepEqual(stringTags, b.lookupTagsConsumers) { + if !reflect.DeepEqual(stringTags, b.lookupTagsConsumerGroups) { utils.MustMergeTags(&cg.ConsumerGroup, b.selectTags) } From 82d71c11b5a42083281650998ec985b3201ec249 Mon Sep 17 00:00:00 2001 From: Antoine Jacquemin Date: Wed, 21 Aug 2024 17:38:43 +0200 Subject: [PATCH 03/12] init --- pkg/file/builder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/file/builder.go b/pkg/file/builder.go index a6a8359..b07d33d 100644 --- a/pkg/file/builder.go +++ b/pkg/file/builder.go @@ -188,8 +188,8 @@ func (b *stateBuilder) consumerGroups() { } sort.Strings(stringTags) sort.Strings(b.lookupTagsConsumerGroups) - // if the consumer tags and the lookup tags are the same, it means - // that the consumer is a global consumer retrieved from upstream, + // if the consumer group tags and the lookup tags are the same, it means + // that the consumer group is a global consumer group retrieved from upstream, // therefore we don't want to merge its tags with the selected tags. if !reflect.DeepEqual(stringTags, b.lookupTagsConsumerGroups) { utils.MustMergeTags(&cg.ConsumerGroup, b.selectTags) From 1f3717272a60985d3891cd1ecc1e2fabfbcf2248 Mon Sep 17 00:00:00 2001 From: Antoine Jacquemin Date: Wed, 21 Aug 2024 17:52:27 +0200 Subject: [PATCH 04/12] deepCopy --- pkg/file/zz_generated.deepcopy.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/file/zz_generated.deepcopy.go b/pkg/file/zz_generated.deepcopy.go index 511e018..436a529 100644 --- a/pkg/file/zz_generated.deepcopy.go +++ b/pkg/file/zz_generated.deepcopy.go @@ -876,6 +876,11 @@ func (in *Konnect) DeepCopy() *Konnect { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LookUpSelectorTags) DeepCopyInto(out *LookUpSelectorTags) { *out = *in + if in.ConsumerGroups != nil { + in, out := &in.ConsumerGroups, &out.ConsumerGroups + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.Consumers != nil { in, out := &in.Consumers, &out.Consumers *out = make([]string, len(*in)) From d4f21607a7234800b58c428e48793ec436b8b25a Mon Sep 17 00:00:00 2001 From: Antoine Jacquemin Date: Wed, 21 Aug 2024 19:04:13 +0200 Subject: [PATCH 05/12] lint --- pkg/dump/dump.go | 6 +++--- pkg/file/types.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/dump/dump.go b/pkg/dump/dump.go index 272470d..7dfc113 100644 --- a/pkg/dump/dump.go +++ b/pkg/dump/dump.go @@ -40,9 +40,9 @@ type Config struct { // these tags. This functionality is essential when using a plugin that references // consumers or routes associated with tags different from those in the sync command. LookUpSelectorTagsConsumerGroups []string - LookUpSelectorTagsConsumers []string - LookUpSelectorTagsRoutes []string - LookUpSelectorTagsServices []string + LookUpSelectorTagsConsumers []string + LookUpSelectorTagsRoutes []string + LookUpSelectorTagsServices []string // KonnectControlPlane KonnectControlPlane string diff --git a/pkg/file/types.go b/pkg/file/types.go index ceb0010..443dbeb 100644 --- a/pkg/file/types.go +++ b/pkg/file/types.go @@ -779,9 +779,9 @@ type Info struct { // +k8s:deepcopy-gen=true type LookUpSelectorTags struct { ConsumerGroups []string `json:"consumer_groups,omitempty" yaml:"consumer_groups,omitempty"` - Consumers []string `json:"consumers,omitempty" yaml:"consumers,omitempty"` - Routes []string `json:"routes,omitempty" yaml:"routes,omitempty"` - Services []string `json:"services,omitempty" yaml:"services,omitempty"` + Consumers []string `json:"consumers,omitempty" yaml:"consumers,omitempty"` + Routes []string `json:"routes,omitempty" yaml:"routes,omitempty"` + Services []string `json:"services,omitempty" yaml:"services,omitempty"` } // Konnect contains configuration specific to Konnect. From 7f48890e2161b79761125ea44c4b9d6c0d88f15e Mon Sep 17 00:00:00 2001 From: Antoine Jacquemin Date: Wed, 21 Aug 2024 19:35:26 +0200 Subject: [PATCH 06/12] fix --- pkg/dump/dump.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/dump/dump.go b/pkg/dump/dump.go index 7dfc113..1d5db48 100644 --- a/pkg/dump/dump.go +++ b/pkg/dump/dump.go @@ -294,7 +294,6 @@ func getProxyConfiguration(ctx context.Context, group *errgroup.Group, plugins = excludeKonnectManagedPlugins(plugins) if config.SkipConsumers { plugins = excludeConsumersPlugins(plugins) - plugins = excludeConsumerGroupsPlugins(plugins) } state.Plugins = plugins return nil From 4ff86b286f920d0041819f287a93981745d28883 Mon Sep 17 00:00:00 2001 From: Antoine Jacquemin Date: Wed, 21 Aug 2024 19:42:50 +0200 Subject: [PATCH 07/12] revert --- pkg/dump/dump.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/dump/dump.go b/pkg/dump/dump.go index 1d5db48..7dfc113 100644 --- a/pkg/dump/dump.go +++ b/pkg/dump/dump.go @@ -294,6 +294,7 @@ func getProxyConfiguration(ctx context.Context, group *errgroup.Group, plugins = excludeKonnectManagedPlugins(plugins) if config.SkipConsumers { plugins = excludeConsumersPlugins(plugins) + plugins = excludeConsumerGroupsPlugins(plugins) } state.Plugins = plugins return nil From 0dbb4f88bcd5ddcee4dd7443d27ec6bd850ef228 Mon Sep 17 00:00:00 2001 From: Antoine Jacquemin Date: Wed, 28 Aug 2024 08:32:49 +0200 Subject: [PATCH 08/12] fix lint --- pkg/dump/dump.go | 2 +- pkg/file/builder.go | 2 +- pkg/file/reader.go | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/dump/dump.go b/pkg/dump/dump.go index 7dfc113..339e94c 100644 --- a/pkg/dump/dump.go +++ b/pkg/dump/dump.go @@ -1138,4 +1138,4 @@ func excludeConsumerGroupsPlugins(plugins []*kong.Plugin) []*kong.Plugin { filtered = append(filtered, p) } return filtered -} \ No newline at end of file +} diff --git a/pkg/file/builder.go b/pkg/file/builder.go index e290af2..bbba2e1 100644 --- a/pkg/file/builder.go +++ b/pkg/file/builder.go @@ -179,7 +179,7 @@ func (b *stateBuilder) consumerGroups() { cg.ID = kong.String(*current.ID) } } - + stringTags := make([]string, len(cg.Tags)) for i, tag := range cg.Tags { if tag != nil { diff --git a/pkg/file/reader.go b/pkg/file/reader.go index 8f1663d..f7d97ba 100644 --- a/pkg/file/reader.go +++ b/pkg/file/reader.go @@ -102,7 +102,6 @@ func Get(ctx context.Context, fileContent *Content, opt RenderConfig, dumpConfig builder.lookupTagsConsumerGroups = dumpConfig.LookUpSelectorTagsConsumerGroups } - if fileContent.Transform != nil && !*fileContent.Transform { return nil, ErrorTransformFalseNotSupported } From b5b73ab71a8fdc01df82fa27be91121bf7eab5e1 Mon Sep 17 00:00:00 2001 From: Antoine Jacquemin Date: Mon, 9 Sep 2024 14:59:21 +0200 Subject: [PATCH 09/12] fix skip consumer --- pkg/dump/dump.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/pkg/dump/dump.go b/pkg/dump/dump.go index 339e94c..c905806 100644 --- a/pkg/dump/dump.go +++ b/pkg/dump/dump.go @@ -291,7 +291,30 @@ func getProxyConfiguration(ctx context.Context, group *errgroup.Group, if err != nil { return fmt.Errorf("plugins: %w", err) } + + if config.LookUpSelectorTagsConsumerGroups != nil { + globalConsumerGroupsPlugins, err := GetAllPlugins(ctx, client, config.LookUpSelectorTagsConsumerGroups) + if err != nil { + return fmt.Errorf("error retrieving global plugins: %w", err) + } + // if globalServices are not present, add them. + + for _, globalConsumerGroupsPlugin := range globalConsumerGroupsPlugins { + found := false + for _, plugin := range plugins { + if *globalConsumerGroupsPlugin.ID == *plugin.ID { + found = true + break + } + } + if !found { + plugins = append(plugins, globalConsumerGroupsPlugin) + } + } + } + plugins = excludeKonnectManagedPlugins(plugins) + if config.SkipConsumers { plugins = excludeConsumersPlugins(plugins) plugins = excludeConsumerGroupsPlugins(plugins) From a35f809a04a716e1cc920a7ff56268f753f15d64 Mon Sep 17 00:00:00 2001 From: Antoine Jacquemin Date: Mon, 9 Sep 2024 17:12:43 +0200 Subject: [PATCH 10/12] fix lint --- pkg/dump/dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/dump/dump.go b/pkg/dump/dump.go index c905806..e321125 100644 --- a/pkg/dump/dump.go +++ b/pkg/dump/dump.go @@ -298,7 +298,7 @@ func getProxyConfiguration(ctx context.Context, group *errgroup.Group, return fmt.Errorf("error retrieving global plugins: %w", err) } // if globalServices are not present, add them. - + for _, globalConsumerGroupsPlugin := range globalConsumerGroupsPlugins { found := false for _, plugin := range plugins { From 89c89e6b94cec7125f0fb0c1432ca1e6e3478f1b Mon Sep 17 00:00:00 2001 From: Michael Heap Date: Fri, 13 Sep 2024 08:49:41 +0100 Subject: [PATCH 11/12] Remove unused nolint:gosec --- pkg/file/readfile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/file/readfile.go b/pkg/file/readfile.go index 466f5f1..f99ad5d 100644 --- a/pkg/file/readfile.go +++ b/pkg/file/readfile.go @@ -67,7 +67,7 @@ func getContent(filenames []string, mockEnvVars bool) (*Content, error) { func getReaders(fileOrDir string) (map[string]io.Reader, error) { // special case where `-` means stdin if fileOrDir == "-" { - if term.IsTerminal(int(os.Stdin.Fd())) && term.IsTerminal(int(os.Stderr.Fd())) { //nolint:gosec + if term.IsTerminal(int(os.Stdin.Fd())) && term.IsTerminal(int(os.Stderr.Fd())) { fmt.Fprintf(os.Stderr, "reading input from stdin...\n") } return map[string]io.Reader{"STDIN": os.Stdin}, nil From b49f6a26fd446ac038d6944ce5ea1c85c07d5d1c Mon Sep 17 00:00:00 2001 From: Antoine Jacquemin Date: Wed, 18 Sep 2024 15:27:36 +0200 Subject: [PATCH 12/12] fix --- pkg/dump/dump.go | 21 --------------------- pkg/file/builder.go | 1 - 2 files changed, 22 deletions(-) diff --git a/pkg/dump/dump.go b/pkg/dump/dump.go index e321125..534fe98 100644 --- a/pkg/dump/dump.go +++ b/pkg/dump/dump.go @@ -292,27 +292,6 @@ func getProxyConfiguration(ctx context.Context, group *errgroup.Group, return fmt.Errorf("plugins: %w", err) } - if config.LookUpSelectorTagsConsumerGroups != nil { - globalConsumerGroupsPlugins, err := GetAllPlugins(ctx, client, config.LookUpSelectorTagsConsumerGroups) - if err != nil { - return fmt.Errorf("error retrieving global plugins: %w", err) - } - // if globalServices are not present, add them. - - for _, globalConsumerGroupsPlugin := range globalConsumerGroupsPlugins { - found := false - for _, plugin := range plugins { - if *globalConsumerGroupsPlugin.ID == *plugin.ID { - found = true - break - } - } - if !found { - plugins = append(plugins, globalConsumerGroupsPlugin) - } - } - } - plugins = excludeKonnectManagedPlugins(plugins) if config.SkipConsumers { diff --git a/pkg/file/builder.go b/pkg/file/builder.go index bbba2e1..35f6780 100644 --- a/pkg/file/builder.go +++ b/pkg/file/builder.go @@ -918,7 +918,6 @@ func (b *stateBuilder) ingestService(s *FService) error { utils.MustMergeTags(&s.Service, b.selectTags) } - // utils.MustMergeTags(&s, b.selectTags) b.defaulter.MustSet(&s.Service) if svc != nil { s.Service.CreatedAt = svc.CreatedAt