diff --git a/pkg/dump/dump.go b/pkg/dump/dump.go index 625f042..1361db8 100644 --- a/pkg/dump/dump.go +++ b/pkg/dump/dump.go @@ -49,6 +49,8 @@ type Config struct { // IsConsumerGroupScopedPluginSupported IsConsumerGroupScopedPluginSupported bool + + IsPartialApply bool } func deduplicate(stringSlice []string) []string { diff --git a/pkg/file/builder.go b/pkg/file/builder.go index 012e74f..936a095 100644 --- a/pkg/file/builder.go +++ b/pkg/file/builder.go @@ -53,6 +53,8 @@ type stateBuilder struct { removePathHandlingFromExpressionRoute bool + isPartialApply bool + err error } @@ -167,6 +169,23 @@ func (b *stateBuilder) consumerGroups() { return } + // Load all existing consumer groups in to the immediate state for + // foreign key lookups if we're doing a partial apply + if b.isPartialApply { + consumerGroups, err := b.currentState.ConsumerGroups.GetAll() + if err != nil { + b.err = err + return + } + for _, cg := range consumerGroups { + err = b.intermediate.ConsumerGroups.Add(*cg) + if err != nil { + b.err = err + return + } + } + } + for _, cg := range b.targetContent.ConsumerGroups { current, err := b.currentState.ConsumerGroups.Get(*cg.Name) if utils.Empty(cg.ID) { @@ -199,7 +218,7 @@ func (b *stateBuilder) consumerGroups() { ConsumerGroup: &cg.ConsumerGroup, } - err = b.intermediate.ConsumerGroups.Add(state.ConsumerGroup{ConsumerGroup: cg.ConsumerGroup}) + err = b.intermediate.ConsumerGroups.AddIgnoringDuplicates(state.ConsumerGroup{ConsumerGroup: cg.ConsumerGroup}) if err != nil { b.err = err return @@ -248,6 +267,23 @@ func (b *stateBuilder) certificates() { return } + // Load all existing certificates in to the immediate state for + // foreign key lookups if we're doing a partial apply + if b.isPartialApply { + certs, err := b.currentState.Certificates.GetAll() + if err != nil { + b.err = err + return + } + for _, c := range certs { + err = b.intermediate.Certificates.Add(*c) + if err != nil { + b.err = err + return + } + } + } + for i := range b.targetContent.Certificates { c := b.targetContent.Certificates[i] if utils.Empty(c.ID) { @@ -314,6 +350,23 @@ func (b *stateBuilder) caCertificates() { return } + // Load all existing CA certificates in to the immediate state for + // foreign key lookups if we're doing a partial apply + if b.isPartialApply { + certs, err := b.currentState.CACertificates.GetAll() + if err != nil { + b.err = err + return + } + for _, c := range certs { + err = b.intermediate.CACertificates.Add(*c) + if err != nil { + b.err = err + return + } + } + } + for _, c := range b.targetContent.CACertificates { cert, err := b.currentState.CACertificates.Get(*c.Cert) if utils.Empty(c.ID) { @@ -388,11 +441,11 @@ func (b *stateBuilder) ingestConsumerGroupConsumer(cgID *string, c *FConsumer) ( } b.rawState.Consumers = append(b.rawState.Consumers, &c.Consumer) - err = b.intermediate.Consumers.Add(state.Consumer{Consumer: c.Consumer}) + err = b.intermediate.Consumers.AddIgnoringDuplicates(state.Consumer{Consumer: c.Consumer}) if err != nil { return nil, err } - err = b.intermediate.ConsumerGroupConsumers.Add(state.ConsumerGroupConsumer{ + err = b.intermediate.ConsumerGroupConsumers.AddIgnoringDuplicates(state.ConsumerGroupConsumer{ ConsumerGroupConsumer: kong.ConsumerGroupConsumer{ ConsumerGroup: &kong.ConsumerGroup{ID: cgID}, Consumer: &c.Consumer, @@ -409,6 +462,24 @@ func (b *stateBuilder) consumers() { return } + // Load all existing consumers in to the immediate state for + // foreign key lookups if we're doing a partial apply + if b.isPartialApply { + consumers, err := b.currentState.Consumers.GetAll() + if err != nil { + b.err = err + return + } + + for _, c := range consumers { + err = b.intermediate.Consumers.Add(*c) + if err != nil { + b.err = err + return + } + } + } + for _, c := range b.targetContent.Consumers { var ( @@ -473,7 +544,7 @@ func (b *stateBuilder) consumers() { } if !consumerAlreadyAdded { b.rawState.Consumers = append(b.rawState.Consumers, &c.Consumer) - err = b.intermediate.Consumers.Add(state.Consumer{Consumer: c.Consumer}) + err = b.intermediate.Consumers.AddIgnoringDuplicates(state.Consumer{Consumer: c.Consumer}) if err != nil { b.err = err return @@ -865,6 +936,23 @@ func (b *stateBuilder) services() { return } + // Load all existing services in to the immediate state for + // foreign key lookups if we're doing a partial apply + if b.isPartialApply { + services, err := b.currentState.Services.GetAll() + if err != nil { + b.err = err + return + } + for _, s := range services { + err = b.intermediate.Services.Add(*s) + if err != nil { + b.err = err + return + } + } + } + for _, s := range b.targetContent.Services { err := b.ingestService(&s) if err != nil { @@ -912,7 +1000,7 @@ func (b *stateBuilder) ingestService(s *FService) error { s.Service.CreatedAt = svc.CreatedAt } b.rawState.Services = append(b.rawState.Services, &s.Service) - err = b.intermediate.Services.Add(state.Service{Service: s.Service}) + err = b.intermediate.Services.AddIgnoringDuplicates(state.Service{Service: s.Service}) if err != nil { return err } @@ -952,6 +1040,24 @@ func (b *stateBuilder) routes() { return } + // Load all existing routes in to the immediate state for + // foreign key lookups if we're doing a partial apply + if b.isPartialApply { + routes, err := b.currentState.Routes.GetAll() + if err != nil { + b.err = err + return + } + + for _, r := range routes { + err = b.intermediate.Routes.Add(*r) + if err != nil { + b.err = err + return + } + } + } + for _, r := range b.targetContent.Routes { if err := b.ingestRoute(r); err != nil { b.err = err @@ -1418,7 +1524,7 @@ func (b *stateBuilder) ingestRoute(r FRoute) error { } b.rawState.Routes = append(b.rawState.Routes, &r.Route) - err = b.intermediate.Routes.Add(state.Route{Route: r.Route}) + err = b.intermediate.Routes.AddIgnoringDuplicates(state.Route{Route: r.Route}) if err != nil { return err } diff --git a/pkg/file/reader.go b/pkg/file/reader.go index f7d97ba..b1b5b4b 100644 --- a/pkg/file/reader.go +++ b/pkg/file/reader.go @@ -81,6 +81,7 @@ func Get(ctx context.Context, fileContent *Content, opt RenderConfig, dumpConfig builder.skipCACerts = dumpConfig.SkipCACerts builder.isKonnect = dumpConfig.KonnectControlPlane != "" builder.includeLicenses = dumpConfig.IncludeLicenses + builder.isPartialApply = dumpConfig.IsPartialApply if len(dumpConfig.SelectorTags) > 0 { builder.selectTags = dumpConfig.SelectorTags diff --git a/pkg/state/consumer.go b/pkg/state/consumer.go index d7f4cfb..07dc5c5 100644 --- a/pkg/state/consumer.go +++ b/pkg/state/consumer.go @@ -39,6 +39,44 @@ var consumerTableSchema = &memdb.TableSchema{ // ConsumersCollection stores and indexes Kong Consumers. type ConsumersCollection collection +func (k *ConsumersCollection) AddIgnoringDuplicates(consumer Consumer) error { + // Detect duplicates + if !utils.Empty(consumer.ID) { + c, err := k.GetByIDOrUsername(*consumer.ID) + if c != nil { + return nil + } + + if err != nil && !errors.Is(err, ErrNotFound) { + return err + } + } + + if !utils.Empty(consumer.Username) { + c, err := k.GetByIDOrUsername(*consumer.Username) + if c != nil { + return nil + } + + if err != nil && !errors.Is(err, ErrNotFound) { + return err + } + } + + // Check for custom ID + if !utils.Empty(consumer.CustomID) { + c, err := k.GetByCustomID(*consumer.CustomID) + if c != nil { + return nil + } + if err != nil && !errors.Is(err, ErrNotFound) { + return err + } + } + + return k.Add(consumer) +} + // Add adds a consumer to the collection // An error is thrown if consumer.ID is empty. func (k *ConsumersCollection) Add(consumer Consumer) error { diff --git a/pkg/state/consumer_group.go b/pkg/state/consumer_group.go index 6c9de07..4f30588 100644 --- a/pkg/state/consumer_group.go +++ b/pkg/state/consumer_group.go @@ -32,6 +32,31 @@ var consumerGroupTableSchema = &memdb.TableSchema{ // consumerGroupsCollection stores and indexes Kong consumerGroups. type ConsumerGroupsCollection collection +func (k *ConsumerGroupsCollection) AddIgnoringDuplicates(consumerGroup ConsumerGroup) error { + // Detect duplicates + if !utils.Empty(consumerGroup.ID) { + cg, err := k.Get(*consumerGroup.ID) + if cg != nil { + return nil + } + if err != nil && !errors.Is(err, ErrNotFound) { + return err + } + } + + if !utils.Empty(consumerGroup.Name) { + cg, err := k.Get(*consumerGroup.Name) + if cg != nil { + return nil + } + if err != nil && !errors.Is(err, ErrNotFound) { + return err + } + } + + return k.Add(consumerGroup) +} + // Add adds an consumerGroup to the collection. // consumerGroup.ID should not be nil else an error is thrown. func (k *ConsumerGroupsCollection) Add(consumerGroup ConsumerGroup) error { diff --git a/pkg/state/consumer_group_consumers.go b/pkg/state/consumer_group_consumers.go index 317565a..aeb9d30 100644 --- a/pkg/state/consumer_group_consumers.go +++ b/pkg/state/consumer_group_consumers.go @@ -110,6 +110,31 @@ func validateConsumerGroup(consumer *ConsumerGroupConsumer) error { // ConsumerGroupConsumersCollection stores and indexes Kong consumerGroupConsumers. type ConsumerGroupConsumersCollection collection +func (k *ConsumerGroupConsumersCollection) AddIgnoringDuplicates(consumer ConsumerGroupConsumer) error { + // Detect duplicates + if !utils.Empty(consumer.Consumer.ID) { + cgc, err := k.Get(*consumer.Consumer.ID, *consumer.ConsumerGroup.ID) + if cgc != nil { + return nil + } + if err != nil && !errors.Is(err, ErrNotFound) { + return err + } + } + + if !utils.Empty(consumer.Consumer.Username) { + cgc, err := k.Get(*consumer.Consumer.Username, *consumer.ConsumerGroup.ID) + if cgc != nil { + return nil + } + if err != nil && !errors.Is(err, ErrNotFound) { + return err + } + } + + return k.Add(consumer) +} + // Add adds a consumerGroupConsumer to the collection. func (k *ConsumerGroupConsumersCollection) Add(consumer ConsumerGroupConsumer) error { if utils.Empty(consumer.Consumer.ID) { diff --git a/pkg/state/consumer_group_test.go b/pkg/state/consumer_group_test.go new file mode 100644 index 0000000..cc24da5 --- /dev/null +++ b/pkg/state/consumer_group_test.go @@ -0,0 +1,41 @@ +package state + +import ( + "testing" + + "github.com/kong/go-kong/kong" + "github.com/stretchr/testify/assert" +) + +func consumerGroupsCollection() *ConsumerGroupsCollection { + return state().ConsumerGroups +} + +func TestConsumerGroupInsert(t *testing.T) { + assert := assert.New(t) + collection := consumerGroupsCollection() + + var cg ConsumerGroup + + assert.NotNil(collection.Add(cg)) + + cg.ID = kong.String("my-id") + cg.Name = kong.String("first") + assert.Nil(collection.Add(cg)) + + // re-insert + assert.NotNil(collection.Add(cg)) +} + +func TestConsumerGroupInsertIgnoreDuplicate(t *testing.T) { + assert := assert.New(t) + collection := consumerGroupsCollection() + + var cg ConsumerGroup + cg.ID = kong.String("my-id") + cg.Name = kong.String("first") + err := collection.Add(cg) + assert.Nil(err) + err = collection.AddIgnoringDuplicates(cg) + assert.Nil(err) +} diff --git a/pkg/state/consumer_test.go b/pkg/state/consumer_test.go index 35ecf53..26bbe31 100644 --- a/pkg/state/consumer_test.go +++ b/pkg/state/consumer_test.go @@ -27,6 +27,32 @@ func TestConsumerInsert(t *testing.T) { assert.NotNil(collection.Add(consumer)) } +func TestConsumerInsertIgnoreDuplicateUsername(t *testing.T) { + assert := assert.New(t) + collection := consumersCollection() + + var consumer Consumer + consumer.ID = kong.String("first") + consumer.Username = kong.String("my-name") + err := collection.Add(consumer) + assert.Nil(err) + err = collection.AddIgnoringDuplicates(consumer) + assert.Nil(err) +} + +func TestConsumerInsertIgnoreDuplicateCustomId(t *testing.T) { + assert := assert.New(t) + collection := consumersCollection() + + var consumer Consumer + consumer.ID = kong.String("first") + consumer.CustomID = kong.String("my-name") + err := collection.Add(consumer) + assert.Nil(err) + err = collection.AddIgnoringDuplicates(consumer) + assert.Nil(err) +} + func TestConsumerGetUpdate(t *testing.T) { assert := assert.New(t) collection := consumersCollection() diff --git a/pkg/state/route.go b/pkg/state/route.go index 0c9b389..00441c1 100644 --- a/pkg/state/route.go +++ b/pkg/state/route.go @@ -48,6 +48,33 @@ var routeTableSchema = &memdb.TableSchema{ // RoutesCollection stores and indexes Kong Routes. type RoutesCollection collection +func (k *RoutesCollection) AddIgnoringDuplicates(route Route) error { + // Detect duplicates + if !utils.Empty(route.ID) { + r, err := k.Get(*route.ID) + if r != nil { + return nil + } + + if err != nil && !errors.Is(err, ErrNotFound) { + return err + } + } + + if !utils.Empty(route.Name) { + r, err := k.Get(*route.Name) + if r != nil { + return nil + } + + if err != nil && !errors.Is(err, ErrNotFound) { + return err + } + } + + return k.Add(route) +} + // Add adds a route into RoutesCollection // route.ID should not be nil else an error is thrown. func (k *RoutesCollection) Add(route Route) error { diff --git a/pkg/state/route_test.go b/pkg/state/route_test.go index 06fce69..78633e8 100644 --- a/pkg/state/route_test.go +++ b/pkg/state/route_test.go @@ -104,6 +104,19 @@ func TestRoutesCollection_Add(t *testing.T) { } } +func TestRouteInsertIgnoreDuplicate(t *testing.T) { + assert := assert.New(t) + collection := routesCollection() + + var r Route + r.ID = kong.String("my-id") + r.Name = kong.String("first") + err := collection.Add(r) + assert.Nil(err) + err = collection.AddIgnoringDuplicates(r) + assert.Nil(err) +} + func TestRoutesCollection_Get(t *testing.T) { type args struct { nameOrID string diff --git a/pkg/state/service.go b/pkg/state/service.go index 232060f..90d6ddc 100644 --- a/pkg/state/service.go +++ b/pkg/state/service.go @@ -33,6 +33,30 @@ var serviceTableSchema = &memdb.TableSchema{ // ServicesCollection stores and indexes Kong Services. type ServicesCollection collection +func (k *ServicesCollection) AddIgnoringDuplicates(service Service) error { + // Detect duplicates + if !utils.Empty(service.ID) { + s, err := k.Get(*service.ID) + if s != nil { + return nil + } + if err != nil && !errors.Is(err, ErrNotFound) { + return err + } + } + + if !utils.Empty(service.Name) { + s, err := k.Get(*service.Name) + if s != nil { + return nil + } + if err != nil && !errors.Is(err, ErrNotFound) { + return err + } + } + return k.Add(service) +} + // Add adds a service to the collection. // service.ID should not be nil else an error is thrown. func (k *ServicesCollection) Add(service Service) error { diff --git a/pkg/state/service_test.go b/pkg/state/service_test.go index fb05aa7..73e42c8 100644 --- a/pkg/state/service_test.go +++ b/pkg/state/service_test.go @@ -104,6 +104,19 @@ func TestServicesCollection_Add(t *testing.T) { } } +func TestServiceInsertIgnoreDuplicate(t *testing.T) { + assert := assert.New(t) + collection := servicesCollection() + + var s Service + s.ID = kong.String("my-id") + s.Name = kong.String("first") + err := collection.Add(s) + assert.Nil(err) + err = collection.AddIgnoringDuplicates(s) + assert.Nil(err) +} + func TestServicesCollection_Get(t *testing.T) { type args struct { nameOrID string