Add support for foreign key references in partial sync

The b.intermediate collection is populated when entities
are discovered in targetState, allowing for in-file foreign
key references.

The new `deck gateway apply` command allows you to sync
partial configuration, and so must populate b.intermediate
from the currentState.

This results in duplicate entity errors, and so an
AddIgnoringDuplicates method has been added to entities
that may be a foreign key

pkg/dump/dump.go

@@ -49,6 +49,8 @@ type Config struct {
 
 	// IsConsumerGroupScopedPluginSupported
 	IsConsumerGroupScopedPluginSupported bool
+
+	IsPartialApply bool
 }
 
 func deduplicate(stringSlice []string) []string {

pkg/file/builder.go

@@ -53,6 +53,8 @@ type stateBuilder struct {
 
 	removePathHandlingFromExpressionRoute bool
 
+	isPartialApply bool
+
 	err error
 }
 
@@ -167,6 +169,23 @@ func (b *stateBuilder) consumerGroups() {
 		return
 	}
 
+	// Load all existing consumer groups in to the immediate state for
+	// foreign key lookups if we're doing a partial apply
+	if b.isPartialApply {
+		consumerGroups, err := b.currentState.ConsumerGroups.GetAll()
+		if err != nil {
+			b.err = err
+			return
+		}
+		for _, cg := range consumerGroups {
+			err = b.intermediate.ConsumerGroups.Add(*cg)
+			if err != nil {
+				b.err = err
+				return
+			}
+		}
+	}
+
 	for _, cg := range b.targetContent.ConsumerGroups {
 		current, err := b.currentState.ConsumerGroups.Get(*cg.Name)
 		if utils.Empty(cg.ID) {
@@ -199,7 +218,7 @@ func (b *stateBuilder) consumerGroups() {
 			ConsumerGroup: &cg.ConsumerGroup,
 		}
 
-		err = b.intermediate.ConsumerGroups.Add(state.ConsumerGroup{ConsumerGroup: cg.ConsumerGroup})
+		err = b.intermediate.ConsumerGroups.AddIgnoringDuplicates(state.ConsumerGroup{ConsumerGroup: cg.ConsumerGroup})
 		if err != nil {
 			b.err = err
 			return
@@ -248,6 +267,23 @@ func (b *stateBuilder) certificates() {
 		return
 	}
 
+	// Load all existing certificates in to the immediate state for
+	// foreign key lookups if we're doing a partial apply
+	if b.isPartialApply {
+		certs, err := b.currentState.Certificates.GetAll()
+		if err != nil {
+			b.err = err
+			return
+		}
+		for _, c := range certs {
+			err = b.intermediate.Certificates.Add(*c)
+			if err != nil {
+				b.err = err
+				return
+			}
+		}
+	}
+
 	for i := range b.targetContent.Certificates {
 		c := b.targetContent.Certificates[i]
 		if utils.Empty(c.ID) {
@@ -314,6 +350,23 @@ func (b *stateBuilder) caCertificates() {
 		return
 	}
 
+	// Load all existing CA certificates in to the immediate state for
+	// foreign key lookups if we're doing a partial apply
+	if b.isPartialApply {
+		certs, err := b.currentState.CACertificates.GetAll()
+		if err != nil {
+			b.err = err
+			return
+		}
+		for _, c := range certs {
+			err = b.intermediate.CACertificates.Add(*c)
+			if err != nil {
+				b.err = err
+				return
+			}
+		}
+	}
+
 	for _, c := range b.targetContent.CACertificates {
 		cert, err := b.currentState.CACertificates.Get(*c.Cert)
 		if utils.Empty(c.ID) {
@@ -388,11 +441,11 @@ func (b *stateBuilder) ingestConsumerGroupConsumer(cgID *string, c *FConsumer) (
 	}
 
 	b.rawState.Consumers = append(b.rawState.Consumers, &c.Consumer)
-	err = b.intermediate.Consumers.Add(state.Consumer{Consumer: c.Consumer})
+	err = b.intermediate.Consumers.AddIgnoringDuplicates(state.Consumer{Consumer: c.Consumer})
 	if err != nil {
 		return nil, err
 	}
-	err = b.intermediate.ConsumerGroupConsumers.Add(state.ConsumerGroupConsumer{
+	err = b.intermediate.ConsumerGroupConsumers.AddIgnoringDuplicates(state.ConsumerGroupConsumer{
 		ConsumerGroupConsumer: kong.ConsumerGroupConsumer{
 			ConsumerGroup: &kong.ConsumerGroup{ID: cgID},
 			Consumer:      &c.Consumer,
@@ -409,6 +462,24 @@ func (b *stateBuilder) consumers() {
 		return
 	}
 
+	// Load all existing consumers in to the immediate state for
+	// foreign key lookups if we're doing a partial apply
+	if b.isPartialApply {
+		consumers, err := b.currentState.Consumers.GetAll()
+		if err != nil {
+			b.err = err
+			return
+		}
+
+		for _, c := range consumers {
+			err = b.intermediate.Consumers.Add(*c)
+			if err != nil {
+				b.err = err
+				return
+			}
+		}
+	}
+
 	for _, c := range b.targetContent.Consumers {
 
 		var (
@@ -473,7 +544,7 @@ func (b *stateBuilder) consumers() {
 		}
 		if !consumerAlreadyAdded {
 			b.rawState.Consumers = append(b.rawState.Consumers, &c.Consumer)
-			err = b.intermediate.Consumers.Add(state.Consumer{Consumer: c.Consumer})
+			err = b.intermediate.Consumers.AddIgnoringDuplicates(state.Consumer{Consumer: c.Consumer})
 			if err != nil {
 				b.err = err
 				return
@@ -865,6 +936,23 @@ func (b *stateBuilder) services() {
 		return
 	}
 
+	// Load all existing services in to the immediate state for
+	// foreign key lookups if we're doing a partial apply
+	if b.isPartialApply {
+		services, err := b.currentState.Services.GetAll()
+		if err != nil {
+			b.err = err
+			return
+		}
+		for _, s := range services {
+			err = b.intermediate.Services.Add(*s)
+			if err != nil {
+				b.err = err
+				return
+			}
+		}
+	}
+
 	for _, s := range b.targetContent.Services {
 		err := b.ingestService(&s)
 		if err != nil {
@@ -912,7 +1000,7 @@ func (b *stateBuilder) ingestService(s *FService) error {
 		s.Service.CreatedAt = svc.CreatedAt
 	}
 	b.rawState.Services = append(b.rawState.Services, &s.Service)
-	err = b.intermediate.Services.Add(state.Service{Service: s.Service})
+	err = b.intermediate.Services.AddIgnoringDuplicates(state.Service{Service: s.Service})
 	if err != nil {
 		return err
 	}
@@ -952,6 +1040,24 @@ func (b *stateBuilder) routes() {
 		return
 	}
 
+	// Load all existing routes in to the immediate state for
+	// foreign key lookups if we're doing a partial apply
+	if b.isPartialApply {
+		routes, err := b.currentState.Routes.GetAll()
+		if err != nil {
+			b.err = err
+			return
+		}
+
+		for _, r := range routes {
+			err = b.intermediate.Routes.Add(*r)
+			if err != nil {
+				b.err = err
+				return
+			}
+		}
+	}
+
 	for _, r := range b.targetContent.Routes {
 		if err := b.ingestRoute(r); err != nil {
 			b.err = err
@@ -1418,7 +1524,7 @@ func (b *stateBuilder) ingestRoute(r FRoute) error {
 	}
 
 	b.rawState.Routes = append(b.rawState.Routes, &r.Route)
-	err = b.intermediate.Routes.Add(state.Route{Route: r.Route})
+	err = b.intermediate.Routes.AddIgnoringDuplicates(state.Route{Route: r.Route})
 	if err != nil {
 		return err
 	}

pkg/file/reader.go

@@ -81,6 +81,7 @@ func Get(ctx context.Context, fileContent *Content, opt RenderConfig, dumpConfig
 	builder.skipCACerts = dumpConfig.SkipCACerts
 	builder.isKonnect = dumpConfig.KonnectControlPlane != ""
 	builder.includeLicenses = dumpConfig.IncludeLicenses
+	builder.isPartialApply = dumpConfig.IsPartialApply
 
 	if len(dumpConfig.SelectorTags) > 0 {
 		builder.selectTags = dumpConfig.SelectorTags

pkg/state/consumer.go

@@ -39,6 +39,44 @@ var consumerTableSchema = &memdb.TableSchema{
 // ConsumersCollection stores and indexes Kong Consumers.
 type ConsumersCollection collection
 
+func (k *ConsumersCollection) AddIgnoringDuplicates(consumer Consumer) error {
+	// Detect duplicates
+	if !utils.Empty(consumer.ID) {
+		c, err := k.GetByIDOrUsername(*consumer.ID)
+		if c != nil {
+			return nil
+		}
+
+		if err != nil && !errors.Is(err, ErrNotFound) {
+			return err
+		}
+	}
+
+	if !utils.Empty(consumer.Username) {
+		c, err := k.GetByIDOrUsername(*consumer.Username)
+		if c != nil {
+			return nil
+		}
+
+		if err != nil && !errors.Is(err, ErrNotFound) {
+			return err
+		}
+	}
+
+	// Check for custom ID
+	if !utils.Empty(consumer.CustomID) {
+		c, err := k.GetByCustomID(*consumer.CustomID)
+		if c != nil {
+			return nil
+		}
+		if err != nil && !errors.Is(err, ErrNotFound) {
+			return err
+		}
+	}
+
+	return k.Add(consumer)
+}
+
 // Add adds a consumer to the collection
 // An error is thrown if consumer.ID is empty.
 func (k *ConsumersCollection) Add(consumer Consumer) error {

pkg/state/consumer_group.go

@@ -32,6 +32,31 @@ var consumerGroupTableSchema = &memdb.TableSchema{
 // consumerGroupsCollection stores and indexes Kong consumerGroups.
 type ConsumerGroupsCollection collection
 
+func (k *ConsumerGroupsCollection) AddIgnoringDuplicates(consumerGroup ConsumerGroup) error {
+	// Detect duplicates
+	if !utils.Empty(consumerGroup.ID) {
+		cg, err := k.Get(*consumerGroup.ID)
+		if cg != nil {
+			return nil
+		}
+		if err != nil && !errors.Is(err, ErrNotFound) {
+			return err
+		}
+	}
+
+	if !utils.Empty(consumerGroup.Name) {
+		cg, err := k.Get(*consumerGroup.Name)
+		if cg != nil {
+			return nil
+		}
+		if err != nil && !errors.Is(err, ErrNotFound) {
+			return err
+		}
+	}
+
+	return k.Add(consumerGroup)
+}
+
 // Add adds an consumerGroup to the collection.
 // consumerGroup.ID should not be nil else an error is thrown.
 func (k *ConsumerGroupsCollection) Add(consumerGroup ConsumerGroup) error {

pkg/state/consumer_group_consumers.go

@@ -110,6 +110,31 @@ func validateConsumerGroup(consumer *ConsumerGroupConsumer) error {
 // ConsumerGroupConsumersCollection stores and indexes Kong consumerGroupConsumers.
 type ConsumerGroupConsumersCollection collection
 
+func (k *ConsumerGroupConsumersCollection) AddIgnoringDuplicates(consumer ConsumerGroupConsumer) error {
+	// Detect duplicates
+	if !utils.Empty(consumer.Consumer.ID) {
+		cgc, err := k.Get(*consumer.Consumer.ID, *consumer.ConsumerGroup.ID)
+		if cgc != nil {
+			return nil
+		}
+		if err != nil && !errors.Is(err, ErrNotFound) {
+			return err
+		}
+	}
+
+	if !utils.Empty(consumer.Consumer.Username) {
+		cgc, err := k.Get(*consumer.Consumer.Username, *consumer.ConsumerGroup.ID)
+		if cgc != nil {
+			return nil
+		}
+		if err != nil && !errors.Is(err, ErrNotFound) {
+			return err
+		}
+	}
+
+	return k.Add(consumer)
+}
+
 // Add adds a consumerGroupConsumer to the collection.
 func (k *ConsumerGroupConsumersCollection) Add(consumer ConsumerGroupConsumer) error {
 	if utils.Empty(consumer.Consumer.ID) {

pkg/state/consumer_group_test.go

@@ -0,0 +1,41 @@
+package state
+
+import (
+	"testing"
+
+	"github.com/kong/go-kong/kong"
+	"github.com/stretchr/testify/assert"
+)
+
+func consumerGroupsCollection() *ConsumerGroupsCollection {
+	return state().ConsumerGroups
+}
+
+func TestConsumerGroupInsert(t *testing.T) {
+	assert := assert.New(t)
+	collection := consumerGroupsCollection()
+
+	var cg ConsumerGroup
+
+	assert.NotNil(collection.Add(cg))
+
+	cg.ID = kong.String("my-id")
+	cg.Name = kong.String("first")
+	assert.Nil(collection.Add(cg))
+
+	// re-insert
+	assert.NotNil(collection.Add(cg))
+}
+
+func TestConsumerGroupInsertIgnoreDuplicate(t *testing.T) {
+	assert := assert.New(t)
+	collection := consumerGroupsCollection()
+
+	var cg ConsumerGroup
+	cg.ID = kong.String("my-id")
+	cg.Name = kong.String("first")
+	err := collection.Add(cg)
+	assert.Nil(err)
+	err = collection.AddIgnoringDuplicates(cg)
+	assert.Nil(err)
+}