Merge branch 'main' into feat/watch-kubernetes-events

.dockerignore

@@ -1,11 +1,17 @@
-Dockerfile
 .bin/
-.vscode/
-.idea/
-.env
+.config-db/
 .DS_Store
+.env
+.github/
+.idea/
+.releaserc
+.vscode/
+build/
+chart/
+CONTRIBUTING.md
 cover.out
+Dockerfile
+PROJECT
+README.md
+SECURITY.md
 test.test
-build/
-.config-db
-.github

.github/workflows/release.yml

@@ -60,6 +60,7 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           snapshot: true
+          dockerfile: build/Dockerfile
           tags: "latest,v${{ needs.semantic-release.outputs.release-version }}"
   helm:
     runs-on: ubuntu-latest

Makefile

@@ -20,7 +20,7 @@ GOBIN=$(shell go env GOBIN)
 endif
 
 docker:
-	docker build . -t ${IMG}
+	docker build . -f build/Dockerfile -t ${IMG}
 
 # Push the docker image
 docker-push:

README.md

@@ -13,7 +13,7 @@ make build
 Starting the server will run the migrations and start scraping in background (The `default-schedule` configuration will run scraping every 60 minutes if configuration is not explicitly specified).
 
 ```bash
-DB_URL=postgres://<username>:<password>@localhost:5432/<db_name> ./.bin/config-db serve --run-migrations
+DB_URL=postgres://<username>:<password>@localhost:5432/<db_name> ./.bin/config-db serve --db-migrations
 ```
 
 ### Scape config

api/context.go

@@ -7,6 +7,7 @@ import (
 
 	v1 "github.com/flanksource/config-db/api/v1"
 	"github.com/flanksource/duty"
+	dutyCtx "github.com/flanksource/duty/context"
 	"github.com/flanksource/duty/models"
 	"github.com/flanksource/duty/types"
 	"github.com/jackc/pgx/v5/pgxpool"
@@ -17,6 +18,7 @@ import (
 
 type ScrapeContext interface {
 	duty.DBContext
+	DutyContext() dutyCtx.Context
 
 	IsTrace() bool
 
@@ -33,6 +35,7 @@ type ScrapeContext interface {
 	GetEnvValueFromCache(env types.EnvVar) (string, error)
 
 	HydrateConnection(connectionIdentifier string) (*models.Connection, error)
+	HydrateConnectionModel(models.Connection) (*models.Connection, error)
 }
 
 type scrapeContext struct {
@@ -69,6 +72,13 @@ func (ctx scrapeContext) WithScrapeConfig(scraper *v1.ScrapeConfig) ScrapeContex
 	return &ctx
 }
 
+func (ctx scrapeContext) DutyContext() dutyCtx.Context {
+	return dutyCtx.NewContext(ctx).
+		WithKubernetes(ctx.kubernetes).
+		WithDB(ctx.db, ctx.pool).
+		WithNamespace(ctx.namespace)
+}
+
 func (ctx scrapeContext) DB() *gorm.DB {
 	return ctx.db
 }
@@ -106,11 +116,15 @@ func (ctx *scrapeContext) HydrateConnection(connectionName string) (*models.Conn
 		return nil, errors.New("db has not been initialized")
 	}
 
+	if ctx.pool == nil {
+		return nil, errors.New("pool has not been initialized")
+	}
+
 	if ctx.kubernetes == nil {
 		return nil, errors.New("kubernetes clientset has not been initialized")
 	}
 
-	connection, err := duty.HydratedConnectionByURL(ctx, ctx.db, ctx.kubernetes, ctx.namespace, connectionName)
+	connection, err := dutyCtx.HydrateConnectionByURL(ctx.DutyContext(), connectionName)
 	if err != nil {
 		return nil, err
 	}
@@ -124,6 +138,33 @@ func (ctx *scrapeContext) HydrateConnection(connectionName string) (*models.Conn
 	return connection, nil
 }
 
+func (ctx *scrapeContext) HydrateConnectionModel(c models.Connection) (*models.Connection, error) {
+	if ctx.db == nil {
+		return nil, errors.New("db has not been initialized")
+	}
+
+	if ctx.pool == nil {
+		return nil, errors.New("pool has not been initialized")
+	}
+
+	if ctx.kubernetes == nil {
+		return nil, errors.New("kubernetes clientset has not been initialized")
+	}
+
+	connection, err := dutyCtx.HydrateConnection(ctx.DutyContext(), &c)
+	if err != nil {
+		return nil, err
+	}
+
+	// Connection name was explicitly provided but was not found.
+	// That's an error.
+	if connection == nil {
+		return nil, fmt.Errorf("connection %s not found", c.Name)
+	}
+
+	return connection, nil
+}
+
 func (c *scrapeContext) GetEnvVarValue(input types.EnvVar) (string, error) {
 	return duty.GetEnvValueFromCache(c.kubernetes, input, c.namespace)
 }

api/v1/azure.go

@@ -15,10 +15,19 @@ type AzureDevops struct {
 
 type Azure struct {
 	BaseScraper    `json:",inline"`
-	ConnectionName string       `yaml:"connection,omitempty" json:"connection,omitempty"`
-	SubscriptionID string       `yaml:"subscriptionID" json:"subscriptionID"`
-	Organisation   string       `yaml:"organisation" json:"organisation"`
-	ClientID       types.EnvVar `yaml:"clientID,omitempty" json:"clientID,omitempty"`
-	ClientSecret   types.EnvVar `yaml:"clientSecret,omitempty" json:"clientSecret,omitempty"`
-	TenantID       string       `yaml:"tenantID" json:"tenantID"`
+	ConnectionName string           `yaml:"connection,omitempty" json:"connection,omitempty"`
+	SubscriptionID string           `yaml:"subscriptionID" json:"subscriptionID"`
+	Organisation   string           `yaml:"organisation" json:"organisation"`
+	ClientID       types.EnvVar     `yaml:"clientID,omitempty" json:"clientID,omitempty"`
+	ClientSecret   types.EnvVar     `yaml:"clientSecret,omitempty" json:"clientSecret,omitempty"`
+	TenantID       string           `yaml:"tenantID" json:"tenantID"`
+	Exclusions     *AzureExclusions `yaml:"exclusions,omitempty" json:"exclusions,omitempty"`
+}
+
+type AzureExclusions struct {
+	// ActivityLogs is a list of operations to exclude from activity logs.
+	// Example:
+	//  "Microsoft.ContainerService/managedClusters/listClusterAdminCredential/action"
+	//  "Microsoft.ContainerService/managedClusters/listClusterUserCredential/action"
+	ActivityLogs []string `yaml:"activityLogs,omitempty" json:"activityLogs,omitempty"`
 }

api/v1/common.go

@@ -95,6 +95,11 @@ func (s MaskList) String() string {
 	return fmt.Sprintf("total_masks=%d", len(s))
 }
 
+type TransformChange struct {
+	// Exclude is a list of CEL expressions that excludes a given change
+	Exclude []string `json:"exclude,omitempty"`
+}
+
 type Transform struct {
 	Script  Script   `yaml:",inline" json:",inline"`
 	Include []Filter `json:"include,omitempty"`
@@ -103,7 +108,8 @@ type Transform struct {
 	Exclude []Filter `json:"exclude,omitempty"`
 	// Masks consist of configurations to replace sensitive fields
 	// with hash functions or static string.
-	Masks MaskList `json:"mask,omitempty"`
+	Masks  MaskList        `json:"mask,omitempty"`
+	Change TransformChange `json:"changes,omitempty"`
 }
 
 func (t Transform) IsEmpty() bool {
@@ -155,8 +161,14 @@ type BaseScraper struct {
 
 	// DeleteFields is a JSONPath expression used to identify the deleted time of the config.
 	// If multiple fields are specified, the first non-empty value will be used.
-	DeleteFields []string      `json:"deleteFields,omitempty"`
-	Tags         JSONStringMap `json:"tags,omitempty"`
+	DeleteFields []string `json:"deleteFields,omitempty"`
+
+	// Tags allow you to set custom tags on the scraped config items.
+	Tags JSONStringMap `json:"tags,omitempty"`
+
+	// Properties are custom templatable properties for the scraped config items
+	// grouped by the config type.
+	Properties map[string]types.Properties `json:"properties,omitempty" template:"true"`
 }
 
 func (base BaseScraper) String() string {

api/v1/interface.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/flanksource/commons/logger"
 	"github.com/flanksource/duty/models"
+	"github.com/flanksource/duty/types"
 )
 
 // Analyzer ...
@@ -78,6 +79,9 @@ type ChangeResult struct {
 	CreatedBy        *string                `json:"created_by"`
 	CreatedAt        *time.Time             `json:"created_at"`
 	Details          map[string]interface{} `json:"details"`
+
+	// UpdateExisting indicates whether to update an existing change
+	UpdateExisting bool `json:"update_existing"`
 }
 
 func (r ChangeResult) AsMap() map[string]any {
@@ -155,9 +159,13 @@ func (t *ScrapeResults) Add(r ...ScrapeResult) {
 }
 
 type RelationshipResult struct {
-	ConfigExternalID  ExternalID
+	ConfigExternalID ExternalID
+	// Related External ID to lookup the actual config item ID.
+	// Used when the config id is not known.
 	RelatedExternalID ExternalID
-	Relationship      string
+	// Config ID of the related config.
+	RelatedConfigID string
+	Relationship    string
 }
 
 type RelationshipResults []RelationshipResult
@@ -190,16 +198,24 @@ func (s *ScrapeResults) Errorf(e error, msg string, args ...interface{}) ScrapeR
 // ScrapeResult ...
 // +kubebuilder:object:generate=false
 type ScrapeResult struct {
+	// ID is the id of the config at it's origin (i.e. the external id)
+	// Eg: For Azure, it's the azure resource id and for kuberetenes it's the object UID.
+	// If it's a valid UUID & ConfigID is nil, it'll be used as the primary key id of the config item in the database.
+	ID string `json:"id,omitempty"`
+
+	// Config ID is used as the primary key id of the config item in the database.
+	ConfigID *string `json:"-"`
+
 	CreatedAt           *time.Time          `json:"created_at,omitempty"`
 	DeletedAt           *time.Time          `json:"deleted_at,omitempty"`
+	DeleteReason        ConfigDeleteReason  `json:"delete_reason,omitempty"`
 	LastModified        time.Time           `json:"last_modified,omitempty"`
 	ConfigClass         string              `json:"config_class,omitempty"`
 	Type                string              `json:"config_type,omitempty"`
 	Status              string              `json:"status,omitempty"` // status extracted from the config itself
 	Name                string              `json:"name,omitempty"`
 	Namespace           string              `json:"namespace,omitempty"`
 	Description         string              `json:"description,omitempty"`
-	ID                  string              `json:"id,omitempty"`
 	Aliases             []string            `json:"aliases,omitempty"`
 	Source              string              `json:"source,omitempty"`
 	Config              interface{}         `json:"config,omitempty"`
@@ -215,6 +231,23 @@ type ScrapeResult struct {
 	Action              string              `json:",omitempty"`
 	ParentExternalID    string              `json:"-"`
 	ParentType          string              `json:"-"`
+	Properties          types.Properties    `json:"properties,omitempty"`
+}
+
+func (s ScrapeResult) AsMap() map[string]any {
+	output := make(map[string]any)
+
+	b, err := json.Marshal(s)
+	if err != nil {
+		logger.Errorf("failed to marshal change result: %v", err)
+		return output
+	}
+
+	if err := json.Unmarshal(b, &output); err != nil {
+		logger.Errorf("failed to unmarshal change result: %v", err)
+	}
+
+	return output
 }
 
 func NewScrapeResult(base BaseScraper) *ScrapeResult {

api/v1/kubernetes.go

@@ -2,11 +2,13 @@ package v1
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"strings"
 
 	"github.com/flanksource/commons/collections"
 	"github.com/flanksource/duty/types"
+	"github.com/flanksource/gomplate/v3"
 	"github.com/flanksource/mapstructure"
 	coreV1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -103,6 +105,49 @@ func (t *KubernetesConfigExclusions) Filter(name, namespace, kind string, labels
 	return false
 }
 
+type KubernetesRelationshipLookup struct {
+	Expr  string `json:"expr,omitempty"`
+	Value string `json:"value,omitempty"`
+	Label string `json:"label,omitempty"`
+}
+
+func (t *KubernetesRelationshipLookup) Eval(labels map[string]string, envVar map[string]any) (string, error) {
+	if t.Value != "" {
+		return t.Value, nil
+	}
+
+	if t.Label != "" {
+		return labels[t.Label], nil
+	}
+
+	if t.Expr != "" {
+		res, err := gomplate.RunTemplate(envVar, gomplate.Template{Expression: t.Expr})
+		if err != nil {
+			return "", err
+		}
+
+		return res, nil
+	}
+
+	return "", errors.New("unknown kubernetes relationship lookup type")
+}
+
+func (t KubernetesRelationshipLookup) IsEmpty() bool {
+	if t.Value == "" && t.Label == "" && t.Expr == "" {
+		return true
+	}
+	return false
+}
+
+type KubernetesRelationship struct {
+	// Kind defines which field to use for the kind lookup
+	Kind KubernetesRelationshipLookup `json:"kind" yaml:"kind"`
+	// Name defines which field to use for the name lookup
+	Name KubernetesRelationshipLookup `json:"name" yaml:"name"`
+	// Namespace defines which field to use for the namespace lookup
+	Namespace KubernetesRelationshipLookup `json:"namespace" yaml:"namespace"`
+}
+
 type Kubernetes struct {
 	BaseScraper     `json:",inline"`
 	ClusterName     string                     `json:"clusterName,omitempty"`
@@ -113,10 +158,13 @@ type Kubernetes struct {
 	Since           string                     `json:"since,omitempty"`
 	Selector        string                     `json:"selector,omitempty"`
 	FieldSelector   string                     `json:"fieldSelector,omitempty"`
-	MaxInflight     int64                      `json:"maxInflight,omitempty"`
 	Kubeconfig      *types.EnvVar              `json:"kubeconfig,omitempty"`
+	MaxInflight     int64                      `json:"maxInflight,omitempty"`
 	Event           KubernetesEventConfig      `json:"event,omitempty"`
 	Exclusions      KubernetesConfigExclusions `json:"exclusions,omitempty"`
+
+	// Relationships specify the fields to use to relate Kubernetes objects.
+	Relationships []KubernetesRelationship `json:"relationships,omitempty"`
 }
 
 type KubernetesFile struct {