From 32e8c9575281fd6244b9371c96805013ef784af1 Mon Sep 17 00:00:00 2001 From: Aditya Thebe Date: Mon, 10 Jun 2024 16:09:24 +0545 Subject: [PATCH] feat: multiple candidate parents for a single config --- api/v1/interface.go | 10 +- db/config.go | 4 +- db/models/config_item.go | 3 +- db/update.go | 37 ++-- scrapers/aws/aws.go | 272 ++++++++++++++---------------- scrapers/azure/azure.go | 20 ++- scrapers/kubernetes/kubernetes.go | 68 ++++---- 7 files changed, 205 insertions(+), 209 deletions(-) diff --git a/api/v1/interface.go b/api/v1/interface.go index fe635d0c..ddf5f717 100644 --- a/api/v1/interface.go +++ b/api/v1/interface.go @@ -347,6 +347,11 @@ func (t Tags) Eval(labels map[string]string, config string) (Tags, error) { return t, nil } +type ConfigExternalKey struct { + ExternalID string + Type string +} + // ScrapeResult ... // +kubebuilder:object:generate=false type ScrapeResult struct { @@ -384,11 +389,12 @@ type ScrapeResult struct { RelationshipResults RelationshipResults `json:"-"` Ignore []string `json:"-"` Action string `json:",omitempty"` - ParentExternalID string `json:"-"` - ParentType string `json:"-"` Properties types.Properties `json:"properties,omitempty"` LastScrapedTime *time.Time `json:"last_scraped_time"` + // List of candidate parents in order of precision. + Parents []ConfigExternalKey `json:"-"` + // RelationshipSelectors are used to form relationship of this scraped item with other items. // Unlike `RelationshipResults`, selectors give you the flexibility to form relationship without // knowing the external ids of the item to be linked. diff --git a/db/config.go b/db/config.go index 6b10f3dd..fbc0e69c 100644 --- a/db/config.go +++ b/db/config.go @@ -138,9 +138,7 @@ func NewConfigItemFromResult(ctx api.ScrapeContext, result v1.ScrapeResult) (*mo Config: &dataStr, Ready: result.Ready, LastScrapedTime: result.LastScrapedTime, - - ParentExternalID: result.ParentExternalID, - ParentType: result.ParentType, + Parents: result.Parents, } if parsed, err := result.Tags.AsMap(); err != nil { diff --git a/db/models/config_item.go b/db/models/config_item.go index a0d7a765..8fc69c7e 100644 --- a/db/models/config_item.go +++ b/db/models/config_item.go @@ -43,8 +43,7 @@ type ConfigItem struct { LastScrapedTime *time.Time `gorm:"column:last_scraped_time" json:"last_scraped_time"` DeleteReason v1.ConfigDeleteReason `gorm:"column:delete_reason" json:"delete_reason"` - ParentExternalID string `gorm:"-" json:"-"` - ParentType string `gorm:"-" json:"-"` + Parents []v1.ConfigExternalKey `gorm:"-" json:"parents,omitempty"` } func (ci ConfigItem) String() string { diff --git a/db/update.go b/db/update.go index 39a2140c..208d9b83 100644 --- a/db/update.go +++ b/db/update.go @@ -721,24 +721,33 @@ func setConfigParents(ctx api.ScrapeContext, parentTypeToConfigMap map[configExt continue // existing item. Parent is already set. } - if ci.ParentExternalID == "" || ci.ParentType == "" { - continue + if len(ci.Parents) == 0 { + continue // these are root items. } - if parentID, found := parentTypeToConfigMap[configExternalKey{ - externalID: ci.ParentExternalID, - parentType: ci.ParentType, - }]; found { - ci.ParentID = &parentID - continue + for _, parent := range ci.Parents { + if parent.ExternalID == "" || parent.Type == "" { + continue + } + + if parentID, found := parentTypeToConfigMap[configExternalKey{ + externalID: parent.ExternalID, + parentType: parent.Type, + }]; found { + ci.ParentID = &parentID + break + } + + if found, err := ctx.TempCache().Find(parent.Type, parent.ExternalID); err != nil { + return err + } else if found != nil { + ci.ParentID = &found.ID + break + } } - if found, err := ctx.TempCache().Find(ci.ParentType, ci.ParentExternalID); err != nil { - return err - } else if found != nil { - ci.ParentID = &found.ID - } else { - ctx.Logger.V(0).Infof("[%s] parent %s/%s not found", ci, ci.ParentType, ci.ParentExternalID) + if ci.ParentID == nil { + ctx.Logger.V(0).Infof("parent not found for config [%s]", ci) } } diff --git a/scrapers/aws/aws.go b/scrapers/aws/aws.go index 585adf0b..5e5e294e 100644 --- a/scrapers/aws/aws.go +++ b/scrapers/aws/aws.go @@ -62,7 +62,7 @@ func getLabels(tags []ec2Types.Tag) v1.JSONStringMap { } func (ctx AWSContext) String() string { - return fmt.Sprintf("account=%s user=%s region=%s", *ctx.Caller.Account, *ctx.Caller.UserId, ctx.Session.Region) + return fmt.Sprintf("account=%s user=%s region=%s", lo.FromPtr(ctx.Caller.Account), *ctx.Caller.UserId, ctx.Session.Region) } func (aws Scraper) getContext(ctx api.ScrapeContext, awsConfig v1.AWS, region string) (*AWSContext, error) { @@ -136,8 +136,7 @@ func (aws Scraper) containerImages(ctx *AWSContext, config v1.AWS, results *v1.S Ignore: []string{ "CreatedAt", "RepositoryArn", "RepositoryUri", "RegistryId", "RepositoryName", }, - ParentExternalID: *ctx.Caller.Account, - ParentType: v1.AWSAccount, + Parents: []v1.ConfigExternalKey{{Type: v1.AWSAccount, ExternalID: lo.FromPtr(ctx.Caller.Account)}}, }) } } @@ -191,7 +190,7 @@ func (aws Scraper) eksClusters(ctx *AWSContext, config v1.AWS, results *v1.Scrap resourceHealth := health.GetAWSResourceHealth(health.AWSResourceTypeEKS, string(cluster.Cluster.Status)) - cluster.Cluster.Tags["account"] = *ctx.Caller.Account + cluster.Cluster.Tags["account"] = lo.FromPtr(ctx.Caller.Account) cluster.Cluster.Tags["region"] = getRegionFromArn(*cluster.Cluster.Arn, "eks") *results = append(*results, v1.ScrapeResult{ @@ -206,8 +205,7 @@ func (aws Scraper) eksClusters(ctx *AWSContext, config v1.AWS, results *v1.Scrap Aliases: []string{*cluster.Cluster.Arn, "AmazonEKS/" + *cluster.Cluster.Arn}, ID: *cluster.Cluster.Name, Ignore: []string{"createdAt", "name"}, - ParentExternalID: *cluster.Cluster.ResourcesVpcConfig.VpcId, - ParentType: v1.AWSEC2VPC, + Parents: []v1.ConfigExternalKey{{Type: v1.AWSEC2VPC, ExternalID: *cluster.Cluster.ResourcesVpcConfig.VpcId}}, RelationshipResults: relationships, }.WithHealthStatus(resourceHealth)) } @@ -257,30 +255,28 @@ func (aws Scraper) availabilityZones(ctx *AWSContext, config v1.AWS, results *v1 var uniqueAvailabilityZoneIDs = map[string]struct{}{} for _, az := range azDescribeOutput.AvailabilityZones { *results = append(*results, v1.ScrapeResult{ - ID: lo.FromPtr(az.ZoneName), - Type: v1.AWSAvailabilityZone, - BaseScraper: config.BaseScraper, - Config: az, - ConfigClass: "AvailabilityZone", - Tags: []v1.Tag{{Name: "region", Value: lo.FromPtr(az.RegionName)}}, - Aliases: nil, - Name: lo.FromPtr(az.ZoneName), - ParentExternalID: lo.FromPtr(az.RegionName), - ParentType: v1.AWSRegion, + ID: lo.FromPtr(az.ZoneName), + Type: v1.AWSAvailabilityZone, + BaseScraper: config.BaseScraper, + Config: az, + ConfigClass: "AvailabilityZone", + Tags: []v1.Tag{{Name: "region", Value: lo.FromPtr(az.RegionName)}}, + Aliases: nil, + Name: lo.FromPtr(az.ZoneName), + Parents: []v1.ConfigExternalKey{{Type: v1.AWSRegion, ExternalID: lo.FromPtr(az.RegionName)}}, }) if _, ok := uniqueAvailabilityZoneIDs[lo.FromPtr(az.ZoneId)]; !ok { *results = append(*results, v1.ScrapeResult{ - ID: lo.FromPtr(az.ZoneId), - Type: v1.AWSAvailabilityZoneID, - Tags: []v1.Tag{{Name: "region", Value: lo.FromPtr(az.RegionName)}}, - BaseScraper: config.BaseScraper, - Config: map[string]string{"RegionName": *az.RegionName}, - ConfigClass: "AvailabilityZone", - Aliases: nil, - Name: lo.FromPtr(az.ZoneId), - ParentExternalID: lo.FromPtr(az.RegionName), - ParentType: v1.AWSRegion, + ID: lo.FromPtr(az.ZoneId), + Type: v1.AWSAvailabilityZoneID, + Tags: []v1.Tag{{Name: "region", Value: lo.FromPtr(az.RegionName)}}, + BaseScraper: config.BaseScraper, + Config: map[string]string{"RegionName": *az.RegionName}, + ConfigClass: "AvailabilityZone", + Aliases: nil, + Name: lo.FromPtr(az.ZoneId), + Parents: []v1.ConfigExternalKey{{Type: v1.AWSRegion, ExternalID: lo.FromPtr(az.RegionName)}}, }) uniqueAvailabilityZoneIDs[lo.FromPtr(az.ZoneId)] = struct{}{} @@ -305,7 +301,7 @@ func (aws Scraper) account(ctx *AWSContext, config v1.AWS, results *v1.ScrapeRes return } - name := *ctx.Caller.Account + name := lo.FromPtr(ctx.Caller.Account) if len(aliases.AccountAliases) > 0 { name = (*aliases).AccountAliases[0] } @@ -320,21 +316,20 @@ func (aws Scraper) account(ctx *AWSContext, config v1.AWS, results *v1.ScrapeRes Name: name, Labels: labels, Aliases: aliases.AccountAliases, - ID: *ctx.Caller.Account, + ID: lo.FromPtr(ctx.Caller.Account), }) *results = append(*results, v1.ScrapeResult{ - Type: v1.AWSIAMUser, - BaseScraper: config.BaseScraper, - Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSIAMUser, "root")}, - Config: summary.SummaryMap, - Labels: labels, - ConfigClass: "User", - Name: "root", - Aliases: []string{""}, - ID: "root", - ParentExternalID: lo.FromPtr(ctx.Caller.Account), - ParentType: v1.AWSAccount, + Type: v1.AWSIAMUser, + BaseScraper: config.BaseScraper, + Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSIAMUser, "root")}, + Config: summary.SummaryMap, + Labels: labels, + ConfigClass: "User", + Name: "root", + Aliases: []string{""}, + ID: "root", + Parents: []v1.ConfigExternalKey{{Type: v1.AWSAccount, ExternalID: lo.FromPtr(ctx.Caller.Account)}}, }) regions, err := ctx.EC2.DescribeRegions(ctx, &ec2.DescribeRegionsInput{}) @@ -373,19 +368,18 @@ func (aws Scraper) users(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResul labels := make(map[string]string) for _, user := range users.Users { *results = append(*results, v1.ScrapeResult{ - Type: v1.AWSIAMUser, - CreatedAt: user.CreateDate, - BaseScraper: config.BaseScraper, - Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSIAMUser, lo.FromPtr(user.UserName))}, - Config: user, - ConfigClass: "User", - Labels: labels, - Name: *user.UserName, - Aliases: []string{*user.UserId, *user.Arn}, - Ignore: []string{"arn", "userId", "createDate", "userName"}, - ID: *user.UserName, // UserId is not often referenced - ParentExternalID: lo.FromPtr(ctx.Caller.Account), - ParentType: v1.AWSAccount, + Type: v1.AWSIAMUser, + CreatedAt: user.CreateDate, + BaseScraper: config.BaseScraper, + Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSIAMUser, lo.FromPtr(user.UserName))}, + Config: user, + ConfigClass: "User", + Labels: labels, + Name: *user.UserName, + Aliases: []string{*user.UserId, *user.Arn}, + Ignore: []string{"arn", "userId", "createDate", "userName"}, + ID: *user.UserName, // UserId is not often referenced + Parents: []v1.ConfigExternalKey{{Type: v1.AWSAccount, ExternalID: lo.FromPtr(ctx.Caller.Account)}}, }) } } @@ -412,18 +406,17 @@ func (aws Scraper) ebs(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults resourceHealth := health.GetAWSResourceHealth(health.AWSResourceTypeEBS, string(volume.State)) *results = append(*results, v1.ScrapeResult{ - Type: v1.AWSEBSVolume, - Labels: labels, - Tags: tags, - BaseScraper: config.BaseScraper, - Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSEBSVolume, lo.FromPtr(volume.VolumeId))}, - Config: volume, - ConfigClass: "DiskStorage", - Aliases: []string{"AmazonEC2/" + *volume.VolumeId}, - Name: getName(labels, *volume.VolumeId), - ID: *volume.VolumeId, - ParentExternalID: lo.FromPtr(ctx.Caller.Account), - ParentType: v1.AWSAccount, + Type: v1.AWSEBSVolume, + Labels: labels, + Tags: tags, + BaseScraper: config.BaseScraper, + Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSEBSVolume, lo.FromPtr(volume.VolumeId))}, + Config: volume, + ConfigClass: "DiskStorage", + Aliases: []string{"AmazonEC2/" + *volume.VolumeId}, + Name: getName(labels, *volume.VolumeId), + ID: *volume.VolumeId, + Parents: []v1.ConfigExternalKey{{Type: v1.AWSAccount, ExternalID: lo.FromPtr(ctx.Caller.Account)}}, }.WithHealthStatus(resourceHealth)) } } @@ -476,8 +469,7 @@ func (aws Scraper) rds(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults Name: getName(labels, *instance.DBInstanceIdentifier), ID: *instance.DBInstanceIdentifier, Aliases: []string{"AmazonRDS/" + *instance.DBInstanceArn}, - ParentExternalID: *instance.DBSubnetGroup.VpcId, - ParentType: v1.AWSEC2VPC, + Parents: []v1.ConfigExternalKey{{Type: v1.AWSEC2VPC, ExternalID: lo.FromPtr(instance.DBSubnetGroup.VpcId)}}, RelationshipResults: relationships, }.WithHealthStatus(resourceHealth)) } @@ -531,8 +523,7 @@ func (aws Scraper) vpcs(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResult Name: getName(labels, *vpc.VpcId), ID: *vpc.VpcId, Aliases: []string{"AmazonEC2/" + *vpc.VpcId}, - ParentExternalID: lo.FromPtr(ctx.Caller.Account), - ParentType: v1.AWSAccount, + Parents: []v1.ConfigExternalKey{{Type: v1.AWSAccount, ExternalID: lo.FromPtr(ctx.Caller.Account)}}, RelationshipResults: relationships, }.WithHealthStatus(resourceHealth)) } @@ -648,8 +639,7 @@ func (aws Scraper) instances(ctx *AWSContext, config v1.AWS, results *v1.ScrapeR Name: instance.GetHostname(), Aliases: []string{"AmazonEC2/" + instance.InstanceID}, ID: instance.InstanceID, - ParentExternalID: instance.VpcID, - ParentType: v1.AWSEC2VPC, + Parents: []v1.ConfigExternalKey{{Type: v1.AWSEC2VPC, ExternalID: instance.VpcID}}, RelationshipResults: relationships, }.WithHealthStatus(resourceHealth)) } @@ -672,16 +662,15 @@ func (aws Scraper) securityGroups(ctx *AWSContext, config v1.AWS, results *v1.Sc labels := getLabels(sg.Tags) labels["network"] = *sg.VpcId *results = append(*results, v1.ScrapeResult{ - Type: v1.AWSEC2SecurityGroup, - Labels: labels, - BaseScraper: config.BaseScraper, - Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSEC2SecurityGroup, lo.FromPtr(sg.GroupId))}, - Config: sg, - ConfigClass: "SecurityGroup", - Name: getName(labels, *sg.GroupId), - ID: *sg.GroupId, - ParentExternalID: *sg.VpcId, - ParentType: v1.AWSEC2VPC, + Type: v1.AWSEC2SecurityGroup, + Labels: labels, + BaseScraper: config.BaseScraper, + Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSEC2SecurityGroup, lo.FromPtr(sg.GroupId))}, + Config: sg, + ConfigClass: "SecurityGroup", + Name: getName(labels, *sg.GroupId), + ID: *sg.GroupId, + Parents: []v1.ConfigExternalKey{{Type: v1.AWSEC2VPC, ExternalID: lo.FromPtr(sg.VpcId)}}, }) } } @@ -700,16 +689,15 @@ func (aws Scraper) routes(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResu labels := getLabels(r.Tags) labels["network"] = *r.VpcId *results = append(*results, v1.ScrapeResult{ - Type: "AWS::EC2::RouteTable", - Labels: labels, - BaseScraper: config.BaseScraper, - Properties: []*types.Property{getConsoleLink(ctx.Session.Region, "AWS::EC2::RouteTable", lo.FromPtr(r.RouteTableId))}, - Config: r, - ConfigClass: "Route", - Name: getName(labels, *r.RouteTableId), - ID: *r.RouteTableId, - ParentExternalID: *r.VpcId, - ParentType: v1.AWSEC2VPC, + Type: "AWS::EC2::RouteTable", + Labels: labels, + BaseScraper: config.BaseScraper, + Properties: []*types.Property{getConsoleLink(ctx.Session.Region, "AWS::EC2::RouteTable", lo.FromPtr(r.RouteTableId))}, + Config: r, + ConfigClass: "Route", + Name: getName(labels, *r.RouteTableId), + ID: *r.RouteTableId, + Parents: []v1.ConfigExternalKey{{Type: v1.AWSEC2VPC, ExternalID: lo.FromPtr(r.VpcId)}}, }) } } @@ -729,16 +717,15 @@ func (aws Scraper) dhcp(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResult for _, d := range describeOutput.DhcpOptions { labels := getLabels(d.Tags) *results = append(*results, v1.ScrapeResult{ - Type: v1.AWSEC2DHCPOptions, - Labels: labels, - BaseScraper: config.BaseScraper, - Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSEC2DHCPOptions, lo.FromPtr(d.DhcpOptionsId))}, - Config: d, - ConfigClass: "DHCP", - Name: getName(labels, *d.DhcpOptionsId), - ID: *d.DhcpOptionsId, - ParentExternalID: lo.FromPtr(ctx.Caller.Account), - ParentType: v1.AWSAccount, + Type: v1.AWSEC2DHCPOptions, + Labels: labels, + BaseScraper: config.BaseScraper, + Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSEC2DHCPOptions, lo.FromPtr(d.DhcpOptionsId))}, + Config: d, + ConfigClass: "DHCP", + Name: getName(labels, *d.DhcpOptionsId), + ID: *d.DhcpOptionsId, + Parents: []v1.ConfigExternalKey{{Type: v1.AWSAccount, ExternalID: lo.FromPtr(ctx.Caller.Account)}}, }) } } @@ -758,19 +745,18 @@ func (aws Scraper) s3Buckets(ctx *AWSContext, config v1.AWS, results *v1.ScrapeR for _, bucket := range buckets.Buckets { labels := make(map[string]string) *results = append(*results, v1.ScrapeResult{ - Type: v1.AWSS3Bucket, - CreatedAt: bucket.CreationDate, - BaseScraper: config.BaseScraper, - Config: bucket, - ConfigClass: "ObjectStorage", - Name: *bucket.Name, - Labels: labels, - Ignore: []string{"name", "creationDate"}, - Aliases: []string{"AmazonS3/" + *bucket.Name}, - ID: *bucket.Name, - ParentExternalID: *ctx.Caller.Account, - ParentType: v1.AWSAccount, - Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSS3Bucket, lo.FromPtr(bucket.Name))}, + Type: v1.AWSS3Bucket, + CreatedAt: bucket.CreationDate, + BaseScraper: config.BaseScraper, + Config: bucket, + ConfigClass: "ObjectStorage", + Name: *bucket.Name, + Labels: labels, + Ignore: []string{"name", "creationDate"}, + Aliases: []string{"AmazonS3/" + *bucket.Name}, + ID: *bucket.Name, + Parents: []v1.ConfigExternalKey{{Type: v1.AWSAccount, ExternalID: lo.FromPtr(ctx.Caller.Account)}}, + Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSS3Bucket, lo.FromPtr(bucket.Name))}, }) } } @@ -788,17 +774,16 @@ func (aws Scraper) dnsZones(ctx *AWSContext, config v1.AWS, results *v1.ScrapeRe for _, zone := range zones.HostedZones { labels := make(map[string]string) *results = append(*results, v1.ScrapeResult{ - Type: v1.AWSZone, - BaseScraper: config.BaseScraper, - Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSZone, strings.ReplaceAll(*zone.Id, "/hostedzone/", ""))}, - Config: zone, - ConfigClass: "DNSZone", - Name: *zone.Name, - Labels: labels, - Aliases: []string{*zone.Id, *zone.Name, "AmazonRoute53/arn:aws:route53:::hostedzone/" + *zone.Id}, - ID: strings.ReplaceAll(*zone.Id, "/hostedzone/", ""), - ParentExternalID: *ctx.Caller.Account, - ParentType: v1.AWSAccount, + Type: v1.AWSZone, + BaseScraper: config.BaseScraper, + Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSZone, strings.ReplaceAll(*zone.Id, "/hostedzone/", ""))}, + Config: zone, + ConfigClass: "DNSZone", + Name: *zone.Name, + Labels: labels, + Aliases: []string{*zone.Id, *zone.Name, "AmazonRoute53/arn:aws:route53:::hostedzone/" + *zone.Id}, + ID: strings.ReplaceAll(*zone.Id, "/hostedzone/", ""), + Parents: []v1.ConfigExternalKey{{Type: v1.AWSAccount, ExternalID: lo.FromPtr(ctx.Caller.Account)}}, }) } } @@ -864,7 +849,7 @@ func (aws Scraper) loadBalancers(ctx *AWSContext, config v1.AWS, results *v1.Scr tags := v1.Tags{} tags.Append("zone", az) tags.Append("region", region) - arn := fmt.Sprintf("arn:aws:elasticloadbalancing:%s:%s:loadbalancer/%s", region, *ctx.Caller.Account, *lb.LoadBalancerName) + arn := fmt.Sprintf("arn:aws:elasticloadbalancing:%s:%s:loadbalancer/%s", region, lo.FromPtr(ctx.Caller.Account), *lb.LoadBalancerName) *results = append(*results, v1.ScrapeResult{ Type: v1.AWSLoadBalancer, CreatedAt: lb.CreatedTime, @@ -878,8 +863,7 @@ func (aws Scraper) loadBalancers(ctx *AWSContext, config v1.AWS, results *v1.Scr Tags: tags, Aliases: []string{"AWSELB/" + arn, arn}, ID: *lb.LoadBalancerName, - ParentExternalID: *lb.VPCId, - ParentType: v1.AWSEC2VPC, + Parents: []v1.ConfigExternalKey{{Type: v1.AWSEC2VPC, ExternalID: lo.FromPtr(lb.VPCId)}}, RelationshipResults: relationships, }) } @@ -936,8 +920,7 @@ func (aws Scraper) loadBalancers(ctx *AWSContext, config v1.AWS, results *v1.Scr Aliases: []string{"AWSELB/" + *lb.LoadBalancerArn}, ID: *lb.LoadBalancerArn, Labels: labels, - ParentExternalID: *lb.VpcId, - ParentType: v1.AWSEC2VPC, + Parents: []v1.ConfigExternalKey{{Type: v1.AWSEC2VPC, ExternalID: lo.FromPtr(lb.VpcId)}}, RelationshipResults: relationships, }.WithHealthStatus(resourceHealth)) } @@ -1006,8 +989,7 @@ func (aws Scraper) subnets(ctx *AWSContext, config v1.AWS, results *v1.ScrapeRes ConfigClass: "Subnet", ID: *subnet.SubnetId, Config: subnet, - ParentExternalID: lo.FromPtr(subnet.VpcId), - ParentType: v1.AWSEC2VPC, + Parents: []v1.ConfigExternalKey{{Type: v1.AWSEC2VPC, ExternalID: lo.FromPtr(subnet.VpcId)}}, Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSEC2Subnet, lo.FromPtr(subnet.SubnetId))}, RelationshipResults: relationships, }.WithHealthStatus(resourceHealth) @@ -1031,18 +1013,17 @@ func (aws Scraper) iamRoles(ctx *AWSContext, config v1.AWS, results *v1.ScrapeRe labels := make(map[string]string) *results = append(*results, v1.ScrapeResult{ - Type: v1.AWSIAMRole, - CreatedAt: role.CreateDate, - BaseScraper: config.BaseScraper, - Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSIAMRole, lo.FromPtr(role.RoleName))}, - Config: role, - ConfigClass: "Role", - Labels: labels, - Name: *role.RoleName, - Aliases: []string{*role.RoleName, *role.Arn}, - ID: *role.RoleId, - ParentExternalID: lo.FromPtr(ctx.Caller.Account), - ParentType: v1.AWSAccount, + Type: v1.AWSIAMRole, + CreatedAt: role.CreateDate, + BaseScraper: config.BaseScraper, + Properties: []*types.Property{getConsoleLink(ctx.Session.Region, v1.AWSIAMRole, lo.FromPtr(role.RoleName))}, + Config: role, + ConfigClass: "Role", + Labels: labels, + Name: *role.RoleName, + Aliases: []string{*role.RoleName, *role.Arn}, + ID: *role.RoleId, + Parents: []v1.ConfigExternalKey{{Type: v1.AWSAccount, ExternalID: lo.FromPtr(ctx.Caller.Account)}}, }) } } @@ -1120,9 +1101,8 @@ func (aws Scraper) iamProfiles(ctx *AWSContext, config v1.AWS, results *v1.Scrap Name: *profile.InstanceProfileName, Aliases: []string{*profile.InstanceProfileName, *profile.Arn}, ID: *profile.InstanceProfileId, - ParentExternalID: lo.FromPtr(ctx.Caller.Account), - ParentType: v1.AWSAccount, RelationshipResults: relationships, + Parents: []v1.ConfigExternalKey{{Type: v1.AWSAccount, ExternalID: lo.FromPtr(ctx.Caller.Account)}}, }) } } @@ -1146,7 +1126,7 @@ func (aws Scraper) ami(ctx *AWSContext, config v1.AWS, results *v1.ScrapeResults } labels := make(map[string]string) - labels["region"] = *ctx.Caller.Account + labels["region"] = lo.FromPtr(ctx.Caller.Account) *results = append(*results, v1.ScrapeResult{ Type: v1.AWSEC2AMI, CreatedAt: &createdAt, diff --git a/scrapers/azure/azure.go b/scrapers/azure/azure.go index 1417c4dc..4731a3bc 100644 --- a/scrapers/azure/azure.go +++ b/scrapers/azure/azure.go @@ -174,15 +174,17 @@ func (azure Scraper) Scrape(ctx api.ScrapeContext) v1.ScrapeResults { }) // Set parents where missing - if results[i].ParentExternalID == "" { - switch results[i].Type { - case getARMType(lo.ToPtr(ResourceTypeSubscription)): - continue // root - - default: - subscriptionID := strings.Split(results[i].ID, "/")[2] - results[i].ParentExternalID = getARMID(lo.ToPtr("/subscriptions/" + subscriptionID)) - results[i].ParentType = getARMType(lo.ToPtr(ResourceTypeSubscription)) + for j, parent := range results[i].Parents { + if parent.ExternalID == "" { + switch parent.Type { + case getARMType(lo.ToPtr(ResourceTypeSubscription)): + continue // root + + default: + subscriptionID := strings.Split(results[i].ID, "/")[2] + results[i].Parents[j].ExternalID = getARMID(lo.ToPtr("/subscriptions/" + subscriptionID)) + results[i].Parents[j].Type = getARMType(lo.ToPtr(ResourceTypeSubscription)) + } } } } diff --git a/scrapers/kubernetes/kubernetes.go b/scrapers/kubernetes/kubernetes.go index 0bd28dbd..34ad6685 100644 --- a/scrapers/kubernetes/kubernetes.go +++ b/scrapers/kubernetes/kubernetes.go @@ -429,7 +429,7 @@ func ExtractResults(ctx context.Context, config v1.Kubernetes, objs []*unstructu return results.Errorf(err, "failed to clean kubernetes object") } - parentType, parentExternalID := getKubernetesParent(obj, config.Exclusions, resourceIDMap) + parents := getKubernetesParent(obj, config.Exclusions, resourceIDMap) results = append(results, v1.ScrapeResult{ BaseScraper: config.BaseScraper, Name: obj.GetName(), @@ -448,8 +448,7 @@ func ExtractResults(ctx context.Context, config v1.Kubernetes, objs []*unstructu Labels: stripLabels(labels, "-hash"), Tags: tags, Aliases: []string{getKubernetesAlias(obj.GetKind(), obj.GetNamespace(), obj.GetName())}, - ParentExternalID: parentExternalID, - ParentType: ConfigTypePrefix + parentType, + Parents: parents, RelationshipResults: relationships, }) } @@ -474,47 +473,50 @@ func ExtractResults(ctx context.Context, config v1.Kubernetes, objs []*unstructu return results } -func getKubernetesParent(obj *unstructured.Unstructured, exclusions v1.KubernetesExclusionConfig, resourceIDMap map[string]map[string]map[string]string) (string, string) { - var parentExternalID, parentConfigType string - - // This will work for pods and replicasets - if len(obj.GetOwnerReferences()) > 0 { - ref := obj.GetOwnerReferences()[0] - - if obj.GetKind() == "Pod" && lo.Contains(exclusions.Kinds, "ReplicaSet") { - // If ReplicaSet is excluded then we want the pod's direct parent to - // be its Deployment - if ref.Kind == "ReplicaSet" { - deployName := extractDeployNameFromReplicaSet(ref.Name) - parentConfigType = "Deployment" - parentExternalID = resourceIDMap[obj.GetNamespace()]["Deployment"][deployName] - return parentConfigType, parentExternalID - } - } - - parentConfigType = ref.Kind - parentExternalID = string(ref.UID) - return parentConfigType, parentExternalID - } +// getKubernetesParent returns a list of potential parents in order. +// Example: For a Pod the parents would be [Replicaset, Namespace, Cluster] +func getKubernetesParent(obj *unstructured.Unstructured, exclusions v1.KubernetesExclusionConfig, resourceIDMap map[string]map[string]map[string]string) []v1.ConfigExternalKey { + var allParents []v1.ConfigExternalKey + allParents = append(allParents, v1.ConfigExternalKey{ + Type: ConfigTypePrefix + "Cluster", + ExternalID: resourceIDMap[""]["Cluster"]["selfRef"], + }) if obj.GetNamespace() != "" { - parentConfigType = "Namespace" - parentExternalID = resourceIDMap[""]["Namespace"][obj.GetNamespace()] - + parentExternalID := resourceIDMap[""]["Namespace"][obj.GetNamespace()] if parentExternalID == "" { // An incremental scraper maynot have the Namespace object. // We can instead use the alias as the external id. parentExternalID = getKubernetesAlias("Namespace", "", obj.GetNamespace()) } - return parentConfigType, parentExternalID + allParents = append([]v1.ConfigExternalKey{{ + Type: ConfigTypePrefix + "Namespace", + ExternalID: parentExternalID, + }}, allParents...) } - // Everything which is not namespaced should be mapped to cluster - parentConfigType = "Cluster" - parentExternalID = resourceIDMap[""]["Cluster"]["selfRef"] + if len(obj.GetOwnerReferences()) > 0 { + ref := obj.GetOwnerReferences()[0] + + // If ReplicaSet is excluded then we want the pod's direct parent to + // be its Deployment + if obj.GetKind() == "Pod" && lo.Contains(exclusions.Kinds, "ReplicaSet") && ref.Kind == "ReplicaSet" { + deployName := extractDeployNameFromReplicaSet(ref.Name) + parentExternalID := resourceIDMap[obj.GetNamespace()]["Deployment"][deployName] + allParents = append([]v1.ConfigExternalKey{{ + Type: ConfigTypePrefix + "Deployment", + ExternalID: parentExternalID, + }}, allParents...) + } else { + allParents = append([]v1.ConfigExternalKey{{ + Type: ConfigTypePrefix + ref.Kind, + ExternalID: string(ref.UID), + }}, allParents...) + } + } - return parentConfigType, parentExternalID + return allParents } func getKubernetesAlias(kind, namespace, name string) string {