Skip to content

Commit

Permalink
feat: cache resource id map
Browse files Browse the repository at this point in the history
[skip ci]
  • Loading branch information
adityathebe committed Jul 15, 2024
1 parent 42a47d0 commit 697f447
Show file tree
Hide file tree
Showing 4 changed files with 140 additions and 39 deletions.
2 changes: 1 addition & 1 deletion api/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ func (t *TempCache) Find(ctx ScrapeContext, lookup v1.ExternalID) (*models.Confi
}

var result models.ConfigItem
query := ctx.DB().Limit(1).Where("deleted_at IS NULL").Where("type = ? and external_id @> ?", lookup.ConfigType, pq.StringArray{externalID})
query := ctx.DB().Limit(1).Order("updated_at DESC").Where("deleted_at IS NULL").Where("type = ? and external_id @> ?", lookup.ConfigType, pq.StringArray{externalID})
if scraperID != "all" && scraperID != "" {
query = query.Where("scraper_id = ?", scraperID)
}
Expand Down
18 changes: 18 additions & 0 deletions api/v1/interface.go
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,24 @@ type RelationshipResult struct {
Relationship string
}

func (t RelationshipResult) WithConfig(id string, ext ExternalID) RelationshipResult {
if id != "" {
t.ConfigID = id
} else {
t.ConfigExternalID = ext
}
return t
}

func (t RelationshipResult) WithRelated(id string, ext ExternalID) RelationshipResult {
if id != "" {
t.RelatedConfigID = id
} else {
t.RelatedExternalID = ext
}
return t
}

func (r RelationshipResult) String() string {
s := ""
if r.ConfigID != "" {
Expand Down
76 changes: 38 additions & 38 deletions scrapers/kubernetes/kubernetes.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"regexp"
"strconv"
"strings"
"sync"
"time"

"github.com/Jeffail/gabs/v2"
Expand Down Expand Up @@ -35,6 +36,10 @@ import (

const ConfigTypePrefix = "Kubernetes::"

// TODO: this lock should be per scraper(cluster)
var allClustersResourceIDMap = map[string]ResourceIDMap{}
var clusterResourceIDLock sync.Mutex

// ReservedAnnotations
const (
// AnnotationIgnoreConfig excludes the object from being scraped
Expand Down Expand Up @@ -254,9 +259,15 @@ func ExtractResults(ctx api.ScrapeContext, config v1.Kubernetes, objs []*unstruc
}

resourceIDMap := getResourceIDsFromObjs(objs)
resourceIDMap[""]["Cluster"] = make(map[string]string)
resourceIDMap[""]["Cluster"][config.ClusterName] = clusterID
resourceIDMap[""]["Cluster"]["selfRef"] = clusterID // For shorthand
resourceIDMap.Set("", "Cluster", config.ClusterName, clusterID)
resourceIDMap.Set("", "Cluster", "selfRef", clusterID) // For shorthand

// For incremental scrape, we do not have all the data in the resource ID map
// we use it from the cached resource id map
clusterResourceIDLock.Lock()
resourceIDMap = mergeResourceIDMap(resourceIDMap, allClustersResourceIDMap[string(ctx.ScrapeConfig().GetUID())])
allClustersResourceIDMap[string(ctx.ScrapeConfig().GetUID())] = resourceIDMap
clusterResourceIDLock.Unlock()

objChangeExclusionByType, objChangeExclusionBySeverity := formObjChangeExclusionMap(objs)

Expand Down Expand Up @@ -396,10 +407,12 @@ func ExtractResults(ctx api.ScrapeContext, config v1.Kubernetes, objs []*unstruc
nodeExternalID := lo.CoalesceOrEmpty(nodeID, getKubernetesAlias("Node", "", nodeName))

relationships = append(relationships, v1.RelationshipResult{
ConfigExternalID: v1.ExternalID{ExternalID: []string{nodeExternalID}, ConfigType: ConfigTypePrefix + "Node"},
RelatedExternalID: v1.ExternalID{ExternalID: []string{string(obj.GetUID())}, ConfigType: ConfigTypePrefix + "Pod"},
Relationship: "NodePod",
})
RelatedConfigID: string(obj.GetUID()),
Relationship: "NodePod",
}.WithConfig(
resourceIDMap.Get("", "Node", nodeName),
v1.ExternalID{ExternalID: []string{nodeExternalID}, ConfigType: ConfigTypePrefix + "Node"},
))
}

if obj.GetLabels()["app.kubernetes.io/name"] == "aws-node" {
Expand Down Expand Up @@ -437,18 +450,22 @@ func ExtractResults(ctx api.ScrapeContext, config v1.Kubernetes, objs []*unstruc

if obj.GetNamespace() != "" {
relationships = append(relationships, v1.RelationshipResult{
ConfigExternalID: v1.ExternalID{ExternalID: []string{fmt.Sprintf("Kubernetes/Namespace//%s", obj.GetNamespace())}, ConfigType: ConfigTypePrefix + "Namespace"},
RelatedExternalID: v1.ExternalID{ExternalID: []string{string(obj.GetUID())}, ConfigType: getConfigTypePrefix(obj.GetAPIVersion()) + obj.GetKind()},
Relationship: "Namespace" + obj.GetKind(),
})
RelatedConfigID: string(obj.GetUID()),
Relationship: "Namespace" + obj.GetKind(),
}.WithConfig(
resourceIDMap.Get("", "Namespace", obj.GetNamespace()),
v1.ExternalID{ExternalID: []string{getKubernetesAlias("Namespace", "", obj.GetNamespace())}, ConfigType: ConfigTypePrefix + "Namespace"},
))
}

for _, ownerRef := range obj.GetOwnerReferences() {
relationships = append(relationships, v1.RelationshipResult{
ConfigExternalID: v1.ExternalID{ExternalID: []string{string(ownerRef.UID)}, ConfigType: getConfigTypePrefix(ownerRef.APIVersion) + ownerRef.Kind},
RelatedExternalID: v1.ExternalID{ExternalID: []string{string(obj.GetUID())}, ConfigType: getConfigTypePrefix(obj.GetAPIVersion()) + obj.GetKind()},
Relationship: ownerRef.Kind + obj.GetKind(),
})
RelatedConfigID: string(obj.GetUID()),
Relationship: ownerRef.Kind + obj.GetKind(),
}.WithConfig(
resourceIDMap.Get(obj.GetNamespace(), ownerRef.Kind, ownerRef.Name),
v1.ExternalID{ExternalID: []string{string(ownerRef.UID)}, ConfigType: getConfigTypePrefix(ownerRef.APIVersion) + ownerRef.Kind},
))
}

for _, f := range config.Relationships {
Expand All @@ -471,9 +488,12 @@ func ExtractResults(ctx api.ScrapeContext, config v1.Kubernetes, objs []*unstruc

for _, id := range linkedConfigItemIDs {
rel := v1.RelationshipResult{
RelatedExternalID: v1.ExternalID{ExternalID: []string{string(obj.GetUID())}, ConfigType: getConfigTypePrefix(obj.GetKind()) + obj.GetKind()},
ConfigID: id.String(),
}
ConfigID: id.String(),
}.WithRelated(
resourceIDMap.Get(obj.GetNamespace(), obj.GetKind(), obj.GetName()),
v1.ExternalID{ExternalID: []string{string(obj.GetUID())}, ConfigType: getConfigTypePrefix(obj.GetAPIVersion()) + obj.GetKind()},
)

relationships = append(relationships, rel)
}
}
Expand Down Expand Up @@ -720,26 +740,6 @@ func extractDeployNameFromReplicaSet(rs string) string {
return strings.Join(split, "-")
}

func getResourceIDsFromObjs(objs []*unstructured.Unstructured) map[string]map[string]map[string]string {
// {Namespace: {Kind: {Name: ID}}}
resourceIDMap := make(map[string]map[string]map[string]string)
resourceIDMap[""] = make(map[string]map[string]string)

for _, obj := range objs {
if collections.Contains([]string{"Namespace", "Deployment", "Node"}, obj.GetKind()) {
if resourceIDMap[obj.GetNamespace()] == nil {
resourceIDMap[obj.GetNamespace()] = make(map[string]map[string]string)
}
if resourceIDMap[obj.GetNamespace()][obj.GetKind()] == nil {
resourceIDMap[obj.GetNamespace()][obj.GetKind()] = make(map[string]string)
}
resourceIDMap[obj.GetNamespace()][obj.GetKind()][obj.GetName()] = string(obj.GetUID())
}
}

return resourceIDMap
}

//nolint:errcheck
func cleanKubernetesObject(obj map[string]any) (map[string]any, error) {
o := gabs.Wrap(obj)
Expand Down
83 changes: 83 additions & 0 deletions scrapers/kubernetes/resource_map.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package kubernetes

import "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"

// map<namespace><kind><name>: <id>
type ResourceIDMap map[string]map[string]map[string]string

func (t ResourceIDMap) Set(namespace, kind, name, id string) {
if t[namespace] == nil {
t[namespace] = make(map[string]map[string]string)
}
if t[namespace][kind] == nil {
t[namespace][kind] = make(map[string]string)
}
t[namespace][kind][name] = id
}

func (t ResourceIDMap) Get(namespace, kind, name string) string {
if kinds, ok := t[namespace]; ok {
if names, ok := kinds[kind]; ok {
if id, ok := names[name]; ok {
return id
}
}
}

return ""
}

func getResourceIDsFromObjs(objs []*unstructured.Unstructured) ResourceIDMap {
resourceIDMap := make(map[string]map[string]map[string]string)
for _, obj := range objs {
if resourceIDMap[obj.GetNamespace()] == nil {
resourceIDMap[obj.GetNamespace()] = make(map[string]map[string]string)
}
if resourceIDMap[obj.GetNamespace()][obj.GetKind()] == nil {
resourceIDMap[obj.GetNamespace()][obj.GetKind()] = make(map[string]string)
}
resourceIDMap[obj.GetNamespace()][obj.GetKind()][obj.GetName()] = string(obj.GetUID())
}

return resourceIDMap
}

func mergeResourceIDMap(latest, cached ResourceIDMap) ResourceIDMap {
if len(latest) == 0 {
return cached
}

if len(cached) == 0 {
return latest
}

output := make(ResourceIDMap)

// First, copy all data from cached
for k, v := range cached {
output[k] = make(map[string]map[string]string)
for k2, v2 := range v {
output[k][k2] = make(map[string]string)
for k3, v3 := range v2 {
output[k][k2][k3] = v3
}
}
}

// Then, update or add data from latest
for k, v := range latest {
if _, ok := output[k]; !ok {
output[k] = make(map[string]map[string]string)
}
for k2, v2 := range v {
if _, ok := output[k][k2]; !ok {
output[k][k2] = make(map[string]string)
}
for k3, v3 := range v2 {
output[k][k2][k3] = v3
}
}
}

return output
}

0 comments on commit 697f447

Please sign in to comment.