DRAFT: feat: add first draft of backstage entity receiver

.vscode/launch.json

@@ -1,6 +1,7 @@
 {
   "version": "0.3.0",
   "configurations": [
+
     {
       "name": "Launch Otel Collector in debug",
       "type": "go",
@@ -13,6 +14,18 @@
         "../config/config.yaml"
       ],
       "envFile": "${workspaceFolder}/.debug.env"
+    },
+    {
+      "name": "Launch Otel Collector in debug (backstage)",
+      "type": "go",
+      "request": "launch",
+      "mode": "auto",
+      "cwd": "${workspaceFolder}/build",
+      "program": "${workspaceFolder}/build",
+      "args": [
+        "--config",
+        "../config/config-backstage.yaml"
+      ]
     }
   ]
 }

config/config-backstage.yaml

@@ -0,0 +1,141 @@
+---
+extensions:
+  health_check:
+
+  pprof:
+    endpoint: 0.0.0.0:1777
+
+  zpages:
+    endpoint: 0.0.0.0:55679
+
+receivers:
+  ## Collect own metrics based on example from OTEL Contrib Dockerfile
+  ## https://github.com/open-telemetry/opentelemetry-collector-releases/blob/main/configs/otelcol-contrib.yaml
+  prometheus:
+    config:
+      scrape_configs:
+        - job_name: liatrio-otel-collector
+          scrape_interval: 10s
+          static_configs:
+            - targets: [0.0.0.0:8888]
+
+  backstage:
+    url: https://backstage.liatr.io
+    objects:
+      - group: scaffolder.backstage.io
+        kind: Template
+        interval: 10m
+
+      - group: backstage.io
+        kind: Component
+        interval: 10m
+
+      - group: backstage.io
+        kind: Group
+        interval: 10m
+
+      - group: backstage.io
+        kind: User
+        interval: 10m
+
+      - group: backstage.io
+        kind: Domain
+        interval: 10m
+
+      - group: backstage.io
+        kind: Location
+        interval: 10m
+
+      - group: backstage.io
+        kind: System
+        interval: 10m
+
+      - group: backstage.io
+        kind: Service
+        interval: 10m
+
+      - group: backstage.io
+        kind: API
+        interval: 10m
+
+processors:
+  batch:
+  resourcedetection:
+    detectors: [env]
+    timeout: 2s
+    override: false
+
+  groupbyattrs:
+    keys:
+      - backstage.entity.kind
+      - backstage.entity.type
+      - backstage.entity.owner
+      - backstage.entity.lifecycle
+      - backstage.entity.name
+      - backstage.entity.namespace
+
+  resource/env:
+    attributes:
+      - key: environment.name
+        value: prod
+        action: upsert
+      - key: service.name
+        value: backstage.liatr.io
+        action: upsert
+
+  attributes/logs:
+    actions:
+      - key: loki.attribute.labels
+        action: upsert
+        value: service.name, environment.name
+
+
+
+  transform/to_guage:
+    #error_mode: ignore
+    metric_statements:
+      - context: datapoint
+        statements:
+        - convert_sum_to_gauge()
+
+  #transform/json:
+  #  error_mode: ignore
+  #  log_statements:
+  #    - context: log
+  #      statements:
+  #      - set(attributes["k8s_resource_name"], body["metadata"]["name"])
+
+connectors:
+  count/entities:
+    logs:
+      backstage_entity_count:
+        name: backstage_entity_count
+        description: The number of backstage entities grouped by kind/type/owner/lifecycle
+
+exporters:
+  debug:
+    verbosity: basic
+    sampling_initial: 2
+    sampling_thereafter: 500
+
+  debug/verbose:
+    verbosity: detailed
+
+  otlphttp:
+    endpoint: https://otel-gateway.dev.k8s-platform-v3.liatr.io
+    tls:
+      insecure: false
+
+service:
+  ## Add bearertokenauth/* if you want to use bearer token auth extensions
+  extensions: [health_check, pprof, zpages]
+  pipelines:
+    logs:
+      receivers: [backstage]
+      processors: [resource/env, attributes/logs, groupbyattrs]
+      exporters: [debug/verbose, otlphttp, count/entities]
+
+    metrics:
+      receivers: [count/entities]
+      processors: [transform/to_guage]
+      exporters: [debug/verbose, otlphttp]

config/manifest.yaml

@@ -42,9 +42,10 @@ processors:
 receivers:
   - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.98.0
   - gomod: github.com/liatrio/liatrio-otel-collector/receiver/gitproviderreceiver v0.1.0
+  - gomod: github.com/liatrio/liatrio-otel-collector/receiver/backstagereceiver v0.1.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.98.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kubeletstatsreceiver v0.98.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/k8sclusterreceiver v0.98.0
+  # - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/k8sclusterreceiver v0.98.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jaegerreceiver v0.98.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/zipkinreceiver v0.98.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/webhookeventreceiver v0.98.0
@@ -59,3 +60,4 @@ connectors:
 
 replaces:
   - github.com/liatrio/liatrio-otel-collector/receiver/gitproviderreceiver => ../receiver/gitproviderreceiver/
+  - github.com/liatrio/liatrio-otel-collector/receiver/backstagereceiver => ../receiver/backstagereceiver/

go.mod

@@ -1,11 +1,17 @@
 module github.com/liatrio/liatrio-otel-collector
 
-go 1.21
+go 1.21.0
 
-require github.com/liatrio/liatrio-otel-collector/receiver/gitproviderreceiver v0.0.0-00010101000000-000000000000
+toolchain go1.22.2
+
+require (
+	github.com/liatrio/liatrio-otel-collector/receiver/gitproviderreceiver v0.0.0-00010101000000-000000000000
+	github.com/liatrio/liatrio-otel-collector/receiver/backstagereceiver v0.0.0-00010101000000-000000000000
+)
 
 require (
 	github.com/Khan/genqlient v0.7.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
@@ -15,6 +21,7 @@ require (
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/go-github/v61 v61.0.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.5 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
@@ -30,23 +37,24 @@ require (
 	github.com/rs/cors v1.10.1 // indirect
 	github.com/vektah/gqlparser/v2 v2.5.11 // indirect
 	github.com/xanzy/go-gitlab v0.103.0 // indirect
-	go.opentelemetry.io/collector v0.98.0 // indirect
-	go.opentelemetry.io/collector/component v0.98.0 // indirect
-	go.opentelemetry.io/collector/config/configauth v0.98.0 // indirect
-	go.opentelemetry.io/collector/config/configcompression v1.5.0 // indirect
-	go.opentelemetry.io/collector/config/confighttp v0.98.0 // indirect
-	go.opentelemetry.io/collector/config/configopaque v1.5.0 // indirect
-	go.opentelemetry.io/collector/config/configtelemetry v0.98.0 // indirect
-	go.opentelemetry.io/collector/config/configtls v0.98.0 // indirect
-	go.opentelemetry.io/collector/config/internal v0.98.0 // indirect
-	go.opentelemetry.io/collector/confmap v0.98.0 // indirect
-	go.opentelemetry.io/collector/consumer v0.98.0 // indirect
-	go.opentelemetry.io/collector/extension v0.98.0 // indirect
-	go.opentelemetry.io/collector/extension/auth v0.98.0 // indirect
-	go.opentelemetry.io/collector/featuregate v1.5.0 // indirect
-	go.opentelemetry.io/collector/pdata v1.5.0 // indirect
-	go.opentelemetry.io/collector/receiver v0.98.0 // indirect
-	go.opentelemetry.io/collector/semconv v0.98.0 // indirect
+	go.einride.tech/backstage v0.8.0 // indirect
+	go.opentelemetry.io/collector v0.99.0 // indirect
+	go.opentelemetry.io/collector/component v0.99.0 // indirect
+	go.opentelemetry.io/collector/config/configauth v0.99.0 // indirect
+	go.opentelemetry.io/collector/config/configcompression v1.6.0 // indirect
+	go.opentelemetry.io/collector/config/confighttp v0.99.0 // indirect
+	go.opentelemetry.io/collector/config/configopaque v1.6.0 // indirect
+	go.opentelemetry.io/collector/config/configtelemetry v0.99.0 // indirect
+	go.opentelemetry.io/collector/config/configtls v0.99.0 // indirect
+	go.opentelemetry.io/collector/config/internal v0.99.0 // indirect
+	go.opentelemetry.io/collector/confmap v0.99.0 // indirect
+	go.opentelemetry.io/collector/consumer v0.99.0 // indirect
+	go.opentelemetry.io/collector/extension v0.99.0 // indirect
+	go.opentelemetry.io/collector/extension/auth v0.99.0 // indirect
+	go.opentelemetry.io/collector/featuregate v1.6.0 // indirect
+	go.opentelemetry.io/collector/pdata v1.6.0 // indirect
+	go.opentelemetry.io/collector/receiver v0.99.0 // indirect
+	go.opentelemetry.io/collector/semconv v0.99.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.50.0 // indirect
 	go.opentelemetry.io/otel v1.25.0 // indirect
 	go.opentelemetry.io/otel/metric v1.25.0 // indirect
@@ -56,11 +64,24 @@ require (
 	golang.org/x/net v0.24.0 // indirect
 	golang.org/x/oauth2 v0.19.0 // indirect
 	golang.org/x/sys v0.19.0 // indirect
+	golang.org/x/term v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect
 	google.golang.org/grpc v1.63.2 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	k8s.io/apimachinery v0.29.3 // indirect
+	k8s.io/client-go v0.29.3 // indirect
+	k8s.io/klog/v2 v2.110.1 // indirect
+	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
-replace github.com/liatrio/liatrio-otel-collector/receiver/gitproviderreceiver => ./receiver/gitproviderreceiver
+replace (
+	github.com/liatrio/liatrio-otel-collector/receiver/gitproviderreceiver => ./receiver/gitproviderreceiver
+	github.com/liatrio/liatrio-otel-collector/receiver/backstagereceiver => ./receiver/backstagereceiver
+)