Build otel-collector distribution

- Use the otel collector builder to build our own distribution of the
  collector rather than using the binaries provided by the
  otelcol-contrib distribution.
- This allows us to build from source, bump Go as necessary and control
  the set of exporters etc that are available.
- The contents of our distribution are defined in
  src/otel-collector/config.yaml and can be re-generated using
  scripts/regenerate-otel-collector-distribution.
- The BOSH packaging scripts for the collector packages perform the
  compilation using the builder, but assume that the generated code has
  already been committed to the repository.
- The builder also has its own go.mod to avoid the need to add the
  builder dependencies to the main repo go.mod.
- Prunes the list of receivers, processors, and exporters provided by
  the otelcol-contrib distribution.

Signed-off-by: Andrew Crump <[email protected]>

config/blobs.yml

@@ -1,8 +1 @@
-otelcol-contrib_0.79.0_linux_amd64.tar.gz:
-  size: 54313545
-  object_id: 9d6c0664-89f3-443d-6a6d-d199b33bbf1c
-  sha: sha256:5d1ca4ceb51b487f7648e361627adb3eb1ffdf6c7a7fdd35a4b3d560eb73e8aa
-otelcol-contrib_0.79.0_windows_amd64.tar.gz:
-  size: 53149526
-  object_id: 5251ce8d-725c-4415-697c-96fbf4498185
-  sha: sha256:c99b0ce58da8f7667f68539573c9356721ff2e63138a7242099b3b808479905e
+{}

packages/otel-collector-windows/packaging

@@ -1,9 +1,27 @@
-. ./exiter.ps1
+. .\exiter.ps1
+. C:\var\vcap\packages\golang-1.20-windows\bosh\compile.ps1
+$env:CGO_ENABLED="0"
 
 $ErrorActionPreference = "Stop";
 trap { $host.SetShouldExit(1) }
 
-# [Powershell can't resolve symlinks correctly](https://github.com/PowerShell/PowerShell/issues/7818)
-# and tar won't extract to a path that includes a symlink. So we're left extracting to D:\tmp
-tar -xvf $(Resolve-Path "otelcol-contrib_*_windows_amd64.tar.gz") -C D:\tmp otelcol-contrib.exe
-move D:\tmp\otelcol-contrib.exe "${env:BOSH_INSTALL_TARGET}\otel-collector.exe"
+$BOSH_INSTALL_TARGET = Resolve-Path "${env:BOSH_INSTALL_TARGET}"
+if ((Get-Item $BOSH_INSTALL_TARGET).LinkType -ne "") {
+  $BOSH_INSTALL_TARGET = (Get-Item $BOSH_INSTALL_TARGET).Target
+}
+
+Push-Location otel-collector-builder\vendor\go.opentelemetry.io\collector\cmd\builder\
+  go.exe install
+  if ($LASTEXITCODE -ne 0) {
+    Write-Error "Error compiling builder"
+  }
+Pop-Location
+
+& ${env:GOPATH}\bin\builder.exe --config otel-collector\config.yaml --skip-generate --skip-get-modules
+if ($LASTEXITCODE -ne 0) {
+  Write-Error "Error building otel-collector"
+}
+
+Copy-Item "otel-collector\cf-otel-collector" -Destination "${env:BOSH_INSTALL_TARGET}\otel-collector.exe"
+
+Exit 0

packages/otel-collector-windows/spec

@@ -1,8 +1,10 @@
 ---
 name: otel-collector-windows
 
-dependencies: []
+dependencies:
+- golang-1.20-windows
 
 files:
-- otelcol-contrib*windows_amd64.tar.gz
 - exiter.ps1
+- otel-collector-builder/**/*
+- otel-collector/**/*

packages/otel-collector/packaging

@@ -1,4 +1,14 @@
-set -ex
+#!/bin/bash
+set -e -x
 
-tar -zxvf otelcol-contrib*linux_amd64.tar.gz otelcol-contrib
-mv otelcol-contrib "${BOSH_INSTALL_TARGET}/otel-collector"
+source /var/vcap/packages/golang-1.20-linux/bosh/compile.env
+
+export CGO_ENABLED=0
+
+pushd otel-collector-builder/vendor/go.opentelemetry.io/collector/cmd/builder/
+  go install
+popd
+
+builder --config otel-collector/config.yaml --skip-generate --skip-get-modules
+
+cp "otel-collector/cf-otel-collector" "${BOSH_INSTALL_TARGET}/otel-collector"

packages/otel-collector/spec

@@ -1,7 +1,9 @@
 ---
 name: otel-collector
 
-dependencies: []
+dependencies:
+- golang-1.20-linux
 
 files:
-- otelcol-contrib*linux_amd64.tar.gz
+- otel-collector-builder/**/*
+- otel-collector/**/*

scripts/regenerate-otel-collector-distribution

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -euxo pipefail
+
+release_dir="$(dirname "$0")/.."
+
+pushd "${release_dir}/src/otel-collector-builder/vendor/go.opentelemetry.io/collector/cmd/builder"
+  CGO_ENABLED=0 go install
+popd
+
+pushd "${release_dir}/src/"
+  builder --skip-compilation --config=otel-collector/config.yaml
+  cd otel-collector
+  go mod vendor
+popd

src/otel-collector-builder/go.mod

@@ -0,0 +1,26 @@
+module code.cloudfoundry.org/loggregator-agent-release/src/otel-collector-builder
+
+go 1.20
+
+require go.opentelemetry.io/collector/cmd/builder v0.82.0
+
+require (
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/knadh/koanf/maps v0.1.1 // indirect
+	github.com/knadh/koanf/parsers/yaml v0.1.0 // indirect
+	github.com/knadh/koanf/providers/env v0.1.0 // indirect
+	github.com/knadh/koanf/providers/file v0.1.0 // indirect
+	github.com/knadh/koanf/providers/fs v0.1.0 // indirect
+	github.com/knadh/koanf/v2 v2.0.1 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/spf13/cobra v1.7.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	go.uber.org/atomic v1.10.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.24.0 // indirect
+	golang.org/x/sys v0.10.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

src/otel-collector-builder/go.sum

@@ -0,0 +1,44 @@
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
+github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs=
+github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI=
+github.com/knadh/koanf/parsers/yaml v0.1.0 h1:ZZ8/iGfRLvKSaMEECEBPM1HQslrZADk8fP1XFUxVI5w=
+github.com/knadh/koanf/parsers/yaml v0.1.0/go.mod h1:cvbUDC7AL23pImuQP0oRw/hPuccrNBS2bps8asS0CwY=
+github.com/knadh/koanf/providers/env v0.1.0 h1:LqKteXqfOWyx5Ab9VfGHmjY9BvRXi+clwyZozgVRiKg=
+github.com/knadh/koanf/providers/env v0.1.0/go.mod h1:RE8K9GbACJkeEnkl8L/Qcj8p4ZyPXZIQ191HJi44ZaQ=
+github.com/knadh/koanf/providers/file v0.1.0 h1:fs6U7nrV58d3CFAFh8VTde8TM262ObYf3ODrc//Lp+c=
+github.com/knadh/koanf/providers/file v0.1.0/go.mod h1:rjJ/nHQl64iYCtAW2QQnF0eSmDEX/YZ/eNFj5yR6BvA=
+github.com/knadh/koanf/providers/fs v0.1.0 h1:9Hln9GS3bWTItAnGVFYyfkoAIxAFq7pvlF64pTNiDdQ=
+github.com/knadh/koanf/providers/fs v0.1.0/go.mod h1:Cva1yH8NBxkEeVZx8CUmF5TunbgO72E+GwqDbqpP2sE=
+github.com/knadh/koanf/v2 v2.0.1 h1:1dYGITt1I23x8cfx8ZnldtezdyaZtfAuRtIFOiRzK7g=
+github.com/knadh/koanf/v2 v2.0.1/go.mod h1:ZeiIlIDXTE7w1lMT6UVcNiRAS2/rCeLn/GdLNvY1Dus=
+github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
+github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
+github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
+github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
+github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+go.opentelemetry.io/collector/cmd/builder v0.82.0 h1:Ps1QQ1kSB7duGn0IBIbH4b/7EyFoHSwBsqbvVZbwIB8=
+go.opentelemetry.io/collector/cmd/builder v0.82.0/go.mod h1:9CbdjNrv6+Y83ko1zh2viVWM20yNT3X+Rvl5FUb4B+4=
+go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
+go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
+go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
+golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
+golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

src/otel-collector-builder/tools.go

@@ -0,0 +1,8 @@
+//go:build tools
+// +build tools
+
+package main
+
+import (
+	_ "go.opentelemetry.io/collector/cmd/builder"
+)