Release v0.36.0 (#783)

* Update deps to v0.36.0

* Bump smart agent bundle to v5.14.0

* Bump github.com/signalfx/signalfx-agent to main

* Update CHANGELOG.md

* Fixes for breaking upstream changes

Unit tests pass (at least on my machine) but we may want to test manually as
well.

* Use new run api from collector v.36

* Update configs for tls/insecure option

* testutil 0.36.0 fixes and mod tidy

* Bump smart agent bundle to v5.14.1

* Bump github.com/signalfx/signalfx-agent to main

* missing test insecure tls settings

* account for updated exec default timeout

* correct failing vault config source test

* Move exporters::otlp::tls::insecure if it exists

* Add a warning for deprecated parameter in otlp exporter

* Update CHANGELOG.md

Co-authored-by: Ryan Fitzpatrick <[email protected]>

* Update CHANGELOG.md

Co-authored-by: Pablo Collins <[email protected]>
Co-authored-by: Ryan Fitzpatrick <[email protected]>
Co-authored-by: Dmitry <[email protected]>
Co-authored-by: Ryan Fitzpatrick <[email protected]>

CHANGELOG.md

@@ -2,6 +2,21 @@
 
 ## Unreleased
 
+## v0.36.0
+
+This Splunk OpenTelemetry Connector release includes changes from the [opentelemetry-collector v0.36.0](https://github.com/open-telemetry/opentelemetry-collector/releases/tag/v0.36.0) and the [opentelemetry-collector-contrib v0.36.0](https://github.com/open-telemetry/opentelemetry-collector-contrib/releases/tag/v0.36.0) releases.
+
+### 💡 Enhancements 💡
+
+- Handle backwards compatibility of internal ballast removal (#759)
+- Update bundled Smart Agent to [v5.14.1](https://github.com/signalfx/signalfx-agent/releases/tag/v5.14.1)
+- Automatically relocate removed OTLP exporter "insecure" field (#783)
+
+### 🧰 Bug fixes 🧰
+
+- Move Heroku buildpack to [https://github.com/signalfx/splunk-otel-collector-heroku](https://github.com/signalfx/splunk-otel-collector-heroku) (#755)
+- Fix rpm installation conflicts with the Smart Agent rpm (#773)
+
 ## v0.35.0
 
 This Splunk OpenTelemetry Connector release includes changes from the [opentelemetry-collector v0.35.0](https://github.com/open-telemetry/opentelemetry-collector/releases/tag/v0.35.0) and the [opentelemetry-collector-contrib v0.35.0](https://github.com/open-telemetry/opentelemetry-collector-contrib/releases/tag/v0.35.0) releases.

cmd/otelcol/config/collector/agent_config.yaml

@@ -142,7 +142,8 @@ exporters:
   # Send to gateway
   otlp:
     endpoint: "${SPLUNK_GATEWAY_URL}:4317"
-    insecure: true
+    tls:
+      insecure: true
   # Debug
   logging:
     loglevel: debug

cmd/otelcol/config/collector/full_config_linux.yaml

@@ -394,7 +394,8 @@ exporters:
   # NOTE: These settings should be updated to the proper destination
   jaeger:
     endpoint: jaeger-all-in-one:14250
-    insecure: true
+    tls:
+      insecure: true
 
   # Enables the otlp grpc exporter
   # Full configuration here: https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter/otlpexporter
@@ -424,7 +425,8 @@ exporters:
   # NOTE: These settings should be updated to the proper destination
   zipkin:
     endpoint: "http://some.url:9411/api/v2/spans"
-    insecure: true
+    tls:
+      insecure: true
 
   #############################################################################
   # Metrics

cmd/otelcol/config/collector/upstream_agent_config.yaml

@@ -139,7 +139,8 @@ exporters:
   # Send to gateway
   otlp:
     endpoint: "${SPLUNK_GATEWAY_URL}:4317"
-    insecure: true
+    tls:
+      insecure: true
   # Debug
   logging:
     loglevel: debug

cmd/otelcol/main.go

@@ -86,13 +86,19 @@ func main() {
 		info,
 		configsources.Get()...,
 	)
+
 	const noConvertConfigFlag = "--no-convert-config"
 	if hasFlag(noConvertConfigFlag) {
 		// the collector complains about this flag if we don't remove it
 		removeFlag(&os.Args, noConvertConfigFlag)
 	} else {
-		parserProvider = configconverter.ParserProvider(parserProvider, configconverter.RemoveBallastKey)
+		parserProvider = configconverter.ParserProvider(
+			parserProvider,
+			configconverter.RemoveBallastKey,
+			configconverter.MoveOTLPInsecureKey,
+		)
 	}
+
 	serviceParams := service.CollectorSettings{
 		BuildInfo:      info,
 		Factories:      factories,
@@ -348,14 +354,14 @@ func newBaseParserProvider() parserprovider.ParserProvider {
 	return parserprovider.Default()
 }
 
-func runInteractive(params service.CollectorSettings) error {
-	app, err := service.New(params)
+func runInteractive(settings service.CollectorSettings) error {
+	app, err := service.New(settings)
 	if err != nil {
 		return fmt.Errorf("failed to construct the application: %w", err)
 	}
 
-	err = app.Run()
-	if err != nil {
+	cmd := service.NewCommand(app)
+	if err = cmd.Execute(); err != nil {
 		return fmt.Errorf("application run finished with error: %w", err)
 	}
 

internal/buildscripts/packaging/smart-agent-release.txt

@@ -1 +1 @@
-v5.13.0
+v5.14.1

internal/components/components.go

@@ -64,7 +64,7 @@ import (
 	"go.opentelemetry.io/collector/extension/ballastextension"
 	"go.opentelemetry.io/collector/extension/zpagesextension"
 	"go.opentelemetry.io/collector/processor/batchprocessor"
-	"go.opentelemetry.io/collector/processor/memorylimiter"
+	"go.opentelemetry.io/collector/processor/memorylimiterprocessor"
 	"go.opentelemetry.io/collector/receiver/otlpreceiver"
 
 	"github.com/signalfx/splunk-otel-collector/internal/extension/smartagentextension"
@@ -136,7 +136,7 @@ func Get() (component.Factories, error) {
 		filterprocessor.NewFactory(),
 		groupbyattrsprocessor.NewFactory(),
 		k8sprocessor.NewFactory(),
-		memorylimiter.NewFactory(),
+		memorylimiterprocessor.NewFactory(),
 		metricstransformprocessor.NewFactory(),
 		probabilisticsamplerprocessor.NewFactory(),
 		resourcedetectionprocessor.NewFactory(),

internal/configconverter/ballast_remover.go

@@ -29,7 +29,7 @@ func RemoveBallastKey(cfgMap *configparser.ConfigMap) *configparser.ConfigMap {
 	const expr = "processors::memory_limiter(/\\w+)?::ballast_size_mib"
 	ballastKeyRegexp, _ := regexp.Compile(expr)
 
-	out := configparser.NewParser()
+	out := configparser.NewConfigMap()
 	for _, k := range cfgMap.AllKeys() {
 		if ballastKeyRegexp.MatchString(k) {
 			log.Println("[WARNING] `ballast_size_mib` parameter in `memory_limiter` processor is " +

internal/configconverter/otlp_insecure_mover.go

@@ -0,0 +1,48 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package configconverter
+
+import (
+	"fmt"
+	"log"
+	"regexp"
+
+	"go.opentelemetry.io/collector/config/configparser"
+)
+
+func MoveOTLPInsecureKey(in *configparser.ConfigMap) *configparser.ConfigMap {
+	const expr = "exporters::otlp(/\\w+)?::insecure"
+	insecureRE, _ := regexp.Compile(expr)
+	out := configparser.NewConfigMap()
+	var deprecatedOTLPConfigFound bool
+	for _, k := range in.AllKeys() {
+		v := in.Get(k)
+		match := insecureRE.FindStringSubmatch(k)
+		if match == nil {
+			out.Set(k, v)
+		} else {
+			tlsKey := fmt.Sprintf("exporters::otlp%s::tls::insecure", match[1])
+			log.Printf("Unsupported key found: %s. Moving to %s\n", k, tlsKey)
+			out.Set(tlsKey, v)
+			deprecatedOTLPConfigFound = true
+		}
+	}
+	if deprecatedOTLPConfigFound {
+		log.Println("[WARNING] `exporters` -> `otlp` -> `insecure` parameter is " +
+			"deprecated. Please update the config according to the guideline: " +
+			"https://github.com/signalfx/splunk-otel-collector#from-0350-to-0360.")
+	}
+	return out
+}

internal/configconverter/otlp_insecure_mover_test.go

@@ -0,0 +1,69 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package configconverter
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConverterProvider_Noop(t *testing.T) {
+	pp := &converterProvider{
+		wrapped: &fileParserProvider{fileName: "testdata/otlp-insecure.yaml"},
+	}
+	v, err := pp.Get(context.Background())
+	require.NoError(t, err)
+	assert.True(t, v.IsSet("exporters::otlp::insecure"))
+}
+
+func TestMoveOTLPInsecureKey(t *testing.T) {
+	pp := &converterProvider{
+		wrapped:     &fileParserProvider{fileName: "testdata/otlp-insecure.yaml"},
+		cfgMapFuncs: []CfgMapFunc{MoveOTLPInsecureKey},
+	}
+	v, err := pp.Get(context.Background())
+	require.NoError(t, err)
+	assert.False(t, v.IsSet("exporters::otlp::insecure"))
+	assert.Equal(t, true, v.Get("exporters::otlp::tls::insecure"))
+}
+
+func TestMoveOTLPInsecureKey_Custom(t *testing.T) {
+	pp := &converterProvider{
+		wrapped:     &fileParserProvider{fileName: "testdata/otlp-insecure-custom.yaml"},
+		cfgMapFuncs: []CfgMapFunc{MoveOTLPInsecureKey},
+	}
+	v, err := pp.Get(context.Background())
+	require.NoError(t, err)
+	assert.False(t, v.IsSet("exporters::otlp/foo::insecure"))
+	assert.Equal(t, true, v.Get("exporters::otlp/foo::tls::insecure"))
+	assert.Equal(t, true, v.Get("exporters::otlp/foo::tls::insecure_skip_verify"))
+}

internal/configconverter/parser_provider.go

@@ -15,6 +15,7 @@
 package configconverter
 
 import (
+	"context"
 	"fmt"
 
 	"go.opentelemetry.io/collector/config/configparser"
@@ -37,8 +38,8 @@ func ParserProvider(wrapped parserprovider.ParserProvider, funcs ...CfgMapFunc)
 	return &converterProvider{wrapped: wrapped, cfgMapFuncs: funcs}
 }
 
-func (p converterProvider) Get() (*configparser.ConfigMap, error) {
-	cfgMap, err := p.wrapped.Get()
+func (p converterProvider) Get(ctx context.Context) (*configparser.ConfigMap, error) {
+	cfgMap, err := p.wrapped.Get(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("converterProvider.Get(): %w", err)
 	}
@@ -47,9 +48,13 @@ func (p converterProvider) Get() (*configparser.ConfigMap, error) {
 		cfgMap = cfgMapFunc(cfgMap)
 	}
 
-	out := configparser.NewParser()
+	out := configparser.NewConfigMap()
 	for _, k := range cfgMap.AllKeys() {
 		out.Set(k, cfgMap.Get(k))
 	}
 	return out, nil
 }
+
+func (p converterProvider) Close(context.Context) error {
+	return nil
+}

internal/configconverter/parser_provider_test.go

@@ -15,6 +15,7 @@
 package configconverter
 
 import (
+	"context"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -45,7 +46,7 @@ func TestMemLimitBallastRemoverPP(t *testing.T) {
 				wrapped:     &fileParserProvider{fileName: test.fname},
 				cfgMapFuncs: []CfgMapFunc{RemoveBallastKey},
 			}
-			cfgMap, err := pp.Get()
+			cfgMap, err := pp.Get(context.Background())
 			require.NoError(t, err)
 			b := cfgMap.IsSet(test.key)
 			assert.False(t, b)
@@ -57,6 +58,10 @@ type fileParserProvider struct {
 	fileName string
 }
 
-func (fpp fileParserProvider) Get() (*configparser.ConfigMap, error) {
-	return configparser.NewParserFromFile(fpp.fileName)
+func (fpp fileParserProvider) Get(context.Context) (*configparser.ConfigMap, error) {
+	return configparser.NewConfigMapFromFile(fpp.fileName)
+}
+
+func (fpp fileParserProvider) Close(context.Context) error {
+	return nil
 }

internal/configconverter/testdata/otlp-insecure-custom.yaml

@@ -0,0 +1,18 @@
+receivers:
+  hostmetrics:
+    collection_interval: 1s
+    scrapers:
+      cpu:
+exporters:
+  otlp/foo:
+    endpoint: localhost:4317
+    insecure: true
+    tls:
+      insecure_skip_verify: true
+service:
+  pipelines:
+    metrics:
+      receivers:
+        - hostmetrics
+      exporters:
+        - otlp/foo

internal/configconverter/testdata/otlp-insecure.yaml

@@ -0,0 +1,16 @@
+receivers:
+  hostmetrics:
+    collection_interval: 1s
+    scrapers:
+      cpu:
+exporters:
+  otlp:
+    endpoint: localhost:4317
+    insecure: true
+service:
+  pipelines:
+    metrics:
+      receivers:
+        - hostmetrics
+      exporters:
+        - otlp

internal/configprovider/config_source_provider.go

@@ -52,8 +52,8 @@ func NewConfigSourceParserProvider(pp parserprovider.ParserProvider, logger *zap
 // Get returns a config.Parser that wraps the parserprovider.Default() with a parser
 // that can load and inject data from config sources. If there are no config sources
 // in the configuration the returned parser behaves like the parserprovider.Default().
-func (c *configSourceParserProvider) Get() (*configparser.ConfigMap, error) {
-	defaultParser, err := c.pp.Get()
+func (c *configSourceParserProvider) Get(ctx context.Context) (*configparser.ConfigMap, error) {
+	defaultParser, err := c.pp.Get(ctx)
 	if err != nil {
 		return nil, err
 	}