make the traceID label name configurable (grafana#3074)

* make the traceID label name configurable, because otel specifies trace_id - see https://opentelemetry.io/docs/specs/otel/compatibility/prometheus_and_openmetrics/#exemplars

* make the traceID label name configurable, because otel specifies trace_id - see https://opentelemetry.io/docs/specs/otel/compatibility/prometheus_and_openmetrics/#exemplars

* make the traceID label name configurable, because otel specifies trace_id - see https://opentelemetry.io/docs/specs/otel/compatibility/prometheus_and_openmetrics/#exemplars

* make the traceID label name configurable, because otel specifies trace_id - see https://opentelemetry.io/docs/specs/otel/compatibility/prometheus_and_openmetrics/#exemplars

* make the traceID label name configurable, because otel specifies trace_id - see https://opentelemetry.io/docs/specs/otel/compatibility/prometheus_and_openmetrics/#exemplars

* make the traceID label name configurable, because otel specifies trace_id - see https://opentelemetry.io/docs/specs/otel/compatibility/prometheus_and_openmetrics/#exemplars

CHANGELOG.md

@@ -7,6 +7,7 @@
 * [FEATURE] Introduce list_blocks_concurrency on GCS and S3 backends to control backend load and performance. [#2652](https://github.com/grafana/tempo/pull/2652) (@zalegrala)
 * [FEATURE] Add per-tenant compaction window [#3129](https://github.com/grafana/tempo/pull/3129) (@zalegrala)
 * [BUGFIX] Include statusMessage intrinsic attribute in tag search. [#3084](https://github.com/grafana/tempo/pull/3084) (@rcrowe)
+* [ENHANCEMENT] Make the trace ID label name configurable for remote written exemplars [#3074](https://github.com/grafana/tempo/pull/3074)
 * [ENHANCEMENT] Update poller to make use of previous results and reduce backend load. [#2652](https://github.com/grafana/tempo/pull/2652) (@zalegrala)
 * [ENHANCEMENT] Improve TraceQL regex performance in certain queries. [#3139](https://github.com/grafana/tempo/pull/3139) (@joe-elliott)
 * [BUGFIX] Readd session token to s3 credentials. [#3144](https://github.com/grafana/tempo/pull/3144) (@farodin91)

docs/sources/tempo/configuration/_index.md

@@ -1367,6 +1367,10 @@ overrides:
       # This setting is useful if you wish to test how many active series a tenant will generate, without
       # actually writing these metrics.
       [disable_collection: <bool> | default = false]
+
+      # Per-user configuration of the trace-id label name. This value will be used as name for the label to store the
+      # trace ID of exemplars in generated metrics. If not set, the default value "trace_id" will be used.  
+      [trace_id_label_name: <string> | default = "trace_id"]  
 
       # This option only allows spans with end time that occur within the configured duration to be
       # considered in metrics generation.

modules/generator/overrides_test.go

@@ -52,6 +52,10 @@ func (m *mockOverrides) MetricsGeneratorDisableCollection(string) bool {
 	return false
 }
 
+func (m *mockOverrides) MetricsGenerationTraceIDLabelName(userID string) string {
+	return ""
+}
+
 func (m *mockOverrides) MetricsGeneratorProcessorServiceGraphsHistogramBuckets(string) []float64 {
 	return m.serviceGraphsHistogramBuckets
 }

modules/generator/registry/histogram.go

@@ -28,6 +28,8 @@ type histogram struct {
 
 	onAddSerie    func(count uint32) bool
 	onRemoveSerie func(count uint32)
+
+	traceIDLabelName string
 }
 
 type histogramSeries struct {
@@ -48,7 +50,7 @@ var (
 	_ metric    = (*histogram)(nil)
 )
 
-func newHistogram(name string, buckets []float64, onAddSeries func(uint32) bool, onRemoveSeries func(count uint32)) *histogram {
+func newHistogram(name string, buckets []float64, onAddSeries func(uint32) bool, onRemoveSeries func(count uint32), traceIDLabelName string) *histogram {
 	if onAddSeries == nil {
 		onAddSeries = func(uint32) bool {
 			return true
@@ -58,6 +60,10 @@ func newHistogram(name string, buckets []float64, onAddSeries func(uint32) bool,
 		onRemoveSeries = func(uint32) {}
 	}
 
+	if traceIDLabelName == "" {
+		traceIDLabelName = "traceID"
+	}
+
 	// add +Inf bucket
 	buckets = append(buckets, math.Inf(1))
 
@@ -67,15 +73,16 @@ func newHistogram(name string, buckets []float64, onAddSeries func(uint32) bool,
 	}
 
 	return &histogram{
-		metricName:    name,
-		nameCount:     fmt.Sprintf("%s_count", name),
-		nameSum:       fmt.Sprintf("%s_sum", name),
-		nameBucket:    fmt.Sprintf("%s_bucket", name),
-		buckets:       buckets,
-		bucketLabels:  bucketLabels,
-		series:        make(map[uint64]*histogramSeries),
-		onAddSerie:    onAddSeries,
-		onRemoveSerie: onRemoveSeries,
+		metricName:       name,
+		nameCount:        fmt.Sprintf("%s_count", name),
+		nameSum:          fmt.Sprintf("%s_sum", name),
+		nameBucket:       fmt.Sprintf("%s_bucket", name),
+		buckets:          buckets,
+		bucketLabels:     bucketLabels,
+		series:           make(map[uint64]*histogramSeries),
+		onAddSerie:       onAddSeries,
+		onRemoveSerie:    onRemoveSeries,
+		traceIDLabelName: traceIDLabelName,
 	}
 }
 
@@ -201,7 +208,7 @@ func (h *histogram) collectMetrics(appender storage.Appender, timeMs int64, exte
 			if ex != "" {
 				_, err = appender.AppendExemplar(ref, lb.Labels(), exemplar.Exemplar{
 					Labels: []labels.Label{{
-						Name:  "traceID",
+						Name:  h.traceIDLabelName,
 						Value: ex,
 					}},
 					Value: s.exemplarValues[i].Load(),

modules/generator/registry/histogram_test.go

@@ -19,7 +19,7 @@ func Test_histogram(t *testing.T) {
 		return true
 	}
 
-	h := newHistogram("my_histogram", []float64{1.0, 2.0}, onAdd, nil)
+	h := newHistogram("my_histogram", []float64{1.0, 2.0}, onAdd, nil, "trace_id")
 
 	h.ObserveWithExemplar(newLabelValueCombo([]string{"label"}, []string{"value-1"}), 1.0, "trace-1", 1.0)
 	h.ObserveWithExemplar(newLabelValueCombo([]string{"label"}, []string{"value-2"}), 1.5, "trace-2", 1.0)
@@ -41,12 +41,12 @@ func Test_histogram(t *testing.T) {
 	}
 	expectedExemplars := []exemplarSample{
 		newExemplar(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "1"}, exemplar.Exemplar{
-			Labels: labels.FromMap(map[string]string{"traceID": "trace-1"}),
+			Labels: labels.FromMap(map[string]string{"trace_id": "trace-1"}),
 			Value:  1.0,
 			Ts:     collectionTimeMs,
 		}),
 		newExemplar(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "2"}, exemplar.Exemplar{
-			Labels: labels.FromMap(map[string]string{"traceID": "trace-2"}),
+			Labels: labels.FromMap(map[string]string{"trace_id": "trace-2"}),
 			Value:  1.5,
 			Ts:     collectionTimeMs,
 		}),
@@ -78,12 +78,12 @@ func Test_histogram(t *testing.T) {
 	}
 	expectedExemplars = []exemplarSample{
 		newExemplar(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "+Inf"}, exemplar.Exemplar{
-			Labels: labels.FromMap(map[string]string{"traceID": "trace-2.2"}),
+			Labels: labels.FromMap(map[string]string{"trace_id": "trace-2.2"}),
 			Value:  2.5,
 			Ts:     collectionTimeMs,
 		}),
 		newExemplar(map[string]string{"__name__": "my_histogram_bucket", "label": "value-3", "le": "+Inf"}, exemplar.Exemplar{
-			Labels: labels.FromMap(map[string]string{"traceID": "trace-3"}),
+			Labels: labels.FromMap(map[string]string{"trace_id": "trace-3"}),
 			Value:  3.0,
 			Ts:     collectionTimeMs,
 		}),
@@ -116,17 +116,17 @@ func Test_histogram(t *testing.T) {
 	}
 	expectedExemplars = []exemplarSample{
 		newExemplar(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "+Inf"}, exemplar.Exemplar{
-			Labels: labels.FromMap(map[string]string{"traceID": "trace-2.2"}),
+			Labels: labels.FromMap(map[string]string{"trace_id": "trace-2.2"}),
 			Value:  2.5,
 			Ts:     collectionTimeMs,
 		}),
 		newExemplar(map[string]string{"__name__": "my_histogram_bucket", "label": "value-3", "le": "1"}, exemplar.Exemplar{
-			Labels: labels.FromMap(map[string]string{"traceID": "trace-3"}),
+			Labels: labels.FromMap(map[string]string{"trace_id": "trace-3"}),
 			Value:  1.0,
 			Ts:     collectionTimeMs,
 		}),
 		newExemplar(map[string]string{"__name__": "my_histogram_bucket", "label": "value-3", "le": "+Inf"}, exemplar.Exemplar{
-			Labels: labels.FromMap(map[string]string{"traceID": "trace-3"}),
+			Labels: labels.FromMap(map[string]string{"trace_id": "trace-3"}),
 			Value:  3.0,
 			Ts:     collectionTimeMs,
 		}),
@@ -141,7 +141,7 @@ func Test_histogram_cantAdd(t *testing.T) {
 		return canAdd
 	}
 
-	h := newHistogram("my_histogram", []float64{1.0, 2.0}, onAdd, nil)
+	h := newHistogram("my_histogram", []float64{1.0, 2.0}, onAdd, nil, "")
 
 	// allow adding new series
 	canAdd = true
@@ -193,7 +193,7 @@ func Test_histogram_removeStaleSeries(t *testing.T) {
 		removedSeries++
 	}
 
-	h := newHistogram("my_histogram", []float64{1.0, 2.0}, nil, onRemove)
+	h := newHistogram("my_histogram", []float64{1.0, 2.0}, nil, onRemove, "")
 
 	timeMs := time.Now().UnixMilli()
 	h.ObserveWithExemplar(newLabelValueCombo([]string{"label"}, []string{"value-1"}), 1.0, "", 1.0)
@@ -240,7 +240,7 @@ func Test_histogram_removeStaleSeries(t *testing.T) {
 }
 
 func Test_histogram_externalLabels(t *testing.T) {
-	h := newHistogram("my_histogram", []float64{1.0, 2.0}, nil, nil)
+	h := newHistogram("my_histogram", []float64{1.0, 2.0}, nil, nil, "")
 
 	h.ObserveWithExemplar(newLabelValueCombo([]string{"label"}, []string{"value-1"}), 1.0, "", 1.0)
 	h.ObserveWithExemplar(newLabelValueCombo([]string{"label"}, []string{"value-2"}), 1.5, "", 1.0)
@@ -262,7 +262,7 @@ func Test_histogram_externalLabels(t *testing.T) {
 }
 
 func Test_histogram_concurrencyDataRace(t *testing.T) {
-	h := newHistogram("my_histogram", []float64{1.0, 2.0}, nil, nil)
+	h := newHistogram("my_histogram", []float64{1.0, 2.0}, nil, nil, "")
 
 	end := make(chan struct{})
 
@@ -308,7 +308,7 @@ func Test_histogram_concurrencyDataRace(t *testing.T) {
 }
 
 func Test_histogram_concurrencyCorrectness(t *testing.T) {
-	h := newHistogram("my_histogram", []float64{1.0, 2.0}, nil, nil)
+	h := newHistogram("my_histogram", []float64{1.0, 2.0}, nil, nil, "")
 
 	var wg sync.WaitGroup
 	end := make(chan struct{})
@@ -348,7 +348,7 @@ func Test_histogram_concurrencyCorrectness(t *testing.T) {
 }
 
 func Test_histogram_span_multiplier(t *testing.T) {
-	h := newHistogram("my_histogram", []float64{1.0, 2.0}, nil, nil)
+	h := newHistogram("my_histogram", []float64{1.0, 2.0}, nil, nil, "")
 	h.ObserveWithExemplar(newLabelValueCombo([]string{"label"}, []string{"value-1"}), 1.0, "", 1.5)
 	h.ObserveWithExemplar(newLabelValueCombo([]string{"label"}, []string{"value-1"}), 2.0, "", 5)
 

modules/generator/registry/overrides.go

@@ -10,6 +10,7 @@ type Overrides interface {
 	MetricsGeneratorMaxActiveSeries(userID string) uint32
 	MetricsGeneratorCollectionInterval(userID string) time.Duration
 	MetricsGeneratorDisableCollection(userID string) bool
+	MetricsGenerationTraceIDLabelName(userID string) string
 }
 
 var _ Overrides = (overrides.Interface)(nil)

modules/generator/registry/registry.go

@@ -144,7 +144,7 @@ func (r *ManagedRegistry) NewCounter(name string) Counter {
 }
 
 func (r *ManagedRegistry) NewHistogram(name string, buckets []float64) Histogram {
-	h := newHistogram(name, buckets, r.onAddMetricSeries, r.onRemoveMetricSeries)
+	h := newHistogram(name, buckets, r.onAddMetricSeries, r.onRemoveMetricSeries, r.overrides.MetricsGenerationTraceIDLabelName(r.tenant))
 	r.registerMetric(h)
 	return h
 }

modules/generator/registry/registry_test.go

@@ -297,6 +297,10 @@ func (m *mockOverrides) MetricsGeneratorDisableCollection(string) bool {
 	return m.disableCollection
 }
 
+func (m *mockOverrides) MetricsGenerationTraceIDLabelName(string) string {
+	return ""
+}
+
 func mustGetHostname() string {
 	hostname, _ := os.Hostname()
 	return hostname

modules/overrides/config.go

@@ -113,6 +113,7 @@ type MetricsGeneratorOverrides struct {
 	MaxActiveSeries    uint32              `yaml:"max_active_series,omitempty" json:"max_active_series,omitempty"`
 	CollectionInterval time.Duration       `yaml:"collection_interval,omitempty" json:"collection_interval,omitempty"`
 	DisableCollection  bool                `yaml:"disable_collection,omitempty" json:"disable_collection,omitempty"`
+	TraceIDLabelName   string              `yaml:"trace_id_label_name,omitempty" json:"trace_id_label_name,omitempty"`
 
 	Forwarder ForwarderOverrides `yaml:"forwarder,omitempty" json:"forwarder,omitempty"`
 

modules/overrides/config_legacy.go

@@ -27,6 +27,7 @@ func (c *Overrides) toLegacy() LegacyOverrides {
 		MetricsGeneratorMaxActiveSeries:                                  c.MetricsGenerator.MaxActiveSeries,
 		MetricsGeneratorCollectionInterval:                               c.MetricsGenerator.CollectionInterval,
 		MetricsGeneratorDisableCollection:                                c.MetricsGenerator.DisableCollection,
+		MetricsGeneratorTraceIDLabelName:                                 c.MetricsGenerator.TraceIDLabelName,
 		MetricsGeneratorForwarderQueueSize:                               c.MetricsGenerator.Forwarder.QueueSize,
 		MetricsGeneratorForwarderWorkers:                                 c.MetricsGenerator.Forwarder.Workers,
 		MetricsGeneratorProcessorServiceGraphsHistogramBuckets:           c.MetricsGenerator.Processor.ServiceGraphs.HistogramBuckets,
@@ -82,6 +83,7 @@ type LegacyOverrides struct {
 	MetricsGeneratorMaxActiveSeries                                  uint32                           `yaml:"metrics_generator_max_active_series" json:"metrics_generator_max_active_series"`
 	MetricsGeneratorCollectionInterval                               time.Duration                    `yaml:"metrics_generator_collection_interval" json:"metrics_generator_collection_interval"`
 	MetricsGeneratorDisableCollection                                bool                             `yaml:"metrics_generator_disable_collection" json:"metrics_generator_disable_collection"`
+	MetricsGeneratorTraceIDLabelName                                 string                           `yaml:"metrics_generator_trace_id_label_name" json:"metrics_generator_trace_id_label_name"`
 	MetricsGeneratorForwarderQueueSize                               int                              `yaml:"metrics_generator_forwarder_queue_size" json:"metrics_generator_forwarder_queue_size"`
 	MetricsGeneratorForwarderWorkers                                 int                              `yaml:"metrics_generator_forwarder_workers" json:"metrics_generator_forwarder_workers"`
 	MetricsGeneratorProcessorServiceGraphsHistogramBuckets           []float64                        `yaml:"metrics_generator_processor_service_graphs_histogram_buckets" json:"metrics_generator_processor_service_graphs_histogram_buckets"`
@@ -146,6 +148,7 @@ func (l *LegacyOverrides) toNewLimits() Overrides {
 			MaxActiveSeries:    l.MetricsGeneratorMaxActiveSeries,
 			CollectionInterval: l.MetricsGeneratorCollectionInterval,
 			DisableCollection:  l.MetricsGeneratorDisableCollection,
+			TraceIDLabelName:   l.MetricsGeneratorTraceIDLabelName,
 			IngestionSlack:     l.MetricsGeneratorIngestionSlack,
 			Forwarder: ForwarderOverrides{
 				QueueSize: l.MetricsGeneratorForwarderQueueSize,

modules/overrides/interface.go

@@ -38,6 +38,7 @@ type Interface interface {
 	MetricsGeneratorMaxActiveSeries(userID string) uint32
 	MetricsGeneratorCollectionInterval(userID string) time.Duration
 	MetricsGeneratorDisableCollection(userID string) bool
+	MetricsGenerationTraceIDLabelName(userID string) string
 	MetricsGeneratorForwarderQueueSize(userID string) int
 	MetricsGeneratorForwarderWorkers(userID string) int
 	MetricsGeneratorProcessorServiceGraphsHistogramBuckets(userID string) []float64

modules/overrides/runtime_config_overrides.go

@@ -335,6 +335,12 @@ func (o *runtimeConfigOverridesManager) MetricsGeneratorDisableCollection(userID
 	return o.getOverridesForUser(userID).MetricsGenerator.DisableCollection
 }
 
+// MetricsGenerationTraceIDLabelName is the label name used for the trace ID in metrics.
+// "TraceID" is used if no value is provided.
+func (o *runtimeConfigOverridesManager) MetricsGenerationTraceIDLabelName(userID string) string {
+	return o.getOverridesForUser(userID).MetricsGenerator.TraceIDLabelName
+}
+
 // MetricsGeneratorForwarderQueueSize is the size of the buffer of requests to send to the metrics-generator
 // from the distributor for this tenant.
 func (o *runtimeConfigOverridesManager) MetricsGeneratorForwarderQueueSize(userID string) int {

modules/overrides/user_configurable_overrides_test.go

@@ -402,6 +402,7 @@ func TestUserConfigOverridesManager_MergeRuntimeConfig(t *testing.T) {
 	assert.Equal(t, mgr.MetricsGeneratorMaxActiveSeries(tenantID), baseMgr.MetricsGeneratorMaxActiveSeries(tenantID))
 	assert.Equal(t, mgr.MetricsGeneratorCollectionInterval(tenantID), baseMgr.MetricsGeneratorCollectionInterval(tenantID))
 	assert.Equal(t, mgr.MetricsGeneratorDisableCollection(tenantID), baseMgr.MetricsGeneratorDisableCollection(tenantID))
+	assert.Equal(t, mgr.MetricsGenerationTraceIDLabelName(tenantID), baseMgr.MetricsGenerationTraceIDLabelName(tenantID))
 	assert.Equal(t, mgr.MetricsGeneratorForwarderQueueSize(tenantID), baseMgr.MetricsGeneratorForwarderQueueSize(tenantID))
 	assert.Equal(t, mgr.MetricsGeneratorForwarderWorkers(tenantID), baseMgr.MetricsGeneratorForwarderWorkers(tenantID))
 	assert.Equal(t, mgr.MetricsGeneratorProcessorServiceGraphsHistogramBuckets(tenantID), baseMgr.MetricsGeneratorProcessorServiceGraphsHistogramBuckets(tenantID))