diff --git a/CHANGELOG.md b/CHANGELOG.md index ff90c5de244..fdcfccfdb36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ * [FEATURE] Introduce list_blocks_concurrency on GCS and S3 backends to control backend load and performance. [#2652](https://github.com/grafana/tempo/pull/2652) (@zalegrala) * [FEATURE] Add per-tenant compaction window [#3129](https://github.com/grafana/tempo/pull/3129) (@zalegrala) * [BUGFIX] Include statusMessage intrinsic attribute in tag search. [#3084](https://github.com/grafana/tempo/pull/3084) (@rcrowe) +* [ENHANCEMENT] Make the trace ID label name configurable for remote written exemplars [#3074](https://github.com/grafana/tempo/pull/3074) * [ENHANCEMENT] Update poller to make use of previous results and reduce backend load. [#2652](https://github.com/grafana/tempo/pull/2652) (@zalegrala) * [ENHANCEMENT] Improve TraceQL regex performance in certain queries. [#3139](https://github.com/grafana/tempo/pull/3139) (@joe-elliott) * [BUGFIX] Readd session token to s3 credentials. [#3144](https://github.com/grafana/tempo/pull/3144) (@farodin91) diff --git a/docs/sources/tempo/configuration/_index.md b/docs/sources/tempo/configuration/_index.md index e9ad084857b..35549633cf8 100644 --- a/docs/sources/tempo/configuration/_index.md +++ b/docs/sources/tempo/configuration/_index.md @@ -1367,6 +1367,10 @@ overrides: # This setting is useful if you wish to test how many active series a tenant will generate, without # actually writing these metrics. [disable_collection: | default = false] + + # Per-user configuration of the trace-id label name. This value will be used as name for the label to store the + # trace ID of exemplars in generated metrics. If not set, the default value "trace_id" will be used. + [trace_id_label_name: | default = "trace_id"] # This option only allows spans with end time that occur within the configured duration to be # considered in metrics generation. diff --git a/modules/generator/overrides_test.go b/modules/generator/overrides_test.go index 4f84edb41ba..32ae2c16856 100644 --- a/modules/generator/overrides_test.go +++ b/modules/generator/overrides_test.go @@ -52,6 +52,10 @@ func (m *mockOverrides) MetricsGeneratorDisableCollection(string) bool { return false } +func (m *mockOverrides) MetricsGenerationTraceIDLabelName(userID string) string { + return "" +} + func (m *mockOverrides) MetricsGeneratorProcessorServiceGraphsHistogramBuckets(string) []float64 { return m.serviceGraphsHistogramBuckets } diff --git a/modules/generator/registry/histogram.go b/modules/generator/registry/histogram.go index 9c6407cef45..27cd95d85ce 100644 --- a/modules/generator/registry/histogram.go +++ b/modules/generator/registry/histogram.go @@ -28,6 +28,8 @@ type histogram struct { onAddSerie func(count uint32) bool onRemoveSerie func(count uint32) + + traceIDLabelName string } type histogramSeries struct { @@ -48,7 +50,7 @@ var ( _ metric = (*histogram)(nil) ) -func newHistogram(name string, buckets []float64, onAddSeries func(uint32) bool, onRemoveSeries func(count uint32)) *histogram { +func newHistogram(name string, buckets []float64, onAddSeries func(uint32) bool, onRemoveSeries func(count uint32), traceIDLabelName string) *histogram { if onAddSeries == nil { onAddSeries = func(uint32) bool { return true @@ -58,6 +60,10 @@ func newHistogram(name string, buckets []float64, onAddSeries func(uint32) bool, onRemoveSeries = func(uint32) {} } + if traceIDLabelName == "" { + traceIDLabelName = "traceID" + } + // add +Inf bucket buckets = append(buckets, math.Inf(1)) @@ -67,15 +73,16 @@ func newHistogram(name string, buckets []float64, onAddSeries func(uint32) bool, } return &histogram{ - metricName: name, - nameCount: fmt.Sprintf("%s_count", name), - nameSum: fmt.Sprintf("%s_sum", name), - nameBucket: fmt.Sprintf("%s_bucket", name), - buckets: buckets, - bucketLabels: bucketLabels, - series: make(map[uint64]*histogramSeries), - onAddSerie: onAddSeries, - onRemoveSerie: onRemoveSeries, + metricName: name, + nameCount: fmt.Sprintf("%s_count", name), + nameSum: fmt.Sprintf("%s_sum", name), + nameBucket: fmt.Sprintf("%s_bucket", name), + buckets: buckets, + bucketLabels: bucketLabels, + series: make(map[uint64]*histogramSeries), + onAddSerie: onAddSeries, + onRemoveSerie: onRemoveSeries, + traceIDLabelName: traceIDLabelName, } } @@ -201,7 +208,7 @@ func (h *histogram) collectMetrics(appender storage.Appender, timeMs int64, exte if ex != "" { _, err = appender.AppendExemplar(ref, lb.Labels(), exemplar.Exemplar{ Labels: []labels.Label{{ - Name: "traceID", + Name: h.traceIDLabelName, Value: ex, }}, Value: s.exemplarValues[i].Load(), diff --git a/modules/generator/registry/histogram_test.go b/modules/generator/registry/histogram_test.go index 0f1159dd0b2..77da6b3e733 100644 --- a/modules/generator/registry/histogram_test.go +++ b/modules/generator/registry/histogram_test.go @@ -19,7 +19,7 @@ func Test_histogram(t *testing.T) { return true } - h := newHistogram("my_histogram", []float64{1.0, 2.0}, onAdd, nil) + h := newHistogram("my_histogram", []float64{1.0, 2.0}, onAdd, nil, "trace_id") h.ObserveWithExemplar(newLabelValueCombo([]string{"label"}, []string{"value-1"}), 1.0, "trace-1", 1.0) h.ObserveWithExemplar(newLabelValueCombo([]string{"label"}, []string{"value-2"}), 1.5, "trace-2", 1.0) @@ -41,12 +41,12 @@ func Test_histogram(t *testing.T) { } expectedExemplars := []exemplarSample{ newExemplar(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "1"}, exemplar.Exemplar{ - Labels: labels.FromMap(map[string]string{"traceID": "trace-1"}), + Labels: labels.FromMap(map[string]string{"trace_id": "trace-1"}), Value: 1.0, Ts: collectionTimeMs, }), newExemplar(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "2"}, exemplar.Exemplar{ - Labels: labels.FromMap(map[string]string{"traceID": "trace-2"}), + Labels: labels.FromMap(map[string]string{"trace_id": "trace-2"}), Value: 1.5, Ts: collectionTimeMs, }), @@ -78,12 +78,12 @@ func Test_histogram(t *testing.T) { } expectedExemplars = []exemplarSample{ newExemplar(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "+Inf"}, exemplar.Exemplar{ - Labels: labels.FromMap(map[string]string{"traceID": "trace-2.2"}), + Labels: labels.FromMap(map[string]string{"trace_id": "trace-2.2"}), Value: 2.5, Ts: collectionTimeMs, }), newExemplar(map[string]string{"__name__": "my_histogram_bucket", "label": "value-3", "le": "+Inf"}, exemplar.Exemplar{ - Labels: labels.FromMap(map[string]string{"traceID": "trace-3"}), + Labels: labels.FromMap(map[string]string{"trace_id": "trace-3"}), Value: 3.0, Ts: collectionTimeMs, }), @@ -116,17 +116,17 @@ func Test_histogram(t *testing.T) { } expectedExemplars = []exemplarSample{ newExemplar(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "+Inf"}, exemplar.Exemplar{ - Labels: labels.FromMap(map[string]string{"traceID": "trace-2.2"}), + Labels: labels.FromMap(map[string]string{"trace_id": "trace-2.2"}), Value: 2.5, Ts: collectionTimeMs, }), newExemplar(map[string]string{"__name__": "my_histogram_bucket", "label": "value-3", "le": "1"}, exemplar.Exemplar{ - Labels: labels.FromMap(map[string]string{"traceID": "trace-3"}), + Labels: labels.FromMap(map[string]string{"trace_id": "trace-3"}), Value: 1.0, Ts: collectionTimeMs, }), newExemplar(map[string]string{"__name__": "my_histogram_bucket", "label": "value-3", "le": "+Inf"}, exemplar.Exemplar{ - Labels: labels.FromMap(map[string]string{"traceID": "trace-3"}), + Labels: labels.FromMap(map[string]string{"trace_id": "trace-3"}), Value: 3.0, Ts: collectionTimeMs, }), @@ -141,7 +141,7 @@ func Test_histogram_cantAdd(t *testing.T) { return canAdd } - h := newHistogram("my_histogram", []float64{1.0, 2.0}, onAdd, nil) + h := newHistogram("my_histogram", []float64{1.0, 2.0}, onAdd, nil, "") // allow adding new series canAdd = true @@ -193,7 +193,7 @@ func Test_histogram_removeStaleSeries(t *testing.T) { removedSeries++ } - h := newHistogram("my_histogram", []float64{1.0, 2.0}, nil, onRemove) + h := newHistogram("my_histogram", []float64{1.0, 2.0}, nil, onRemove, "") timeMs := time.Now().UnixMilli() h.ObserveWithExemplar(newLabelValueCombo([]string{"label"}, []string{"value-1"}), 1.0, "", 1.0) @@ -240,7 +240,7 @@ func Test_histogram_removeStaleSeries(t *testing.T) { } func Test_histogram_externalLabels(t *testing.T) { - h := newHistogram("my_histogram", []float64{1.0, 2.0}, nil, nil) + h := newHistogram("my_histogram", []float64{1.0, 2.0}, nil, nil, "") h.ObserveWithExemplar(newLabelValueCombo([]string{"label"}, []string{"value-1"}), 1.0, "", 1.0) h.ObserveWithExemplar(newLabelValueCombo([]string{"label"}, []string{"value-2"}), 1.5, "", 1.0) @@ -262,7 +262,7 @@ func Test_histogram_externalLabels(t *testing.T) { } func Test_histogram_concurrencyDataRace(t *testing.T) { - h := newHistogram("my_histogram", []float64{1.0, 2.0}, nil, nil) + h := newHistogram("my_histogram", []float64{1.0, 2.0}, nil, nil, "") end := make(chan struct{}) @@ -308,7 +308,7 @@ func Test_histogram_concurrencyDataRace(t *testing.T) { } func Test_histogram_concurrencyCorrectness(t *testing.T) { - h := newHistogram("my_histogram", []float64{1.0, 2.0}, nil, nil) + h := newHistogram("my_histogram", []float64{1.0, 2.0}, nil, nil, "") var wg sync.WaitGroup end := make(chan struct{}) @@ -348,7 +348,7 @@ func Test_histogram_concurrencyCorrectness(t *testing.T) { } func Test_histogram_span_multiplier(t *testing.T) { - h := newHistogram("my_histogram", []float64{1.0, 2.0}, nil, nil) + h := newHistogram("my_histogram", []float64{1.0, 2.0}, nil, nil, "") h.ObserveWithExemplar(newLabelValueCombo([]string{"label"}, []string{"value-1"}), 1.0, "", 1.5) h.ObserveWithExemplar(newLabelValueCombo([]string{"label"}, []string{"value-1"}), 2.0, "", 5) diff --git a/modules/generator/registry/overrides.go b/modules/generator/registry/overrides.go index 89f763006f6..c308637be53 100644 --- a/modules/generator/registry/overrides.go +++ b/modules/generator/registry/overrides.go @@ -10,6 +10,7 @@ type Overrides interface { MetricsGeneratorMaxActiveSeries(userID string) uint32 MetricsGeneratorCollectionInterval(userID string) time.Duration MetricsGeneratorDisableCollection(userID string) bool + MetricsGenerationTraceIDLabelName(userID string) string } var _ Overrides = (overrides.Interface)(nil) diff --git a/modules/generator/registry/registry.go b/modules/generator/registry/registry.go index 58eefba8e33..12b42d916f3 100644 --- a/modules/generator/registry/registry.go +++ b/modules/generator/registry/registry.go @@ -144,7 +144,7 @@ func (r *ManagedRegistry) NewCounter(name string) Counter { } func (r *ManagedRegistry) NewHistogram(name string, buckets []float64) Histogram { - h := newHistogram(name, buckets, r.onAddMetricSeries, r.onRemoveMetricSeries) + h := newHistogram(name, buckets, r.onAddMetricSeries, r.onRemoveMetricSeries, r.overrides.MetricsGenerationTraceIDLabelName(r.tenant)) r.registerMetric(h) return h } diff --git a/modules/generator/registry/registry_test.go b/modules/generator/registry/registry_test.go index df5cd345964..a13c83450f8 100644 --- a/modules/generator/registry/registry_test.go +++ b/modules/generator/registry/registry_test.go @@ -297,6 +297,10 @@ func (m *mockOverrides) MetricsGeneratorDisableCollection(string) bool { return m.disableCollection } +func (m *mockOverrides) MetricsGenerationTraceIDLabelName(string) string { + return "" +} + func mustGetHostname() string { hostname, _ := os.Hostname() return hostname diff --git a/modules/overrides/config.go b/modules/overrides/config.go index 7fcb2590c24..47abba9cd43 100644 --- a/modules/overrides/config.go +++ b/modules/overrides/config.go @@ -113,6 +113,7 @@ type MetricsGeneratorOverrides struct { MaxActiveSeries uint32 `yaml:"max_active_series,omitempty" json:"max_active_series,omitempty"` CollectionInterval time.Duration `yaml:"collection_interval,omitempty" json:"collection_interval,omitempty"` DisableCollection bool `yaml:"disable_collection,omitempty" json:"disable_collection,omitempty"` + TraceIDLabelName string `yaml:"trace_id_label_name,omitempty" json:"trace_id_label_name,omitempty"` Forwarder ForwarderOverrides `yaml:"forwarder,omitempty" json:"forwarder,omitempty"` diff --git a/modules/overrides/config_legacy.go b/modules/overrides/config_legacy.go index 442f14f798b..6e4685061f0 100644 --- a/modules/overrides/config_legacy.go +++ b/modules/overrides/config_legacy.go @@ -27,6 +27,7 @@ func (c *Overrides) toLegacy() LegacyOverrides { MetricsGeneratorMaxActiveSeries: c.MetricsGenerator.MaxActiveSeries, MetricsGeneratorCollectionInterval: c.MetricsGenerator.CollectionInterval, MetricsGeneratorDisableCollection: c.MetricsGenerator.DisableCollection, + MetricsGeneratorTraceIDLabelName: c.MetricsGenerator.TraceIDLabelName, MetricsGeneratorForwarderQueueSize: c.MetricsGenerator.Forwarder.QueueSize, MetricsGeneratorForwarderWorkers: c.MetricsGenerator.Forwarder.Workers, MetricsGeneratorProcessorServiceGraphsHistogramBuckets: c.MetricsGenerator.Processor.ServiceGraphs.HistogramBuckets, @@ -82,6 +83,7 @@ type LegacyOverrides struct { MetricsGeneratorMaxActiveSeries uint32 `yaml:"metrics_generator_max_active_series" json:"metrics_generator_max_active_series"` MetricsGeneratorCollectionInterval time.Duration `yaml:"metrics_generator_collection_interval" json:"metrics_generator_collection_interval"` MetricsGeneratorDisableCollection bool `yaml:"metrics_generator_disable_collection" json:"metrics_generator_disable_collection"` + MetricsGeneratorTraceIDLabelName string `yaml:"metrics_generator_trace_id_label_name" json:"metrics_generator_trace_id_label_name"` MetricsGeneratorForwarderQueueSize int `yaml:"metrics_generator_forwarder_queue_size" json:"metrics_generator_forwarder_queue_size"` MetricsGeneratorForwarderWorkers int `yaml:"metrics_generator_forwarder_workers" json:"metrics_generator_forwarder_workers"` MetricsGeneratorProcessorServiceGraphsHistogramBuckets []float64 `yaml:"metrics_generator_processor_service_graphs_histogram_buckets" json:"metrics_generator_processor_service_graphs_histogram_buckets"` @@ -146,6 +148,7 @@ func (l *LegacyOverrides) toNewLimits() Overrides { MaxActiveSeries: l.MetricsGeneratorMaxActiveSeries, CollectionInterval: l.MetricsGeneratorCollectionInterval, DisableCollection: l.MetricsGeneratorDisableCollection, + TraceIDLabelName: l.MetricsGeneratorTraceIDLabelName, IngestionSlack: l.MetricsGeneratorIngestionSlack, Forwarder: ForwarderOverrides{ QueueSize: l.MetricsGeneratorForwarderQueueSize, diff --git a/modules/overrides/interface.go b/modules/overrides/interface.go index 3af54ad44c4..6e389e804ae 100644 --- a/modules/overrides/interface.go +++ b/modules/overrides/interface.go @@ -38,6 +38,7 @@ type Interface interface { MetricsGeneratorMaxActiveSeries(userID string) uint32 MetricsGeneratorCollectionInterval(userID string) time.Duration MetricsGeneratorDisableCollection(userID string) bool + MetricsGenerationTraceIDLabelName(userID string) string MetricsGeneratorForwarderQueueSize(userID string) int MetricsGeneratorForwarderWorkers(userID string) int MetricsGeneratorProcessorServiceGraphsHistogramBuckets(userID string) []float64 diff --git a/modules/overrides/runtime_config_overrides.go b/modules/overrides/runtime_config_overrides.go index a8b9a94c760..55b3348272b 100644 --- a/modules/overrides/runtime_config_overrides.go +++ b/modules/overrides/runtime_config_overrides.go @@ -335,6 +335,12 @@ func (o *runtimeConfigOverridesManager) MetricsGeneratorDisableCollection(userID return o.getOverridesForUser(userID).MetricsGenerator.DisableCollection } +// MetricsGenerationTraceIDLabelName is the label name used for the trace ID in metrics. +// "TraceID" is used if no value is provided. +func (o *runtimeConfigOverridesManager) MetricsGenerationTraceIDLabelName(userID string) string { + return o.getOverridesForUser(userID).MetricsGenerator.TraceIDLabelName +} + // MetricsGeneratorForwarderQueueSize is the size of the buffer of requests to send to the metrics-generator // from the distributor for this tenant. func (o *runtimeConfigOverridesManager) MetricsGeneratorForwarderQueueSize(userID string) int { diff --git a/modules/overrides/user_configurable_overrides_test.go b/modules/overrides/user_configurable_overrides_test.go index ee463bcd7f9..19afa99cd3a 100644 --- a/modules/overrides/user_configurable_overrides_test.go +++ b/modules/overrides/user_configurable_overrides_test.go @@ -402,6 +402,7 @@ func TestUserConfigOverridesManager_MergeRuntimeConfig(t *testing.T) { assert.Equal(t, mgr.MetricsGeneratorMaxActiveSeries(tenantID), baseMgr.MetricsGeneratorMaxActiveSeries(tenantID)) assert.Equal(t, mgr.MetricsGeneratorCollectionInterval(tenantID), baseMgr.MetricsGeneratorCollectionInterval(tenantID)) assert.Equal(t, mgr.MetricsGeneratorDisableCollection(tenantID), baseMgr.MetricsGeneratorDisableCollection(tenantID)) + assert.Equal(t, mgr.MetricsGenerationTraceIDLabelName(tenantID), baseMgr.MetricsGenerationTraceIDLabelName(tenantID)) assert.Equal(t, mgr.MetricsGeneratorForwarderQueueSize(tenantID), baseMgr.MetricsGeneratorForwarderQueueSize(tenantID)) assert.Equal(t, mgr.MetricsGeneratorForwarderWorkers(tenantID), baseMgr.MetricsGeneratorForwarderWorkers(tenantID)) assert.Equal(t, mgr.MetricsGeneratorProcessorServiceGraphsHistogramBuckets(tenantID), baseMgr.MetricsGeneratorProcessorServiceGraphsHistogramBuckets(tenantID))