Skip to content

Commit

Permalink
Metrics summary API root spans (grafana#3011)
Browse files Browse the repository at this point in the history
* local blocks processor also save root spans

* Add metric to track spans after filtering
mdisibio authored Oct 11, 2023

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent b20c612 commit 2467e1b
Showing 2 changed files with 16 additions and 6 deletions.
6 changes: 6 additions & 0 deletions modules/generator/processor/localblocks/metrics.go
Original file line number Diff line number Diff line change
@@ -19,6 +19,12 @@ var (
Name: "traces_total",
Help: "Total number of traces created",
}, []string{"tenant"})
metricTotalSpans = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "spans_total",
Help: "Total number of spans after filtering",
}, []string{"tenant"})
metricLiveTraces = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: subsystem,
16 changes: 10 additions & 6 deletions modules/generator/processor/localblocks/processor.go
Original file line number Diff line number Diff line change
@@ -104,14 +104,16 @@ func (p *Processor) PushSpans(_ context.Context, req *tempopb.PushSpansRequest)
p.liveTracesMtx.Lock()
defer p.liveTracesMtx.Unlock()

var count int
before := p.liveTraces.Len()

for _, batch := range req.Batches {
if batch = filterBatch(batch); batch != nil {
if batch, count = filterBatch(batch); batch != nil {
err := p.liveTraces.Push(batch, p.Cfg.MaxLiveTraces)
if errors.Is(err, errMaxExceeded) {
metricDroppedTraces.WithLabelValues(p.tenant, reasonLiveTracesExceeded).Inc()
}
metricTotalSpans.WithLabelValues(p.tenant).Add(float64(count))
}
}

@@ -620,16 +622,17 @@ func (p *Processor) recordBlockBytes() {
metricBlockSize.WithLabelValues(p.tenant).Set(float64(sum))
}

// filterBatch to only spans with kind==server. Does not modify the input
// filterBatch to only root spans or kind==server. Does not modify the input
// but returns a new struct referencing the same input pointers. Returns nil
// if there were no matching spans.
func filterBatch(batch *v1.ResourceSpans) *v1.ResourceSpans {
func filterBatch(batch *v1.ResourceSpans) (*v1.ResourceSpans, int) {
var keep int
var keepSS []*v1.ScopeSpans
for _, ss := range batch.ScopeSpans {

var keepSpans []*v1.Span
for _, s := range ss.Spans {
if s.Kind == v1.Span_SPAN_KIND_SERVER {
if s.Kind == v1.Span_SPAN_KIND_SERVER || len(s.ParentSpanId) == 0 {
keepSpans = append(keepSpans, s)
}
}
@@ -639,17 +642,18 @@ func filterBatch(batch *v1.ResourceSpans) *v1.ResourceSpans {
Scope: ss.Scope,
Spans: keepSpans,
})
keep += len(keepSpans)
}
}

if len(keepSS) > 0 {
return &v1.ResourceSpans{
Resource: batch.Resource,
ScopeSpans: keepSS,
}
}, keep
}

return nil
return nil, 0
}

func metricSeriesToProto(series traceqlmetrics.MetricSeries) []*tempopb.KeyValue {

0 comments on commit 2467e1b

Please sign in to comment.