tetragon: Add size histograms for data events

Adding good/bad histograms to keep track of data events sizes. It's enabled with new --enable-data-events-size-metric option. Signed-off-by: Jiri Olsa <[email protected]>
cilium · Jul 16, 2023 · e9cd5dc · e9cd5dc
1 parent bcf1d8f
commit e9cd5dc
Show file tree

Hide file tree

Showing 5 changed files with 33 additions and 0 deletions.
diff --git a/cmd/tetragon/flags.go b/cmd/tetragon/flags.go
@@ -77,6 +77,8 @@ const (
 	keyEnablePidSetFilter = "enable-pid-set-filter"
 
 	keyEnableMsgHandlingLatency = "enable-msg-handling-latency"
+
+	keyEnableDataEventsSizeMetric = "enable-data-events-size-metric"
 )
 
 func readAndSetFlags() {
@@ -134,6 +136,7 @@ func readAndSetFlags() {
 	option.Config.EnablePolicyFilter = viper.GetBool(keyEnablePolicyFilter)
 	option.Config.EnablePolicyFilterDebug = viper.GetBool(keyEnablePolicyFilterDebug)
 	option.Config.EnableMsgHandlingLatency = viper.GetBool(keyEnableMsgHandlingLatency)
+	option.Config.EnableDataEventsSizeMetric = viper.GetBool(keyEnableDataEventsSizeMetric)
 
 	option.Config.EnablePidSetFilter = viper.GetBool(keyEnablePidSetFilter)
 

diff --git a/cmd/tetragon/main.go b/cmd/tetragon/main.go
@@ -683,6 +683,8 @@ func execute() error {
 
 	flags.Bool(keyEnableMsgHandlingLatency, false, "Enable metrics for message handling latency")
 
+	flags.Bool(keyEnableDataEventsSizeMetric, false, "Enable metrics for data events size")
+
 	viper.BindPFlags(flags)
 	return rootCmd.Execute()
 }

diff --git a/pkg/observer/data.go b/pkg/observer/data.go
@@ -66,9 +66,12 @@ func DataGet(desc dataapi.DataEventDesc) ([]byte, error) {
 	// make sure we did not loose anything on the way through ring buffer
 	if len(data) != int(desc.Size-desc.Leftover) {
 		DataEventMetricInc(DataEventBad)
+		DataEventMetricSizeBad(desc.Size)
 		return nil, fmt.Errorf("failed to get correct data for id: %v", desc.Id)
 	}
 
+	DataEventMetricSizeOk(desc.Size)
+
 	dataMap.Remove(desc.Id)
 	logger.GetLogger().WithFields(nil).Tracef("Data message used id %v, data len %v", desc.Id, len(data))
 	DataEventMetricInc(DataEventMatched)

diff --git a/pkg/observer/data_stats.go b/pkg/observer/data_stats.go
@@ -2,6 +2,7 @@ package observer
 
 import (
 	"github.com/cilium/tetragon/pkg/metrics/consts"
+	"github.com/cilium/tetragon/pkg/option"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 )
@@ -14,6 +15,14 @@ var (
 		Help:        "Data event statistics. For internal use only.",
 		ConstLabels: nil,
 	}, []string{"event"})
+
+	DataEventSizeHist = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace:   consts.MetricsNamespace,
+		Name:        "data_event_stats_size",
+		Help:        "The size of received data events.",
+		Buckets:     prometheus.LinearBuckets(1000, 2000, 20),
+		ConstLabels: nil,
+	}, []string{"op"})
 )
 
 type DataEventType int
@@ -40,3 +49,17 @@ var DataEventTypeStrings = map[DataEventType]string{
 func DataEventMetricInc(event DataEventType) {
 	DataEventStats.WithLabelValues(DataEventTypeStrings[event]).Inc()
 }
+
+func addSize(op string, size uint32) {
+	if option.Config.EnableDataEventsSizeMetric {
+		DataEventSizeHist.WithLabelValues(op).Observe(float64(size))
+	}
+}
+
+func DataEventMetricSizeOk(size uint32) {
+	addSize("ok", size)
+}
+
+func DataEventMetricSizeBad(size uint32) {
+	addSize("bad", size)
+}
diff --git a/pkg/option/config.go b/pkg/option/config.go
@@ -75,6 +75,8 @@ type config struct {
 	EnablePidSetFilter bool
 
 	EnableMsgHandlingLatency bool
+
+	EnableDataEventsSizeMetric bool
 }
 
 var (