Introduce eBPF prog coverage

Signed-off-by: talon <[email protected]>

daemon/options/bpf.go

@@ -27,12 +27,13 @@ import (
 )
 
 type BpfConfig struct {
-	Mode             string
-	BpfFsPath        string
-	Cgroup2Path      string
-	EnableMda        bool
-	BpfVerifyLogSize int
-	EnableBpfLog     bool
+	Mode              string
+	BpfFsPath         string
+	Cgroup2Path       string
+	EnableMda         bool
+	BpfVerifyLogSize  int
+	EnableBpfLog      bool
+	EnableBpfCoverage bool
 }
 
 func (c *BpfConfig) AttachFlags(cmd *cobra.Command) {
@@ -41,6 +42,7 @@ func (c *BpfConfig) AttachFlags(cmd *cobra.Command) {
 	cmd.PersistentFlags().StringVar(&c.Mode, "mode", "workload", "controller plane mode, valid values are [ads, workload]")
 	cmd.PersistentFlags().BoolVar(&c.EnableMda, "enable-mda", false, "enable mda")
 	cmd.PersistentFlags().BoolVar(&c.EnableBpfLog, "enable-bpf-log", false, "enable ebpf log in daemon process")
+	cmd.PersistentFlags().BoolVar(&c.EnableBpfCoverage, "enable-bpf-coverage", false, "enable ebpf code coverage in daemon process")
 }
 
 func (c *BpfConfig) ParseConfig() error {

go.mod

@@ -53,12 +53,14 @@ require (
 	github.com/Masterminds/semver/v3 v3.2.1 // indirect
 	github.com/Masterminds/sprig/v3 v3.2.3 // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect
+	github.com/alecthomas/participle/v2 v2.1.0 // indirect
 	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
 	github.com/chai2010/gettext-go v1.0.2 // indirect
 	github.com/cheggaaa/pb/v3 v3.1.5 // indirect
+	github.com/cilium/coverbee v0.3.2 // indirect
 	github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect

go.sum

@@ -92,6 +92,8 @@ github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNS
 github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA=
 github.com/cheggaaa/pb/v3 v3.1.5 h1:QuuUzeM2WsAqG2gMqtzaWithDJv0i+i6UlnwSCI4QLk=
 github.com/cheggaaa/pb/v3 v3.1.5/go.mod h1:CrxkeghYTXi1lQBEI7jSn+3svI3cuc19haAj6jM60XI=
+github.com/cilium/coverbee v0.3.2 h1:RaJN5OaHf/M8tmeLemHmdO0H5bFUhvtTAoYbStDIX14=
+github.com/cilium/coverbee v0.3.2/go.mod h1:p9Q2SRC/sPA0qATNfY19GXBUPdcQP6UVV2LKgOHRIzQ=
 github.com/cilium/ebpf v0.15.0 h1:7NxJhNiBT3NG8pZJ3c+yfrVdHY8ScgKD27sScgjLMMk=
 github.com/cilium/ebpf v0.15.0/go.mod h1:DHp1WyrLeiBh19Cf/tfiSMhqheEiK8fXFZ4No0P1Hso=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=

pkg/bpf/bpf_kmesh_workload.go

@@ -20,11 +20,15 @@ package bpf
 // #include "kmesh/include/kmesh_common.h"
 
 import (
+	"bufio"
+	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
 	"reflect"
 	"syscall"
 
+	"github.com/cilium/coverbee"
 	"github.com/cilium/ebpf"
 	"github.com/cilium/ebpf/link"
 
@@ -205,7 +209,7 @@ func (sc *BpfSockConnWorkload) Detach() error {
 type BpfSockOpsWorkload struct {
 	Info BpfInfo
 	Link link.Link
-	bpf2go.KmeshSockopsWorkloadObjects
+	Coll *ebpf.Collection
 }
 
 func (so *BpfSockOpsWorkload) NewBpf(cfg *options.BpfConfig) error {
@@ -237,7 +241,7 @@ func (so *BpfSockOpsWorkload) loadKmeshSockopsObjects() (*ebpf.CollectionSpec, e
 	)
 
 	opts.Maps.PinPath = so.Info.MapPath
-	opts.Programs.LogSize = so.Info.BpfVerifyLogSize
+	opts.Programs.LogSize = 32 << 20
 
 	spec, err = bpf2go.LoadKmeshSockopsWorkload()
 	if err != nil {
@@ -248,18 +252,62 @@ func (so *BpfSockOpsWorkload) loadKmeshSockopsObjects() (*ebpf.CollectionSpec, e
 	}
 
 	setMapPinType(spec, ebpf.PinByName)
-	if err = spec.LoadAndAssign(&so.KmeshSockopsWorkloadObjects, &opts); err != nil {
-		return nil, err
+
+	// Create parameter logWriter passed to coverbee InstrumentAndLoadCollection()
+	logFile, err := os.Create("/kmesh/instrument-load.log")
+	if err != nil {
+		return nil, fmt.Errorf("open log file: %w", err)
 	}
+	defer logFile.Close()
+	logWriter := bufio.NewWriter(logFile)
+	defer logWriter.Flush()
+
+	// Instrument and load ELF
+	coll, cfg, err := coverbee.InstrumentAndLoadCollection(spec, opts, logWriter)
+	if err != nil {
+		return nil, fmt.Errorf("error while instrumenting and loading program: %w", err)
+	}
+	so.Coll = coll
+	// defer coll.Close()
+
+	// Pin programs
+	for name, prog := range coll.Programs {
+		if err = prog.Pin(filepath.Join(so.Info.BpfFsPath, name)); err != nil {
+			return nil, fmt.Errorf("error pinning program '%s': %w", name, err)
+		}
+	}
+
+	// Pin coverage map
+	if err = coll.Maps["coverbee_covermap"].Pin(filepath.Join(so.Info.MapPath, "coverbee_covermap")); err != nil {
+		return nil, fmt.Errorf("error pinning covermap: %w", err)
+	}
+
+	// Create block list and exported to a file
+	blockList := coverbee.CFGToBlockList(cfg)
+	blockListFile, err := os.Create("/kmesh/sockops-blocklist.json")
+	if err != nil {
+		return nil, fmt.Errorf("error create block-list: %w", err)
+	}
+	defer blockListFile.Close()
+
+	if err = json.NewEncoder(blockListFile).Encode(&blockList); err != nil {
+		return nil, fmt.Errorf("error encoding block-list: %w", err)
+	}
+
+	fmt.Println("Sockops: Programs instrumented and loaded")
+
+	// if err = spec.LoadAndAssign(&so.KmeshSockopsWorkloadObjects, &opts); err != nil {
+	// 	return nil, err
+	// }
 
 	if GetStartType() == Restart {
 		return spec, nil
 	}
 
-	value := reflect.ValueOf(so.KmeshSockopsWorkloadObjects.KmeshSockopsWorkloadPrograms)
-	if err = pinPrograms(&value, so.Info.BpfFsPath); err != nil {
-		return nil, err
-	}
+	// value := reflect.ValueOf(so.KmeshSockopsWorkloadObjects.KmeshSockopsWorkloadPrograms)
+	// if err = pinPrograms(&value, so.Info.BpfFsPath); err != nil {
+	// 	return nil, err
+	// }
 
 	return spec, nil
 }
@@ -282,7 +330,7 @@ func (so *BpfSockOpsWorkload) Attach() error {
 	cgopt := link.CgroupOptions{
 		Path:    so.Info.Cgroup2Path,
 		Attach:  so.Info.AttachType,
-		Program: so.KmeshSockopsWorkloadObjects.SockopsProg,
+		Program: so.Coll.Programs["sockops_prog"],
 	}
 
 	lk, err := link.AttachCgroup(cgopt)
@@ -301,27 +349,79 @@ func (so *BpfSockOpsWorkload) Attach() error {
 	return nil
 }
 
-func (so *BpfSockOpsWorkload) close() error {
-	if err := so.KmeshSockopsWorkloadObjects.Close(); err != nil {
-		return err
-	}
-	return nil
-}
+// func (so *BpfSockOpsWorkload) close() error {
+// 	if err := so.KmeshSockopsWorkloadObjects.Close(); err != nil {
+// 		return err
+// 	}
+// 	return nil
+// }
 
 func (so *BpfSockOpsWorkload) Detach() error {
-	if err := so.close(); err != nil {
-		return err
+	// Generate coverage info before close
+	// Load coverage map
+	coverMap, err := ebpf.LoadPinnedMap(filepath.Join(so.Info.MapPath, "coverbee_covermap"), nil)
+	if err != nil {
+		return fmt.Errorf("load covermap pin: %w", err)
 	}
 
-	program_value := reflect.ValueOf(so.KmeshSockopsWorkloadObjects.KmeshSockopsWorkloadPrograms)
-	if err := unpinPrograms(&program_value); err != nil {
-		return err
+	// Open block list file and get coverage info from coverage map
+	blockList := make([][]coverbee.CoverBlock, 0)
+	blockListPath, err := os.Open("/kmesh/sockops-blocklist.json")
+	if err != nil {
+		return fmt.Errorf("open block-list: %w", err)
 	}
 
-	map_value := reflect.ValueOf(so.KmeshSockopsWorkloadObjects.KmeshSockopsWorkloadMaps)
-	if err := unpinMaps(&map_value); err != nil {
-		return err
+	if err = json.NewDecoder(blockListPath).Decode(&blockList); err != nil {
+		return fmt.Errorf("decode block-list: %w", err)
+	}
+
+	if err = coverbee.ApplyCoverMapToBlockList(coverMap, blockList); err != nil {
+		return fmt.Errorf("apply covermap: %w", err)
+	}
+
+	outBlocks, err := coverbee.SourceCodeInterpolation(blockList, nil)
+	if err != nil {
+		fmt.Printf("Warning error while interpolating using source files, falling back: %s", err.Error())
+		outBlocks = blockList
+	}
+
+	output, err := os.Create("/kmesh/sockops-cov.html")
+	if err != nil {
+		return fmt.Errorf("error creating output file: %w", err)
 	}
+	defer output.Close()
+
+	// Generate HTML coverage report
+	if err = coverbee.BlockListToHTML(outBlocks, output, "count"); err != nil {
+		return fmt.Errorf("block list to HTML: %w", err)
+	}
+
+	// Unpin progs and maps, then close the Collection
+	for _, p := range so.Coll.Programs {
+		if err := p.Unpin(); err != nil {
+			return err
+		}
+	}
+	for _, m := range so.Coll.Maps {
+		if err := m.Unpin(); err != nil {
+			return err
+		}
+	}
+	so.Coll.Close()
+
+	// if err := so.close(); err != nil {
+	// 	return err
+	// }
+
+	// program_value := reflect.ValueOf(so.KmeshSockopsWorkloadObjects.KmeshSockopsWorkloadPrograms)
+	// if err := unpinPrograms(&program_value); err != nil {
+	// 	return err
+	// }
+
+	// map_value := reflect.ValueOf(so.KmeshSockopsWorkloadObjects.KmeshSockopsWorkloadMaps)
+	// if err := unpinMaps(&map_value); err != nil {
+	// 	return err
+	// }
 
 	if err := os.RemoveAll(so.Info.BpfFsPath); err != nil && !os.IsNotExist(err) {
 		return err
@@ -334,7 +434,7 @@ func (so *BpfSockOpsWorkload) Detach() error {
 }
 
 func (so *BpfSockOpsWorkload) GetSockMapFD() int {
-	return so.KmeshSockopsWorkloadObjects.KmeshSockopsWorkloadMaps.MapOfKmeshSocket.FD()
+	return so.Coll.Maps["map_of_kmesh_socket"].FD()
 }
 
 type BpfSendMsgWorkload struct {

pkg/controller/workload/workload_controller.go

@@ -54,7 +54,7 @@ func NewController(bpfWorkload *bpf.BpfKmeshWorkload) *Controller {
 }
 
 func (c *Controller) Run(ctx context.Context) {
-	go c.Rbac.Run(ctx, c.bpfWorkloadObj.SockOps.MapOfTuple, c.bpfWorkloadObj.XdpAuth.MapOfAuth)
+	go c.Rbac.Run(ctx, c.bpfWorkloadObj.SockOps.Coll.Maps["map_of_tuple"], c.bpfWorkloadObj.XdpAuth.MapOfAuth)
 	go c.MetricController.Run(ctx, c.bpfWorkloadObj.SockConn.MapOfMetricNotify, c.bpfWorkloadObj.SockConn.MapOfMetrics)
 }