diff --git a/cmd/bpfman-agent/main.go b/cmd/bpfman-agent/main.go index 02f03e2c1..82da32bd3 100644 --- a/cmd/bpfman-agent/main.go +++ b/cmd/bpfman-agent/main.go @@ -139,12 +139,19 @@ func main() { os.Exit(1) } + containerGetter, err := bpfmanagent.NewRealContainerGetter(nodeName) + if err != nil { + setupLog.Error(err, "unable to create containerGetter") + os.Exit(1) + } + common := bpfmanagent.ReconcilerCommon{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), GrpcConn: conn, BpfmanClient: gobpfman.NewBpfmanClient(conn), NodeName: nodeName, + Containers: containerGetter, } if err = (&bpfmanagent.XdpProgramReconciler{ diff --git a/controllers/bpfman-agent/common.go b/controllers/bpfman-agent/common.go index e60ddeecf..a18e796c8 100644 --- a/controllers/bpfman-agent/common.go +++ b/controllers/bpfman-agent/common.go @@ -80,6 +80,7 @@ type ReconcilerCommon struct { finalizer string recType string appOwner metav1.Object // Set if the owner is an application + Containers ContainerGetter } // bpfmanReconciler defines a generic bpfProgram K8s object reconciler which can diff --git a/controllers/bpfman-agent/containers.go b/controllers/bpfman-agent/containers.go index 54775bc49..c02f9de58 100644 --- a/controllers/bpfman-agent/containers.go +++ b/controllers/bpfman-agent/containers.go @@ -32,21 +32,46 @@ import ( "github.com/go-logr/logr" ) -// Figure out the list of container pids in which the program should be -// attached. -func getContainers( - ctx context.Context, - containerSelector *bpfmaniov1alpha1.ContainerSelector, - nodeName string, - logger logr.Logger) (*[]containerInfo, error) { +type ContainerInfo struct { + podName string + containerName string + pid int64 +} +// Create an interface for getting the list of containers in which the program +// should be attached so we can mock it in unit tests. +type ContainerGetter interface { + // Get the list of containers on this node that match the containerSelector. + GetContainers(ctx context.Context, containerSelector *bpfmaniov1alpha1.ContainerSelector, + logger logr.Logger) (*[]ContainerInfo, error) +} + +type RealContainerGetter struct { + nodeName string + clientSet kubernetes.Interface +} + +func NewRealContainerGetter(nodeName string) (*RealContainerGetter, error) { clientSet, err := getClientset() if err != nil { return nil, fmt.Errorf("failed to get clientset: %v", err) } + containerGetter := RealContainerGetter{ + nodeName: nodeName, + clientSet: clientSet, + } + + return &containerGetter, nil +} + +func (c *RealContainerGetter) GetContainers( + ctx context.Context, + containerSelector *bpfmaniov1alpha1.ContainerSelector, + logger logr.Logger) (*[]ContainerInfo, error) { + // Get the list of pods that match the selector. - podList, err := getPodsForNode(ctx, clientSet, containerSelector, nodeName) + podList, err := c.getPodsForNode(ctx, containerSelector) if err != nil { return nil, fmt.Errorf("failed to get pod list: %v", err) } @@ -64,8 +89,8 @@ func getContainers( // getPodsForNode returns a list of pods on the given node that match the given // container selector. -func getPodsForNode(ctx context.Context, clientset kubernetes.Interface, - containerSelector *bpfmaniov1alpha1.ContainerSelector, nodeName string) (*v1.PodList, error) { +func (c *RealContainerGetter) getPodsForNode(ctx context.Context, + containerSelector *bpfmaniov1alpha1.ContainerSelector) (*v1.PodList, error) { selectorString := metav1.FormatLabelSelector(&containerSelector.Pods) @@ -74,14 +99,14 @@ func getPodsForNode(ctx context.Context, clientset kubernetes.Interface, } listOptions := metav1.ListOptions{ - FieldSelector: "spec.nodeName=" + nodeName, + FieldSelector: "spec.nodeName=" + c.nodeName, } if selectorString != "" { listOptions.LabelSelector = selectorString } - podList, err := clientset.CoreV1().Pods(containerSelector.Namespace).List(ctx, listOptions) + podList, err := c.clientSet.CoreV1().Pods(containerSelector.Namespace).List(ctx, listOptions) if err != nil { return nil, fmt.Errorf("error getting pod list: %v", err) } @@ -89,18 +114,12 @@ func getPodsForNode(ctx context.Context, clientset kubernetes.Interface, return podList, nil } -type containerInfo struct { - podName string - containerName string - pid int64 -} - // getContainerInfo returns a list of containerInfo for the given pod list and container names. -func getContainerInfo(podList *v1.PodList, containerNames *[]string, logger logr.Logger) (*[]containerInfo, error) { +func getContainerInfo(podList *v1.PodList, containerNames *[]string, logger logr.Logger) (*[]ContainerInfo, error) { crictl := "/usr/local/bin/crictl" - containers := []containerInfo{} + containers := []ContainerInfo{} for i, pod := range podList.Items { logger.V(1).Info("Pod", "index", i, "Name", pod.Name, "Namespace", pod.Namespace, "NodeName", pod.Spec.NodeName) @@ -196,7 +215,7 @@ func getContainerInfo(podList *v1.PodList, containerNames *[]string, logger logr continue } - container := containerInfo{ + container := ContainerInfo{ podName: pod.Name, containerName: containerName, pid: containerPid, diff --git a/controllers/bpfman-agent/tc-program.go b/controllers/bpfman-agent/tc-program.go index 45b478e74..d65134e06 100644 --- a/controllers/bpfman-agent/tc-program.go +++ b/controllers/bpfman-agent/tc-program.go @@ -183,7 +183,7 @@ func (r *TcProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (*bpfm // There is a container selector, so see if there are any matching // containers on this node. - containerInfo, err := getContainers(ctx, r.currentTcProgram.Spec.Containers, r.NodeName, r.Logger) + containerInfo, err := r.Containers.GetContainers(ctx, r.currentTcProgram.Spec.Containers, r.Logger) if err != nil { return nil, fmt.Errorf("failed to get container pids: %v", err) } diff --git a/controllers/bpfman-agent/tcx-program.go b/controllers/bpfman-agent/tcx-program.go index e377c61d1..8df8661f2 100644 --- a/controllers/bpfman-agent/tcx-program.go +++ b/controllers/bpfman-agent/tcx-program.go @@ -149,7 +149,7 @@ func (r *TcxProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (*bpf // There is a container selector, so see if there are any matching // containers on this node. - containerInfo, err := getContainers(ctx, r.currentTcxProgram.Spec.Containers, r.NodeName, r.Logger) + containerInfo, err := r.Containers.GetContainers(ctx, r.currentTcxProgram.Spec.Containers, r.Logger) if err != nil { return nil, fmt.Errorf("failed to get container pids: %v", err) } diff --git a/controllers/bpfman-agent/test_common.go b/controllers/bpfman-agent/test_common.go new file mode 100644 index 000000000..b6e795036 --- /dev/null +++ b/controllers/bpfman-agent/test_common.go @@ -0,0 +1,84 @@ +package bpfmanagent + +import ( + "context" + "testing" + + bpfmaniov1alpha1 "github.com/bpfman/bpfman-operator/apis/v1alpha1" + "github.com/go-logr/logr" + + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientGoFake "k8s.io/client-go/kubernetes/fake" +) + +type FakeContainerGetter struct { + containerList *[]ContainerInfo +} + +func (f *FakeContainerGetter) GetContainers(ctx context.Context, containerSelector *bpfmaniov1alpha1.ContainerSelector, + logger logr.Logger) (*[]ContainerInfo, error) { + return f.containerList, nil +} + +func TestGetPods(t *testing.T) { + ctx := context.TODO() + + // Create a fake clientset + clientset := clientGoFake.NewSimpleClientset() + + // Create a ContainerSelector + containerSelector := &bpfmaniov1alpha1.ContainerSelector{ + Pods: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test", + }, + }, + Namespace: "default", + } + + nodeName := "test-node" + + // Create a Pod that matches the label selector and is on the correct node + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Labels: map[string]string{ + "app": "test", + }, + }, + Spec: v1.PodSpec{ + NodeName: nodeName, + }, + } + + containerGetter := RealContainerGetter{ + nodeName: nodeName, + clientSet: clientset, + } + + // Add the Pod to the fake clientset + _, err := clientset.CoreV1().Pods("default").Create(ctx, pod, metav1.CreateOptions{}) + require.NoError(t, err) + + // Call getPods and check the returned PodList + podList, err := containerGetter.getPodsForNode(ctx, containerSelector) + require.NoError(t, err) + require.Len(t, podList.Items, 1) + require.Equal(t, "test-pod", podList.Items[0].Name) + + // Try another selector + // Create a ContainerSelector + containerSelector = &bpfmaniov1alpha1.ContainerSelector{ + Pods: metav1.LabelSelector{ + MatchLabels: map[string]string{}, + }, + } + + podList, err = containerGetter.getPodsForNode(ctx, containerSelector) + require.NoError(t, err) + require.Len(t, podList.Items, 1) + require.Equal(t, "test-pod", podList.Items[0].Name) +} diff --git a/controllers/bpfman-agent/uprobe-program.go b/controllers/bpfman-agent/uprobe-program.go index a4a609b83..72f90ed31 100644 --- a/controllers/bpfman-agent/uprobe-program.go +++ b/controllers/bpfman-agent/uprobe-program.go @@ -140,7 +140,7 @@ func (r *UprobeProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (* // There is a container selector, so see if there are any matching // containers on this node. - containerInfo, err := getContainers(ctx, r.currentUprobeProgram.Spec.Containers, r.NodeName, r.Logger) + containerInfo, err := r.Containers.GetContainers(ctx, r.currentUprobeProgram.Spec.Containers, r.Logger) if err != nil { return nil, fmt.Errorf("failed to get container pids: %v", err) } diff --git a/controllers/bpfman-agent/uprobe-program_test.go b/controllers/bpfman-agent/uprobe-program_test.go index fbcd48a98..a22d367d2 100644 --- a/controllers/bpfman-agent/uprobe-program_test.go +++ b/controllers/bpfman-agent/uprobe-program_test.go @@ -18,6 +18,7 @@ package bpfmanagent import ( "context" + "fmt" "testing" "github.com/google/go-cmp/cmp" @@ -31,11 +32,9 @@ import ( gobpfman "github.com/bpfman/bpfman/clients/gobpfman/v1" "github.com/stretchr/testify/require" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - clientGoFake "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -129,7 +128,7 @@ func TestUprobeProgramControllerCreate(t *testing.T) { } // Check the BpfProgram Object was created successfully - err = rc.getBpfProgram(ctx, name, appProgramId, attachPoint, bpfProg) + err = r.getBpfProgram(ctx, name, appProgramId, attachPoint, bpfProg) require.NoError(t, err) require.NotEmpty(t, bpfProg) @@ -181,7 +180,7 @@ func TestUprobeProgramControllerCreate(t *testing.T) { } // Check that the bpfProgram's programs was correctly updated - err = rc.getBpfProgram(ctx, name, appProgramId, attachPoint, bpfProg) + err = r.getBpfProgram(ctx, name, appProgramId, attachPoint, bpfProg) require.NoError(t, err) // prog ID should already have been set @@ -205,64 +204,203 @@ func TestUprobeProgramControllerCreate(t *testing.T) { require.False(t, res.Requeue) // Check that the bpfProgram's status was correctly updated - err = rc.getBpfProgram(ctx, name, appProgramId, attachPoint, bpfProg) + err = r.getBpfProgram(ctx, name, appProgramId, attachPoint, bpfProg) require.NoError(t, err) require.Equal(t, string(bpfmaniov1alpha1.BpfProgCondLoaded), bpfProg.Status.Conditions[0].Type) } -func TestGetPods(t *testing.T) { - ctx := context.TODO() +func TestUprobeProgramControllerCreateContainer(t *testing.T) { + var ( + name = "fakeUprobeProgram" + namespace = "bpfman" + bytecodePath = "/tmp/hello.o" + bpfFunctionName = "test" + functionName = "malloc" + target = "libc" + offset = 0 + retprobe = false + fakeNode = testutils.NewNode("fake-control-plane") + ctx = context.TODO() + appProgramId = "" + fakePodName = "fake-pod-1" + fakeContainerName = "fake-container-1" + fakePid = int64(1001) + attachPoint = fmt.Sprintf("%s-%s-%s-%s", + sanitize(target), + sanitize(functionName), + fakePodName, + fakeContainerName, + ) + bpfProg = &bpfmaniov1alpha1.BpfProgram{} + fakeUID = "ef71d42c-aa21-48e8-a697-82391d801a81" + ) - // Create a fake clientset - clientset := clientGoFake.NewSimpleClientset() + containerSelector := bpfmaniov1alpha1.ContainerSelector{ + Pods: metav1.LabelSelector{}, + } - // Create a ContainerSelector - containerSelector := &bpfmaniov1alpha1.ContainerSelector{ - Pods: metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "test", + // A UprobeProgram object with metadata and spec. + Uprobe := &bpfmaniov1alpha1.UprobeProgram{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: bpfmaniov1alpha1.UprobeProgramSpec{ + BpfAppCommon: bpfmaniov1alpha1.BpfAppCommon{ + NodeSelector: metav1.LabelSelector{}, + ByteCode: bpfmaniov1alpha1.BytecodeSelector{ + Path: &bytecodePath, + }, + }, + UprobeProgramInfo: bpfmaniov1alpha1.UprobeProgramInfo{ + BpfProgramCommon: bpfmaniov1alpha1.BpfProgramCommon{ + BpfFunctionName: bpfFunctionName, + }, + FunctionName: functionName, + Target: target, + Offset: uint64(offset), + RetProbe: retprobe, + Containers: &containerSelector, }, }, - Namespace: "default", } - nodeName := "test-node" + // Objects to track in the fake client. + objs := []runtime.Object{fakeNode, Uprobe} - // Create a Pod that matches the label selector and is on the correct node - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod", - Namespace: "default", - Labels: map[string]string{ - "app": "test", + // Register operator types with the runtime scheme. + s := scheme.Scheme + s.AddKnownTypes(bpfmaniov1alpha1.SchemeGroupVersion, Uprobe) + s.AddKnownTypes(bpfmaniov1alpha1.SchemeGroupVersion, &bpfmaniov1alpha1.UprobeProgramList{}) + s.AddKnownTypes(bpfmaniov1alpha1.SchemeGroupVersion, &bpfmaniov1alpha1.BpfProgram{}) + s.AddKnownTypes(bpfmaniov1alpha1.SchemeGroupVersion, &bpfmaniov1alpha1.BpfProgramList{}) + + // Create a fake client to mock API calls. + cl := fake.NewClientBuilder().WithStatusSubresource(Uprobe).WithStatusSubresource(&bpfmaniov1alpha1.BpfProgram{}).WithRuntimeObjects(objs...).Build() + + cli := agenttestutils.NewBpfmanClientFake() + + testContainers := FakeContainerGetter{ + containerList: &[]ContainerInfo{ + { + podName: fakePodName, + containerName: fakeContainerName, + pid: fakePid, }, }, - Spec: v1.PodSpec{ - NodeName: nodeName, + } + + rc := ReconcilerCommon{ + Client: cl, + Scheme: s, + BpfmanClient: cli, + NodeName: fakeNode.Name, + Containers: &testContainers, + } + + // Set development Logger so we can see all logs in tests. + logf.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{Development: true}))) + + // Create a ReconcileMemcached object with the scheme and fake client. + r := &UprobeProgramReconciler{ReconcilerCommon: rc, ourNode: fakeNode} + + // Mock request to simulate Reconcile() being called on an event for a + // watched resource . + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: name, + Namespace: namespace, }, } - // Add the Pod to the fake clientset - _, err := clientset.CoreV1().Pods("default").Create(ctx, pod, metav1.CreateOptions{}) + // First reconcile should create the bpf program object + res, err := r.Reconcile(ctx, req) + if err != nil { + t.Fatalf("reconcile: (%v)", err) + } + + // Check the BpfProgram Object was created successfully + err = r.getBpfProgram(ctx, name, appProgramId, attachPoint, bpfProg) require.NoError(t, err) - // Call getPods and check the returned PodList - podList, err := getPodsForNode(ctx, clientset, containerSelector, nodeName) + require.NotEmpty(t, bpfProg) + // Finalizer is written + require.Equal(t, r.getFinalizer(), bpfProg.Finalizers[0]) + // owningConfig Label was correctly set + require.Equal(t, bpfProg.Labels[internal.BpfProgramOwner], name) + // node Label was correctly set + require.Equal(t, bpfProg.Labels[internal.K8sHostLabel], fakeNode.Name) + // uprobe function Annotation was correctly set + require.Equal(t, bpfProg.Annotations[internal.UprobeProgramTarget], target) + // Type is set + require.Equal(t, r.getRecType(), bpfProg.Spec.Type) + // Require no requeue + require.False(t, res.Requeue) + + // Update UID of bpfProgram with Fake UID since the fake API server won't + bpfProg.UID = types.UID(fakeUID) + err = cl.Update(ctx, bpfProg) require.NoError(t, err) - require.Len(t, podList.Items, 1) - require.Equal(t, "test-pod", podList.Items[0].Name) - - // Try another selector - // Create a ContainerSelector - containerSelector = &bpfmaniov1alpha1.ContainerSelector{ - Pods: metav1.LabelSelector{ - MatchLabels: map[string]string{}, + + // Second reconcile should create the bpfman Load Request and update the + // BpfProgram object's maps field and id annotation. + res, err = r.Reconcile(ctx, req) + if err != nil { + t.Fatalf("reconcile: (%v)", err) + } + + pid32 := int32(fakePid) + + // Require no requeue + require.False(t, res.Requeue) + expectedLoadReq := &gobpfman.LoadRequest{ + Bytecode: &gobpfman.BytecodeLocation{ + Location: &gobpfman.BytecodeLocation_File{File: bytecodePath}, + }, + Name: bpfFunctionName, + ProgramType: *internal.Kprobe.Uint32(), + Metadata: map[string]string{internal.UuidMetadataKey: string(bpfProg.UID), internal.ProgramNameKey: name}, + MapOwnerId: nil, + Attach: &gobpfman.AttachInfo{ + Info: &gobpfman.AttachInfo_UprobeAttachInfo{ + UprobeAttachInfo: &gobpfman.UprobeAttachInfo{ + FnName: &functionName, + Target: target, + Offset: uint64(offset), + Retprobe: retprobe, + ContainerPid: &pid32, + }, + }, }, } - podList, err = getPodsForNode(ctx, clientset, containerSelector, nodeName) + // Check that the bpfProgram's programs was correctly updated + err = r.getBpfProgram(ctx, name, appProgramId, attachPoint, bpfProg) require.NoError(t, err) - require.Len(t, podList.Items, 1) - require.Equal(t, "test-pod", podList.Items[0].Name) + + // prog ID should already have been set + id, err := bpfmanagentinternal.GetID(bpfProg) + require.NoError(t, err) + + // Check the bpfLoadRequest was correctly Built + if !cmp.Equal(expectedLoadReq, cli.LoadRequests[int(*id)], protocmp.Transform()) { + cmp.Diff(expectedLoadReq, cli.LoadRequests[int(*id)], protocmp.Transform()) + t.Logf("Diff %v", cmp.Diff(expectedLoadReq, cli.LoadRequests[int(*id)], protocmp.Transform())) + t.Fatal("Built bpfman LoadRequest does not match expected") + } + + // Third reconcile should set the status to loaded + res, err = r.Reconcile(ctx, req) + if err != nil { + t.Fatalf("reconcile: (%v)", err) + } + + // Require no requeue + require.False(t, res.Requeue) + + // Check that the bpfProgram's status was correctly updated + err = r.getBpfProgram(ctx, name, appProgramId, attachPoint, bpfProg) + require.NoError(t, err) + + require.Equal(t, string(bpfmaniov1alpha1.BpfProgCondLoaded), bpfProg.Status.Conditions[0].Type) } diff --git a/controllers/bpfman-agent/xdp-program.go b/controllers/bpfman-agent/xdp-program.go index ab1d51c20..364217908 100644 --- a/controllers/bpfman-agent/xdp-program.go +++ b/controllers/bpfman-agent/xdp-program.go @@ -168,7 +168,7 @@ func (r *XdpProgramReconciler) getExpectedBpfPrograms(ctx context.Context) (*bpf // There is a container selector, so see if there are any matching // containers on this node. - containerInfo, err := getContainers(ctx, r.currentXdpProgram.Spec.Containers, r.NodeName, r.Logger) + containerInfo, err := r.Containers.GetContainers(ctx, r.currentXdpProgram.Spec.Containers, r.Logger) if err != nil { return nil, fmt.Errorf("failed to get container pids: %v", err) }