From 1ab091219be1bbf42d3bcf4c7b4c847f53ae45d0 Mon Sep 17 00:00:00 2001 From: "Alex K." <8418476+fearful-symmetry@users.noreply.github.com> Date: Mon, 23 Sep 2024 08:36:15 -0700 Subject: [PATCH] Add UDP probes in anticipation of DNS monitoring (#206) * add upd send/recv msg probes * format * first pass at DNS packet fetching * use skb functions * Revert "use skb functions" This reverts commit fbf16088834708c3c8f7ce6f34d891aaa3e3f38c. * try to make recvmsg portable * only use kprobes for udp methods, pray this works * verifier logic has changed? * testing skb methods * clean up skb code * format * still testing * typo * still cleaning up test code * cleanup * rename enums * format * cleanup, use proper skbuff offsets * use varlen for packet body * use json for output * further cleanup * cleanup * skip peeked calls in kprobe * change DNS max packet size * add counter for headlen==0 events * cleanup, error handling * add new counter for sk_buff failures * format.. * update name * update docs, var name * tiny rewording * add tests, clean up json * use host command * use path.. * use arg array * use bash for test * use bash for test * use sh * test with a special binary... * format * clean up * format... * set correct read len for outgoing packets * revert changes to utils * use better udp send * format * clean up --------- Co-authored-by: Christiano Haesbaert --- GPL/Events/EbpfEventProto.h | 28 +++- GPL/Events/Network/Probe.bpf.c | 160 +++++++++++++++++++++++ non-GPL/Events/EventsTrace/EventsTrace.c | 52 +++++++- non-GPL/Events/Lib/EbpfEvents.c | 7 +- testing/test_bins/udp_send.c | 42 ++++++ testing/testrunner/main.go | 5 +- testing/testrunner/tests.go | 24 ++++ testing/testrunner/utils.go | 6 + 8 files changed, 314 insertions(+), 10 deletions(-) create mode 100644 testing/test_bins/udp_send.c diff --git a/GPL/Events/EbpfEventProto.h b/GPL/Events/EbpfEventProto.h index c3919c9e..0277d72f 100644 --- a/GPL/Events/EbpfEventProto.h +++ b/GPL/Events/EbpfEventProto.h @@ -11,6 +11,10 @@ #define EBPF_EVENTPROBE_EBPFEVENTPROTO_H #define TASK_COMM_LEN 16 +// The theoretical max size of DNS packets over UDP is 512. +// Like so many things in DNS this number probaby isn't 100% accurate. +// DNS extensions in RFC2671 and RFC6891 mean the actual size can be larger. +#define MAX_DNS_PACKET 1500 #ifndef __KERNEL__ #include @@ -40,6 +44,7 @@ enum ebpf_event_type { EBPF_EVENT_PROCESS_SHMGET = (1 << 17), EBPF_EVENT_PROCESS_PTRACE = (1 << 18), EBPF_EVENT_PROCESS_LOAD_MODULE = (1 << 19), + EBPF_EVENT_NETWORK_DNS_PKT = (1 << 20), }; struct ebpf_event_header { @@ -66,6 +71,7 @@ enum ebpf_varlen_field_type { EBPF_VL_FIELD_SYMLINK_TARGET_PATH, EBPF_VL_FIELD_MOD_VERSION, EBPF_VL_FIELD_MOD_SRCVERSION, + EBPF_VL_FIELD_DNS_BODY, }; // Convenience macro to iterate all the variable length fields in an event @@ -341,6 +347,7 @@ struct ebpf_process_load_module_event { enum ebpf_net_info_transport { EBPF_NETWORK_EVENT_TRANSPORT_TCP = 1, + EBPF_NETWORK_EVENT_TRANSPORT_UDP = 2, }; enum ebpf_net_info_af { @@ -348,6 +355,11 @@ enum ebpf_net_info_af { EBPF_NETWORK_EVENT_AF_INET6 = 2, }; +enum ebpf_net_udp_info { + EBPF_NETWORK_EVENT_SKB_CONSUME_UDP = 1, + EBPF_NETWORK_EVENT_IP_SEND_UDP = 2, +}; + struct ebpf_net_info_tcp_close { uint64_t bytes_sent; uint64_t bytes_received; @@ -379,10 +391,22 @@ struct ebpf_net_event { char comm[TASK_COMM_LEN]; } __attribute__((packed)); +struct ebpf_dns_event { + struct ebpf_event_header hdr; + struct ebpf_pid_info pids; + struct ebpf_net_info net; + char comm[TASK_COMM_LEN]; + enum ebpf_net_udp_info udp_evt; + uint64_t original_len; + // Variable length fields: dns body + struct ebpf_varlen_fields_start vl_fields; +} __attribute__((packed)); + // Basic event statistics struct ebpf_event_stats { - uint64_t lost; // lost events due to a full ringbuffer - uint64_t sent; // events sent through the ringbuffer + uint64_t lost; // lost events due to a full ringbuffer + uint64_t sent; // events sent through the ringbuffer + uint64_t dns_zero_body; // indicates that the dns body of a sk_buff was unavailable }; #endif // EBPF_EVENTPROBE_EBPFEVENTPROTO_H diff --git a/GPL/Events/Network/Probe.bpf.c b/GPL/Events/Network/Probe.bpf.c index 6a7f43b0..dedb11af 100644 --- a/GPL/Events/Network/Probe.bpf.c +++ b/GPL/Events/Network/Probe.bpf.c @@ -17,6 +17,7 @@ #include "Helpers.h" #include "Network.h" #include "State.h" +#include "Varlen.h" DECL_FUNC_RET(inet_csk_accept); @@ -43,6 +44,165 @@ static int inet_csk_accept__exit(struct sock *sk) return 0; } +static int udp_skb_handle(struct sk_buff *skb, enum ebpf_net_udp_info evt_type) +{ + if (skb == NULL) { + goto out; + } + + if (ebpf_events_is_trusted_pid()) + goto out; + + struct ebpf_dns_event *event = get_event_buffer(); + if (event == NULL) + goto out; + + // read from skbuf + unsigned char *skb_head = BPF_CORE_READ(skb, head); + // get lengths + u16 net_header_offset = BPF_CORE_READ(skb, network_header); + u16 transport_header_offset = BPF_CORE_READ(skb, transport_header); + size_t network_header_size = 0; + u8 proto = 0; + + struct iphdr ip_hdr; + bpf_core_read(&ip_hdr, sizeof(struct iphdr), skb_head + net_header_offset); + if (ip_hdr.version == 4) { + proto = ip_hdr.protocol; + + if (bpf_probe_read(event->net.saddr, 4, &ip_hdr.saddr) != 0) { + goto out; + }; + + if (bpf_probe_read(event->net.daddr, 4, &ip_hdr.daddr) != 0) { + goto out; + } + network_header_size = sizeof(struct iphdr); + event->net.family = EBPF_NETWORK_EVENT_AF_INET; + } else if (ip_hdr.version == 6) { + struct ipv6hdr ip6_hdr; + bpf_core_read(&ip6_hdr, sizeof(struct ipv6hdr), skb_head + net_header_offset); + proto = ip6_hdr.nexthdr; + + if (bpf_probe_read(event->net.saddr6, 16, ip6_hdr.saddr.in6_u.u6_addr8) != 0) { + goto out; + } + + if (bpf_probe_read(event->net.daddr6, 16, ip6_hdr.daddr.in6_u.u6_addr8) != 0) { + goto out; + } + + network_header_size = sizeof(struct ipv6hdr); + event->net.family = EBPF_NETWORK_EVENT_AF_INET6; + } else { + goto out; + } + + if (proto != IPPROTO_UDP) { + goto out; + } + + struct udphdr udp_hdr; + if (bpf_core_read(&udp_hdr, sizeof(struct udphdr), skb_head + transport_header_offset) != 0) { + goto out; + } + + uint16_t dport = bpf_ntohs(udp_hdr.dest); + uint16_t sport = bpf_ntohs(udp_hdr.source); + // filter out non-DNS packets + if (sport != 53 && dport != 53) { + goto out; + } + + event->net.dport = dport; + event->net.sport = sport; + event->net.transport = EBPF_NETWORK_EVENT_TRANSPORT_UDP; + + struct task_struct *task = (struct task_struct *)bpf_get_current_task(); + ebpf_pid_info__fill(&event->pids, task); + bpf_get_current_comm(event->comm, TASK_COMM_LEN); + event->hdr.ts = bpf_ktime_get_ns(); + + // constrain the read size to make the verifier happy + // see skb_headlen() in skbuff.h + size_t readsize = BPF_CORE_READ(skb, len); + size_t datalen = BPF_CORE_READ(skb, data_len); + size_t headlen = readsize - datalen; + // headlen of zero indicates we have no non-paged data, and thus cannot read + // anything from the root data node + if (headlen == 0) { + u32 zero = 0; + struct ebpf_event_stats *es = bpf_map_lookup_elem(&ringbuf_stats, &zero); + if (es != NULL) { + es->dns_zero_body++; + } + goto out; + } + + size_t body_size = headlen; + // for ip_send_skb(), we're at a point in the network stack where we've just prepended the IP + // header, so the normal headlen for the skb_buff includes the headers. Reset them so we *just* + // read the application body. + if (evt_type == EBPF_NETWORK_EVENT_IP_SEND_UDP) { + body_size = headlen - (sizeof(struct udphdr) + network_header_size); + } + + event->original_len = headlen; + if (body_size > MAX_DNS_PACKET) { + body_size = MAX_DNS_PACKET; + } + + ebpf_vl_fields__init(&event->vl_fields); + struct ebpf_varlen_field *field; + field = ebpf_vl_field__add(&event->vl_fields, EBPF_VL_FIELD_DNS_BODY); + long ret = bpf_probe_read_kernel(field->data, body_size, + skb_head + transport_header_offset + sizeof(struct udphdr)); + if (ret != 0) { + bpf_printk("error reading in data buffer: %d", ret); + goto out; + } + ebpf_vl_field__set_size(&event->vl_fields, field, body_size); + + event->hdr.type = EBPF_EVENT_NETWORK_DNS_PKT; + event->udp_evt = evt_type; + ebpf_ringbuf_write(&ringbuf, event, EVENT_SIZE(event), 0); + +out: + return 0; +} + +SEC("fentry/ip_send_skb") +int BPF_PROG(fentry__ip_send_skb, struct net *net, struct sk_buff *skb) +{ + return udp_skb_handle(skb, EBPF_NETWORK_EVENT_IP_SEND_UDP); +} + +SEC("fentry/skb_consume_udp") +int BPF_PROG(fentry__skb_consume_udp, struct sock *sk, struct sk_buff *skb, int len) +{ + // skip peek operations + if (len < 0) { + return 0; + } + return udp_skb_handle(skb, EBPF_NETWORK_EVENT_SKB_CONSUME_UDP); +} + +SEC("kprobe/ip_send_skb") +int BPF_KPROBE(kprobe__ip_send_udp, struct net *net, struct sk_buff *skb) +{ + return udp_skb_handle(skb, EBPF_NETWORK_EVENT_IP_SEND_UDP); +} + +SEC("kprobe/skb_consume_udp") +int BPF_KPROBE(kprobe__skb_consume_udp, struct net *net, struct sk_buff *skb, int len) +{ + // skip peek operations + if (len < 0) { + return 0; + } + return udp_skb_handle(skb, EBPF_NETWORK_EVENT_SKB_CONSUME_UDP); +} + SEC("fexit/inet_csk_accept") int BPF_PROG(fexit__inet_csk_accept) { diff --git a/non-GPL/Events/EventsTrace/EventsTrace.c b/non-GPL/Events/EventsTrace/EventsTrace.c index 9ff12d05..ac07548f 100644 --- a/non-GPL/Events/EventsTrace/EventsTrace.c +++ b/non-GPL/Events/EventsTrace/EventsTrace.c @@ -63,6 +63,7 @@ enum cmdline_opts { NETWORK_CONNECTION_ATTEMPTED, NETWORK_CONNECTION_ACCEPTED, NETWORK_CONNECTION_CLOSED, + NETWORK_DNS_PKT, CMDLINE_MAX }; @@ -89,6 +90,7 @@ static uint64_t cmdline_to_lib[CMDLINE_MAX] = { x(NETWORK_CONNECTION_ATTEMPTED) x(NETWORK_CONNECTION_ACCEPTED) x(NETWORK_CONNECTION_CLOSED) + x(NETWORK_DNS_PKT) #undef x // clang-format on }; @@ -114,6 +116,7 @@ static const struct argp_option opts[] = { {"process-load-module", PROCESS_LOAD_MODULE, NULL, false, "Print kernel module load events", 0}, {"net-conn-accept", NETWORK_CONNECTION_ACCEPTED, NULL, false, "Print network connection accepted events", 0}, + {"net-conn-dns-pkt", NETWORK_DNS_PKT, NULL, false, "Print DNS events", 0}, {"net-conn-attempt", NETWORK_CONNECTION_ATTEMPTED, NULL, false, "Print network connection attempted events", 0}, {"net-conn-closed", NETWORK_CONNECTION_CLOSED, NULL, false, @@ -173,6 +176,7 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) case NETWORK_CONNECTION_ACCEPTED: case NETWORK_CONNECTION_ATTEMPTED: case NETWORK_CONNECTION_CLOSED: + case NETWORK_DNS_PKT: g_events_env |= cmdline_to_lib[key]; break; case ARGP_KEY_ARG: @@ -965,9 +969,8 @@ static void out_ip6_addr(const char *name, const void *addr) printf("\"%s\":\"%s\"", name, buf); } -static void out_net_info(const char *name, struct ebpf_net_event *evt) +static void out_net_info(const char *name, struct ebpf_net_info *net, struct ebpf_event_header *hdr) { - struct ebpf_net_info *net = &evt->net; printf("\"%s\":", name); out_object_start(); @@ -977,6 +980,10 @@ static void out_net_info(const char *name, struct ebpf_net_event *evt) out_string("transport", "TCP"); out_comma(); break; + case EBPF_NETWORK_EVENT_TRANSPORT_UDP: + out_string("transport", "UDP"); + out_comma(); + break; } switch (net->family) { @@ -1015,7 +1022,7 @@ static void out_net_info(const char *name, struct ebpf_net_event *evt) out_comma(); out_int("network_namespace", net->netns); - switch (evt->hdr.type) { + switch (hdr->type) { case EBPF_EVENT_NETWORK_CONNECTION_CLOSED: out_comma(); out_uint("bytes_sent", net->tcp.close.bytes_sent); @@ -1037,7 +1044,7 @@ static void out_network_event(const char *name, struct ebpf_net_event *evt) out_pid_info("pids", &evt->pids); out_comma(); - out_net_info("net", evt); + out_net_info("net", &evt->net, &evt->hdr); out_comma(); out_string("comm", (const char *)&evt->comm); @@ -1051,6 +1058,40 @@ static void out_network_connection_accepted_event(struct ebpf_net_event *evt) out_network_event("NETWORK_CONNECTION_ACCEPTED", evt); } +static void out_network_dns_event(struct ebpf_dns_event *event) +{ + out_object_start(); + out_event_type("DNS_EVENT"); + out_comma(); + + out_pid_info("pids", &event->pids); + out_comma(); + + out_net_info("net", &event->net, &event->hdr); + out_comma(); + + out_string("comm", (const char *)&event->comm); + out_comma(); + + printf("\"data\":"); + out_array_start(); + struct ebpf_varlen_field *field; + FOR_EACH_VARLEN_FIELD(event->vl_fields, field) + { + for (size_t i = 0; i < field->size; i++) { + uint8_t part = field->data[i]; + printf("%d", part); + if (i < field->size - 1) { + printf(", "); + } + } + } + out_array_end(); + + out_object_end(); + out_newline(); +} + static void out_network_connection_attempted_event(struct ebpf_net_event *evt) { out_network_event("NETWORK_CONNECTION_ATTEMPTED", evt); @@ -1130,6 +1171,9 @@ static int event_ctx_callback(struct ebpf_event_header *evt_hdr) case EBPF_EVENT_NETWORK_CONNECTION_CLOSED: out_network_connection_closed_event((struct ebpf_net_event *)evt_hdr); break; + case EBPF_EVENT_NETWORK_DNS_PKT: + out_network_dns_event((struct ebpf_dns_event *)evt_hdr); + break; } return 0; diff --git a/non-GPL/Events/Lib/EbpfEvents.c b/non-GPL/Events/Lib/EbpfEvents.c index de456775..c48c1fe4 100644 --- a/non-GPL/Events/Lib/EbpfEvents.c +++ b/non-GPL/Events/Lib/EbpfEvents.c @@ -386,6 +386,8 @@ static inline int probe_set_autoload(struct btf *btf, struct EventProbe_bpf *obj err = err ?: bpf_program__set_autoload(obj->progs.kretprobe__vfs_write, false); err = err ?: bpf_program__set_autoload(obj->progs.kprobe__chown_common, false); err = err ?: bpf_program__set_autoload(obj->progs.kretprobe__chown_common, false); + err = err ?: bpf_program__set_autoload(obj->progs.kprobe__ip_send_udp, false); + err = err ?: bpf_program__set_autoload(obj->progs.kprobe__skb_consume_udp, false); } else { err = err ?: bpf_program__set_autoload(obj->progs.fentry__do_unlinkat, false); err = err ?: bpf_program__set_autoload(obj->progs.fentry__mnt_want_write, false); @@ -403,6 +405,8 @@ static inline int probe_set_autoload(struct btf *btf, struct EventProbe_bpf *obj err = err ?: bpf_program__set_autoload(obj->progs.fexit__do_truncate, false); err = err ?: bpf_program__set_autoload(obj->progs.fexit__vfs_write, false); err = err ?: bpf_program__set_autoload(obj->progs.fexit__chown_common, false); + err = err ?: bpf_program__set_autoload(obj->progs.fentry__ip_send_skb, false); + err = err ?: bpf_program__set_autoload(obj->progs.fentry__skb_consume_udp, false); } return err; @@ -816,6 +820,7 @@ int ebpf_event_ctx__read_stats(struct ebpf_event_ctx *ctx, struct ebpf_event_sta for (i = 0; i < libbpf_num_possible_cpus(); i++) { ees->lost += pcpu_ees[i].lost; ees->sent += pcpu_ees[i].sent; + ees->dns_zero_body += pcpu_ees[i].dns_zero_body; } return 0; @@ -938,4 +943,4 @@ int ebpf_set_process_trustlist(struct bpf_map *map, uint32_t *pids, int count) } return rv; -} +} \ No newline at end of file diff --git a/testing/test_bins/udp_send.c b/testing/test_bins/udp_send.c new file mode 100644 index 00000000..04a99d20 --- /dev/null +++ b/testing/test_bins/udp_send.c @@ -0,0 +1,42 @@ +#define _GNU_SOURCE /* program_invocation_name */ +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include + +int main(int argc, char *argv[]) +{ + struct sockaddr_in sin; + int sock; + ssize_t n; + char *inaddr = "127.0.0.1"; + uint64_t buf[] = {0xdeadbeef, 0xdeadbeef, 0xdeadbeef, 0xdeadbeef, + 0xdeadbeef, 0xdeadbeef, 0xdeadbeef, 0xdeadbeef}; + + bzero(&sin, sizeof(sin)); + + if (inet_aton(inaddr, &sin.sin_addr) == 0) + errx(1, "inet_aton(%s)", inaddr); + sin.sin_port = htons(53); + sin.sin_family = AF_INET; + + if ((sock = socket(AF_INET, SOCK_DGRAM, 0)) == -1) + err(1, "socket"); + + n = sendto(sock, buf, sizeof(buf), 0, (struct sockaddr *)&sin, sizeof(sin)); + if (n == -1) + err(1, "sendto"); + else if (n != sizeof(buf)) + errx(1, "sendto: shortcount"); + + return (0); +} \ No newline at end of file diff --git a/testing/testrunner/main.go b/testing/testrunner/main.go index f0c13232..e73781f1 100644 --- a/testing/testrunner/main.go +++ b/testing/testrunner/main.go @@ -9,9 +9,7 @@ package main -import ( - "fmt" -) +import "fmt" func main() { RunEventsTest(TestFeaturesCorrect) @@ -32,6 +30,7 @@ func main() { RunEventsTest(TestTcpv6ConnectionAttempt, "--net-conn-attempt") RunEventsTest(TestTcpv6ConnectionAccept, "--net-conn-accept") RunEventsTest(TestTcpv6ConnectionClose, "--net-conn-close") + RunEventsTest(TestDNSMonitor, "--net-conn-dns-pkt") RunTest(TestTcFilter) diff --git a/testing/testrunner/tests.go b/testing/testrunner/tests.go index a8a45710..e254d8a3 100644 --- a/testing/testrunner/tests.go +++ b/testing/testrunner/tests.go @@ -152,6 +152,30 @@ func TestForkExec(et *EventsTraceInstance) { AssertStringsEqual(execEvent.Cwd, "/") } +func TestDNSMonitor(et *EventsTraceInstance) { + runTestBin("udp_send") + + type dnsOutput struct { + Data []uint8 `json:"data"` + NetConnAcceptEvent + } + + line := et.GetNextEventJson("DNS_EVENT") + lineData := dnsOutput{} + err := json.Unmarshal([]byte(line), &lineData) + if err != nil { + TestFail("failed to unmarshal JSON body", err) + } + + AssertStringsEqual(lineData.Net.Transport, "UDP") + AssertStringsEqual(lineData.Net.Family, "AF_INET") + // first two bytes of a DNS body will be the session ID for the query, and + // should not be zero + AssertNotZero(lineData.Data[0]) + AssertNotZero(lineData.Data[1]) + +} + func TestFileCreate(et *EventsTraceInstance) { outputStr := runTestBin("create_rename_delete_file") var binOutput struct { diff --git a/testing/testrunner/utils.go b/testing/testrunner/utils.go index 50880e55..345d04b1 100644 --- a/testing/testrunner/utils.go +++ b/testing/testrunner/utils.go @@ -220,6 +220,12 @@ func AssertPidInfoEqual(tpi TestPidInfo, pi PidInfo) { AssertInt64Equal(pi.Sid, tpi.Sid) } +func AssertNotZero(a uint8) { + if a == 0 { + TestFail(fmt.Sprintf("Test assertion failed %d == 0", a)) + } +} + func AssertTrue(val bool) { if !val { TestFail(fmt.Sprintf("Expected %t to be true", val))