-
Notifications
You must be signed in to change notification settings - Fork 85
/
bpf_helpers.h
1017 lines (830 loc) · 34.8 KB
/
bpf_helpers.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// Copyright (c) 2019 Dropbox, Inc.
// Full license can be found in the LICENSE file.
// BPF helpers
// Set of defines / prototypes to use from eBPF programs as well as from regular
// linux/mac "cross" compilation.
#ifndef __BPF_HELPERS_H
#define __BPF_HELPERS_H
// Standard types.
// Due to tooons of dependencies in standard linux kernel headers
// Define types explicitly.
typedef unsigned short __u16; // NOLINT
typedef unsigned char __u8;
typedef unsigned int __u32;
typedef unsigned long long __u64;
typedef int __s32;
typedef unsigned long size_t;
typedef __u32 __be32;
typedef __u16 __be16;
// BPF map types
enum bpf_map_type {
BPF_MAP_TYPE_UNSPEC = 0,
BPF_MAP_TYPE_HASH,
BPF_MAP_TYPE_ARRAY,
BPF_MAP_TYPE_PROG_ARRAY,
BPF_MAP_TYPE_PERF_EVENT_ARRAY,
BPF_MAP_TYPE_PERCPU_HASH,
BPF_MAP_TYPE_PERCPU_ARRAY,
BPF_MAP_TYPE_STACK_TRACE,
BPF_MAP_TYPE_CGROUP_ARRAY,
BPF_MAP_TYPE_LRU_HASH,
BPF_MAP_TYPE_LRU_PERCPU_HASH,
BPF_MAP_TYPE_LPM_TRIE,
BPF_MAP_TYPE_ARRAY_OF_MAPS,
BPF_MAP_TYPE_HASH_OF_MAPS,
BPF_MAP_TYPE_DEVMAP,
BPF_MAP_TYPE_SOCKMAP,
BPF_MAP_TYPE_CPUMAP,
BPF_MAP_TYPE_XSKMAP,
BPF_MAP_TYPE_SOCKHASH,
BPF_MAP_TYPE_CGROUP_STORAGE,
BPF_MAP_TYPE_REUSEPORT_SOCKARRAY,
BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE,
BPF_MAP_TYPE_QUEUE,
BPF_MAP_TYPE_STACK,
BPF_MAP_TYPE_SK_STORAGE,
};
/* BPF_FUNC_skb_store_bytes flags. */
enum {
BPF_F_RECOMPUTE_CSUM = (1ULL << 0),
BPF_F_INVALIDATE_HASH = (1ULL << 1),
};
/* BPF_FUNC_l3_csum_replace and BPF_FUNC_l4_csum_replace flags.
* First 4 bits are for passing the header field size.
*/
enum {
BPF_F_HDR_FIELD_MASK = 0xfULL,
};
/* BPF_FUNC_l4_csum_replace flags. */
enum {
BPF_F_PSEUDO_HDR = (1ULL << 4),
BPF_F_MARK_MANGLED_0 = (1ULL << 5),
BPF_F_MARK_ENFORCE = (1ULL << 6),
};
/* BPF_FUNC_clone_redirect and BPF_FUNC_redirect flags. */
enum {
BPF_F_INGRESS = (1ULL << 0),
};
/* BPF_FUNC_skb_set_tunnel_key flags. */
enum {
BPF_F_ZERO_CSUM_TX = (1ULL << 1),
BPF_F_DONT_FRAGMENT = (1ULL << 2),
BPF_F_SEQ_NUMBER = (1ULL << 3),
};
/* BPF_FUNC_skb_adjust_room flags. */
enum {
BPF_F_ADJ_ROOM_FIXED_GSO = (1ULL << 0),
BPF_F_ADJ_ROOM_ENCAP_L3_IPV4 = (1ULL << 1),
BPF_F_ADJ_ROOM_ENCAP_L3_IPV6 = (1ULL << 2),
BPF_F_ADJ_ROOM_ENCAP_L4_GRE = (1ULL << 3),
BPF_F_ADJ_ROOM_ENCAP_L4_UDP = (1ULL << 4),
BPF_F_ADJ_ROOM_NO_CSUM_RESET = (1ULL << 5),
BPF_F_ADJ_ROOM_ENCAP_L2_ETH = (1ULL << 6),
};
/* flags for BPF_MAP_UPDATE_ELEM command */
#define BPF_ANY 0 /* create new element or update existing */
#define BPF_NOEXIST 1 /* create new element if it didn't exist */
#define BPF_EXIST 2 /* update existing element */
#define BPF_F_LOCK 4 /* spin_lock-ed map_lookup/map_update */
/* BPF_FUNC_perf_event_output, BPF_FUNC_perf_event_read and
* BPF_FUNC_perf_event_read_value flags.
*/
#define BPF_F_INDEX_MASK 0xffffffffULL
#define BPF_F_CURRENT_CPU BPF_F_INDEX_MASK
// A helper structure used by eBPF C program
// to describe map attributes to BPF program loader
struct bpf_map_def {
__u32 map_type;
__u32 key_size;
__u32 value_size;
__u32 max_entries;
__u32 map_flags;
// Array/Hash of maps use case: pointer to inner map template
void *inner_map_def;
// Define this to make map system wide ("object pinning")
// path could be anything, like '/sys/fs/bpf/foo'
// WARN: You must have BPF filesystem mounted on provided location
const char *persistent_path;
};
#define BPF_MAP_DEF_SIZE sizeof(struct bpf_map_def)
#define BPF_MAP_OFFSET_PERSISTENT offsetof(struct bpf_map_def, persistent_path)
#define BPF_MAP_OFFSET_INNER_MAP offsetof(struct bpf_map_def, inner_map_def)
/* Generic BPF return codes which all BPF program types may support.
* The values are binary compatible with their TC_ACT_* counter-part to
* provide backwards compatibility with existing SCHED_CLS and SCHED_ACT
* programs.
*
* XDP is handled seprately, see XDP_*.
*/
enum bpf_ret_code {
BPF_OK = 0,
/* 1 reserved */
BPF_DROP = 2,
/* 3-6 reserved */
BPF_REDIRECT = 7,
/* >127 are reserved for prog type specific return codes.
*
* BPF_LWT_REROUTE: used by BPF_PROG_TYPE_LWT_IN and
* BPF_PROG_TYPE_LWT_XMIT to indicate that skb had been
* changed and should be routed based on its new L3 header.
* (This is an L3 redirect, as opposed to L2 redirect
* represented by BPF_REDIRECT above).
*/
BPF_LWT_REROUTE = 128,
};
// XDP related constants
enum xdp_action {
XDP_ABORTED = 0,
XDP_DROP,
XDP_PASS,
XDP_TX,
XDP_REDIRECT,
};
// Socket Filter programs return code
enum socket_filter_action {
SOCKET_FILTER_DENY = 0,
SOCKET_FILTER_ALLOW,
};
// Kprobe required constants / structs
// (arch/x86/include/asm/ptrace.h)
#define PT_REGS_PARM1(x) ((x)->di)
#define PT_REGS_PARM2(x) ((x)->si)
#define PT_REGS_PARM3(x) ((x)->dx)
#define PT_REGS_PARM4(x) ((x)->r10)
#define PT_REGS_PARM5(x) ((x)->r8)
#define PT_REGS_PARM6(x) ((x)->r9)
#define PT_REGS_RET(x) ((x)->sp)
#define PT_REGS_FP(x) ((x)->bp)
#define PT_REGS_RC(x) ((x)->ax)
#define PT_REGS_SP(x) ((x)->sp)
#define PT_REGS_IP(x) ((x)->ip)
struct pt_regs {
unsigned long r15;
unsigned long r14;
unsigned long r13;
unsigned long r12;
unsigned long bp;
unsigned long bx;
unsigned long r11;
unsigned long r10;
unsigned long r9;
unsigned long r8;
unsigned long ax;
unsigned long cx;
unsigned long dx;
unsigned long si;
unsigned long di;
unsigned long orig_ax;
unsigned long ip;
unsigned long cs;
unsigned long flags;
unsigned long sp;
unsigned long ss;
};
#define bpf_likely(X) __builtin_expect(!!(X), 1)
#define bpf_unlikely(X) __builtin_expect(!!(X), 0)
#define UNUSED __attribute__((unused))
// In order to cross compile BPF program for BPF / Linux / Mac
// we need to define platform specific things like:
// 1. Custom (non kernel) implementation for bpf_map_* functions
// 2. For BPF we need to put programs into special sections, but, for
// regular linux target (mostly for tests) we don't.
// 3. BPF does not support function calls, so __always_inline__ is must have.
// However, for testing it doesn't make sense.
// 4. Debug prints - for BPF it is done by calling helper, for linux just
// regular printf()
#ifdef __BPF__
// Clang for eBPF missed static_assert declaration because programs are C, not
// CPP
#define static_assert _Static_assert
// Helper macro to place programs, maps, license in
// different sections in ELF file.
#define SEC(NAME) __attribute__((section(NAME), used))
// eBPF does not support functions (yet), so, all functions MUST be inlined.
// Starting from kernel 4.16 it is not required to always inline functions
// since support has been added
#define INLINE __attribute__((__always_inline__))
// XDP metadata - basically data packet
// P.S. for some reason XDP programs uses 32bit pointers
struct xdp_md {
__u32 data;
__u32 data_end;
__u32 data_meta;
/* Below access go through struct xdp_rxq_info */
__u32 ingress_ifindex; /* rxq->dev->ifindex */
__u32 rx_queue_index; /* rxq->queue_index */
__u32 egress_ifindex; /* txq->dev->ifindex */
};
/* user accessible mirror of in-kernel sk_buff.
* new fields can only be added to the end of this structure
*/
struct __sk_buff {
__u32 len;
__u32 pkt_type;
__u32 mark;
__u32 queue_mapping;
__u32 protocol;
__u32 vlan_present;
__u32 vlan_tci;
__u32 vlan_proto;
__u32 priority;
__u32 ingress_ifindex;
__u32 ifindex;
__u32 tc_index;
__u32 cb[5];
__u32 hash;
__u32 tc_classid;
__u32 data;
__u32 data_end;
__u32 napi_id;
/* Accessed by BPF_PROG_TYPE_sk_skb types from here to ... */
__u32 family;
__u32 remote_ip4; /* Stored in network byte order */
__u32 local_ip4; /* Stored in network byte order */
__u32 remote_ip6[4]; /* Stored in network byte order */
__u32 local_ip6[4]; /* Stored in network byte order */
__u32 remote_port; /* Stored in network byte order */
__u32 local_port; /* stored in host byte order */
/* ... here. */
__u32 data_meta;
};
struct bpf_sock_tuple {
union {
struct {
__be32 saddr;
__be32 daddr;
__be16 sport;
__be16 dport;
} ipv4;
struct {
__be32 saddr[4];
__be32 daddr[4];
__be16 sport;
__be16 dport;
} ipv6;
};
};
struct bpf_spin_lock {
__u32 val;
};
struct bpf_sysctl {
__u32 write; /* Sysctl is being read (= 0) or written (= 1).
* Allows 1,2,4-byte read, but no write.
*/
__u32 file_pos; /* Sysctl file position to read from, write to.
* Allows 1,2,4-byte read an 4-byte write.
*/
};
// BPF helper functions supported on linux kernel 5.2+
// clang-format off
#define __BPF_FUNC_MAPPER(FN) \
FN(unspec), \
FN(map_lookup_elem), \
FN(map_update_elem), \
FN(map_delete_elem), \
FN(probe_read), \
FN(ktime_get_ns), \
FN(trace_printk), \
FN(get_prandom_u32), \
FN(get_smp_processor_id), \
FN(skb_store_bytes), \
FN(l3_csum_replace), \
FN(l4_csum_replace), \
FN(tail_call), \
FN(clone_redirect), \
FN(get_current_pid_tgid), \
FN(get_current_uid_gid), \
FN(get_current_comm), \
FN(get_cgroup_classid), \
FN(skb_vlan_push), \
FN(skb_vlan_pop), \
FN(skb_get_tunnel_key), \
FN(skb_set_tunnel_key), \
FN(perf_event_read), \
FN(redirect), \
FN(get_route_realm), \
FN(perf_event_output), \
FN(skb_load_bytes), \
FN(get_stackid), \
FN(csum_diff), \
FN(skb_get_tunnel_opt), \
FN(skb_set_tunnel_opt), \
FN(skb_change_proto), \
FN(skb_change_type), \
FN(skb_under_cgroup), \
FN(get_hash_recalc), \
FN(get_current_task), \
FN(probe_write_user), \
FN(current_task_under_cgroup), \
FN(skb_change_tail), \
FN(skb_pull_data), \
FN(csum_update), \
FN(set_hash_invalid), \
FN(get_numa_node_id), \
FN(skb_change_head), \
FN(xdp_adjust_head), \
FN(probe_read_str), \
FN(get_socket_cookie), \
FN(get_socket_uid), \
FN(set_hash), \
FN(setsockopt), \
FN(skb_adjust_room), \
FN(redirect_map), \
FN(sk_redirect_map), \
FN(sock_map_update), \
FN(xdp_adjust_meta), \
FN(perf_event_read_value), \
FN(perf_prog_read_value), \
FN(getsockopt), \
FN(override_return), \
FN(sock_ops_cb_flags_set), \
FN(msg_redirect_map), \
FN(msg_apply_bytes), \
FN(msg_cork_bytes), \
FN(msg_pull_data), \
FN(bind), \
FN(xdp_adjust_tail), \
FN(skb_get_xfrm_state), \
FN(get_stack), \
FN(skb_load_bytes_relative), \
FN(fib_lookup), \
FN(sock_hash_update), \
FN(msg_redirect_hash), \
FN(sk_redirect_hash), \
FN(lwt_push_encap), \
FN(lwt_seg6_store_bytes), \
FN(lwt_seg6_adjust_srh), \
FN(lwt_seg6_action), \
FN(rc_repeat), \
FN(rc_keydown), \
FN(skb_cgroup_id), \
FN(get_current_cgroup_id), \
FN(get_local_storage), \
FN(sk_select_reuseport), \
FN(skb_ancestor_cgroup_id), \
FN(sk_lookup_tcp), \
FN(sk_lookup_udp), \
FN(sk_release), \
FN(map_push_elem), \
FN(map_pop_elem), \
FN(map_peek_elem), \
FN(msg_push_data), \
FN(msg_pop_data), \
FN(rc_pointer_rel), \
FN(spin_lock), \
FN(spin_unlock), \
FN(sk_fullsock), \
FN(tcp_sock), \
FN(skb_ecn_set_ce), \
FN(get_listener_sock), \
FN(skc_lookup_tcp), \
FN(tcp_check_syncookie), \
FN(sysctl_get_name), \
FN(sysctl_get_current_value), \
FN(sysctl_get_new_value), \
FN(sysctl_set_new_value), \
FN(strtol), \
FN(strtoul), \
FN(sk_storage_get), \
FN(sk_storage_delete), \
FN(send_signal),
#define __BPF_ENUM_FN(x) BPF_FUNC_ ## x
enum bpf_func_id {
__BPF_FUNC_MAPPER(__BPF_ENUM_FN)
__BPF_FUNC_MAX_ID,
};
#undef __BPF_ENUM_FN
// clang-format on
// BPF helper functions - this construction looks complicated, but actually
// it explained to just:
// static void* bpf_map_lookup_elem(void *map, void *key) = 1
// In other words bpf_map_lookup_elem points to memory address 0x1 - which is
// BPF function number 1.
// More details about helper functions at: http://docs.cilium.io/en/v1.1/bpf/
// Search for "Helper Functions"
// clang-format off
// Lookup bpf map element by key.
// Return: Map value or NULL
static void *(*bpf_map_lookup_elem)(const void *map, const void *key) = (void *) // NOLINT
BPF_FUNC_map_lookup_elem;
// Update bpf map element by key to value
// Return: 0 on success or negative error
static int (*bpf_map_update_elem)(const void *map, const void *key,
const void *value, __u64 flags) = (void *) // NOLINT
BPF_FUNC_map_update_elem;
// Delete element. Actually applicable on HASH maps
// Return: 0 on success or negative error
static int (*bpf_map_delete_elem)(const void *map, void *key) = (void *) // NOLINT
BPF_FUNC_map_delete_elem;
static int (*bpf_probe_read)(void *dst, __u64 size, const void *unsafe_ptr) = (void *) // NOLINT
BPF_FUNC_probe_read;
static __u64 (*bpf_ktime_get_ns)(void) = (void *) // NOLINT
BPF_FUNC_ktime_get_ns;
static __u32 (*bpf_get_prandom_u32)(void) = (void *) // NOLINT
BPF_FUNC_get_prandom_u32;
// Like printf() for BPF
// Return: length of buffer written or negative error
static int (*bpf_trace_printk)(const char *fmt, int fmt_size, ...) = (void *) // NOLINT
BPF_FUNC_trace_printk;
static int (*bpf_probe_read_str)(void *dst, __u64 size, const void *unsafe_ptr) = (void *) // NOLINT
BPF_FUNC_probe_read_str;
// Jump into another BPF program
// prog_array_map: pointer to map which type is BPF_MAP_TYPE_PROG_ARRAY
// index: 32-bit index inside array that selects specific program to run
// Return: 0 on success or negative error
static void (*bpf_tail_call)(const void *ctx, void *map, int index) = (void *) // NOLINT
BPF_FUNC_tail_call;
static int (*bpf_clone_redirect)(void *ctx, int ifindex, __u32 flags) = (void*) // NOLINT
BPF_FUNC_clone_redirect;
static __u64 (*bpf_get_smp_processor_id)(void) = (void*) // NOLINT
BPF_FUNC_get_smp_processor_id;
static __u64 (*bpf_get_current_pid_tgid)(void) = (void*) // NOLINT
BPF_FUNC_get_current_pid_tgid;
static __u64 (*bpf_get_current_uid_gid)(void) = (void*) // NOLINT
BPF_FUNC_get_current_uid_gid;
static int (*bpf_get_current_comm)(void *buf, int buf_size) = (void*) // NOLINT
BPF_FUNC_get_current_comm;
static __u64 (*bpf_get_cgroup_classid)(void *ctx) = (void*) // NOLINT
BPF_FUNC_get_cgroup_classid;
static __u64 (*bpf_skb_vlan_push)(void *ctx, __u16 proto, __u16 vlan_tci) = (void*) // NOLINT
BPF_FUNC_skb_vlan_push;
static __u64 (*bpf_skb_vlan_pop)(void *ctx) = (void*) // NOLINT
BPF_FUNC_skb_vlan_pop;
static int (*bpf_skb_get_tunnel_key)(void *ctx, void *to, __u32 size, __u64 flags) = (void*) // NOLINT
BPF_FUNC_skb_get_tunnel_key;
static int (*bpf_skb_set_tunnel_key)(void *ctx, void *from, __u32 size, __u64 flags) = (void*) // NOLINT
BPF_FUNC_skb_set_tunnel_key;
static __u64 (*bpf_perf_event_read)(void *map, __u64 flags) = (void*) // NOLINT
BPF_FUNC_perf_event_read;
static int (*bpf_redirect)(int ifindex, __u32 flags) = (void*) // NOLINT
BPF_FUNC_redirect;
static __u32 (*bpf_get_route_realm)(void *ctx) = (void*) // NOLINT
BPF_FUNC_get_route_realm;
static int (*bpf_perf_event_output)(void *ctx, void *map, __u64 index, void *data, __u32 size) = (void*) // NOLINT
BPF_FUNC_perf_event_output;
static int (*bpf_l3_csum_replace)(void *ctx, int offset, __u64 from, __u64 to, __u64 size) = (void *) // NOLINT
BPF_FUNC_l3_csum_replace;
static int (*bpf_l4_csum_replace)(void *ctx, int offset, __u64 from, __u64 to, __u64 flags) = (void *) // NOLINT
BPF_FUNC_l4_csum_replace;
static int (*bpf_skb_load_bytes)(void *ctx, int offset, void *to, __u32 len) = (void*) // NOLINT
BPF_FUNC_skb_load_bytes;
static int (*bpf_skb_store_bytes)(void *ctx, int offset, const void *from, __u32 len, __u64 flags) = (void *) // NOLINT
BPF_FUNC_skb_store_bytes;
static int (*bpf_perf_event_read_value)(void *map, __u64 flags, void *buf, __u32 buf_size) = (void*) // NOLINT
BPF_FUNC_perf_event_read_value;
static int (*bpf_perf_prog_read_value)(void *ctx, void *buf, __u32 buf_size) = (void*) // NOLINT
BPF_FUNC_perf_prog_read_value;
static int (*bpf_current_task_under_cgroup)(void *map, int index) = (void*) // NOLINT
BPF_FUNC_current_task_under_cgroup;
static __u32 (*bpf_get_socket_cookie)(void *ctx) = (void*) // NOLINT
BPF_FUNC_get_socket_cookie;
static __u64 (*bpf_get_socket_uid)(void *ctx) = (void*) // NOLINT
BPF_FUNC_get_socket_uid;
static int (*bpf_getsockopt)(void *ctx, int level, int optname, void *optval, int optlen) = (void*) // NOLINT
BPF_FUNC_getsockopt;
static int (*bpf_redirect_map)(void *map, __u32 key, __u64 flags) = (void*) // NOLINT
BPF_FUNC_redirect_map;
static int (*bpf_set_hash)(void *ctx, __u32 hash) = (void*) // NOLINT
BPF_FUNC_set_hash;
static int (*bpf_setsockopt)(void *ctx, int level, int optname, void *optval, int optlen) = (void*) // NOLINT
BPF_FUNC_setsockopt;
static int (*bpf_skb_adjust_room)(void *ctx, int len_diff, __u32 mode, __u64 flags) = (void*) // NOLINT
BPF_FUNC_skb_adjust_room;
static int (*bpf_skb_under_cgroup)(void *ctx, void *map, int index) = (void*) // NOLINT
BPF_FUNC_skb_under_cgroup;
static struct bpf_sock *(*bpf_skc_lookup_tcp)(void *ctx, struct bpf_sock_tuple *tuple, int size,
unsigned long long netns_id,
unsigned long long flags) = (void*) // NOLINT
BPF_FUNC_skc_lookup_tcp;
static int (*bpf_sk_redirect_map)(void *ctx, void *map, int key, int flags) = (void*) // NOLINT
BPF_FUNC_sk_redirect_map;
static int (*bpf_sock_map_update)(void *map, void *key, void *value, unsigned long long flags) = (void*) // NOLINT
BPF_FUNC_sock_map_update;
static int (*bpf_strtol)(const char *buf, size_t buf_len, __u64 flags, long *res) = (void*) // NOLINT
BPF_FUNC_strtol;
static int (*bpf_strtoul)(const char *buf, size_t buf_len, __u64 flags, unsigned long *res) = (void*) // NOLINT
BPF_FUNC_strtoul;
static int (*bpf_sysctl_get_current_value)(struct bpf_sysctl *ctx, char *buf, size_t buf_len) = (void*) // NOLINT
BPF_FUNC_sysctl_get_current_value;
static int (*bpf_sysctl_get_name)(struct bpf_sysctl *ctx, char *buf, size_t buf_len, __u64 flags) = (void*) // NOLINT
BPF_FUNC_sysctl_get_name;
static int (*bpf_sysctl_get_new_value)(struct bpf_sysctl *ctx, char *buf, size_t buf_len) = (void*) // NOLINT
BPF_FUNC_sysctl_get_new_value;
static int (*bpf_sysctl_set_new_value)(struct bpf_sysctl *ctx, const char *buf, size_t buf_len) = (void*) // NOLINT
BPF_FUNC_sysctl_set_new_value;
static int (*bpf_tcp_check_syncookie)(struct bpf_sock *sk, void *ip, int ip_len, void *tcp,
int tcp_len) = (void*) // NOLINT
BPF_FUNC_tcp_check_syncookie;
// Adjust the xdp_md.data_meta by delta
// ctx: pointer to xdp_md
// delta: An positive/negative integer to be added to ctx.data_meta
// Return: 0 on success or negative on error
static int (*bpf_xdp_adjust_meta)(void *ctx, int offset) = (void*) // NOLINT
BPF_FUNC_xdp_adjust_meta;
static int (*bpf_get_stackid)(void *ctx, void *map, __u64 flags) = (void*) // NOLINT
BPF_FUNC_get_stackid;
static int (*bpf_csum_diff)(void *from, __u64 from_size, void *to, __u64 to_size, __u64 seed) = (void*) // NOLINT
BPF_FUNC_csum_diff;
static int (*bpf_skb_get_tunnel_opt)(void *ctx, void *md, __u32 size) = (void*) // NOLINT
BPF_FUNC_skb_get_tunnel_opt;
static int (*bpf_skb_set_tunnel_opt)(void *ctx, void *md, __u32 size) = (void*) // NOLINT
BPF_FUNC_skb_set_tunnel_opt;
static int (*bpf_skb_change_proto)(void *ctx, __u16 proto, __u64 flags) = (void*) // NOLINT
BPF_FUNC_skb_change_proto;
static int (*bpf_skb_change_type)(void *ctx, __u32 type) = (void*) // NOLINT
BPF_FUNC_skb_change_type;
static __u32 (*bpf_get_hash_recalc)(void *ctx) = (void*) // NOLINT
BPF_FUNC_get_hash_recalc;
static __u64 (*bpf_get_current_task)(void) = (void*) // NOLINT
BPF_FUNC_get_current_task;
static int (*bpf_probe_write_user)(void *dst, void *src, __u32 size) = (void*) // NOLINT
BPF_FUNC_probe_write_user;
static int (*bpf_skb_change_tail)(void *ctx, __u32 new_len, __u64 flags) = (void*) // NOLINT
BPF_FUNC_skb_change_tail;
static int (*bpf_skb_pull_data)(void *ctx, __u32 len) = (void*) // NOLINT
BPF_FUNC_skb_pull_data;
static int (*bpf_csum_update)(void *ctx, __u16 csum) = (void*) // NOLINT
BPF_FUNC_csum_update;
static int (*bpf_set_hash_invalid)(void *ctx) = (void*) // NOLINT
BPF_FUNC_set_hash_invalid;
static int (*bpf_get_numa_node_id)(void) = (void*) // NOLINT
BPF_FUNC_get_numa_node_id;
static int (*bpf_skb_change_head)(void *ctx, __u32 len, __u64 flags) = (void*) // NOLINT
BPF_FUNC_skb_change_head;
static int (*bpf_override_return)(void *pt_regs, unsigned long rc) = (void*) // NOLINT
BPF_FUNC_override_return;
static int (*bpf_sock_ops_cb_flags_set)(void *skops, int flags) = (void*) // NOLINT
BPF_FUNC_sock_ops_cb_flags_set;
static int (*bpf_msg_redirect_map)(void *msg, void *map, __u32 key, __u64 flags) = (void*) // NOLINT
BPF_FUNC_msg_redirect_map;
static int (*bpf_msg_apply_bytes)(void *msg, __u32 bytes) = (void*) // NOLINT
BPF_FUNC_msg_apply_bytes;
static int (*bpf_msg_cork_bytes)(void *msg, __u32 bytes) = (void*) // NOLINT
BPF_FUNC_msg_cork_bytes;
static int (*bpf_msg_pull_data)(void *msg, __u32 start, __u32 end, __u64 flags) = (void*) // NOLINT
BPF_FUNC_msg_pull_data;
static int (*bpf_bind)(void *ctx, void *addr, int addr_len) = (void*) // NOLINT
BPF_FUNC_bind;
static int (*bpf_xdp_adjust_tail)(void *ctx, int offset) = (void*) // NOLINT
BPF_FUNC_xdp_adjust_tail;
static int (*bpf_skb_get_xfrm_state)(void *ctx, __u32 index, void *xfrm_state, __u32 size, __u64 flags) = (void*) // NOLINT
BPF_FUNC_skb_get_xfrm_state;
static int (*bpf_get_stack)(void *ctx, void *buf, __u32 size, __u64 flags) = (void*) // NOLINT
BPF_FUNC_get_stack;
static int (*bpf_skb_load_bytes_relative)(void *ctx, __u32 offset, void *to, __u32 len, __u32 start_header) = (void*) // NOLINT
BPF_FUNC_skb_load_bytes_relative;
static int (*bpf_fib_lookup)(void *ctx, void *params, int plen, __u32 flags) = (void*) // NOLINT
BPF_FUNC_fib_lookup;
static int (*bpf_sock_hash_update)(void *ctx, void *map, void *key, __u64 flags) = (void*) // NOLINT
BPF_FUNC_sock_hash_update;
static int (*bpf_msg_redirect_hash)(void *ctx, void *map, void *key, __u64 flags) = (void*) // NOLINT
BPF_FUNC_msg_redirect_hash;
static int (*bpf_sk_redirect_hash)(void *ctx, void *map, void *key, __u64 flags) = (void*) // NOLINT
BPF_FUNC_sk_redirect_hash;
static int (*bpf_lwt_push_encap)(void *skb, __u32 type, void *hdr, __u32 len) = (void*) // NOLINT
BPF_FUNC_lwt_push_encap;
static int (*bpf_lwt_seg6_store_bytes)(void *ctx, __u32 offset, const void *from, __u32 len) = (void*) // NOLINT
BPF_FUNC_lwt_seg6_store_bytes;
static int (*bpf_lwt_seg6_adjust_srh)(void *ctx, __u32 offset, __s32 delta) = (void*) // NOLINT
BPF_FUNC_lwt_seg6_adjust_srh;
static int (*bpf_lwt_seg6_action)(void *ctx, __u32 action, void *param, __u32 param_len) = (void*) // NOLINT
BPF_FUNC_lwt_seg6_action;
static int (*bpf_rc_keydown)(void *ctx, __u32 protocol, __u64 scancode, __u32 toggle) = (void*) // NOLINT
BPF_FUNC_rc_keydown;
static int (*bpf_rc_repeat)(void *ctx) = (void*) // NOLINT
BPF_FUNC_rc_repeat;
static __u64 (*bpf_skb_cgroup_id)(void *skb) = (void*) // NOLINT
BPF_FUNC_skb_cgroup_id;
static __u64 (*bpf_get_current_cgroup_id)(void) = (void*) // NOLINT
BPF_FUNC_get_current_cgroup_id;
static __u64 (*bpf_skb_ancestor_cgroup_id)(void *skb, int ancestor_level) = (void*) // NOLINT
BPF_FUNC_skb_ancestor_cgroup_id;
static void * (*bpf_get_local_storage)(void *map, __u64 flags) = (void*) // NOLINT
BPF_FUNC_get_local_storage;
static int (*bpf_sk_select_reuseport)(void *reuse, void *map, void *key, __u64 flags) = (void*) // NOLINT
BPF_FUNC_sk_select_reuseport;
static struct bpf_sock *(*bpf_sk_lookup_tcp)(void *ctx,
struct bpf_sock_tuple *tuple,
int size, unsigned int netns_id,
unsigned long long flags) = (void*) // NOLINT
BPF_FUNC_sk_lookup_tcp;
static struct bpf_sock *(*bpf_sk_lookup_udp)(void *ctx,
struct bpf_sock_tuple *tuple,
int size, unsigned int netns_id,
unsigned long long flags) = (void*) // NOLINT
BPF_FUNC_sk_lookup_udp;
static int (*bpf_sk_release)(struct bpf_sock *sk) = (void*) // NOLINT
BPF_FUNC_sk_release;
static int (*bpf_map_push_elem)(void *map, const void *value, __u64 flags) = (void*) // NOLINT
BPF_FUNC_map_push_elem;
static int (*bpf_map_pop_elem)(void *map, void *value) = (void*) // NOLINT
BPF_FUNC_map_pop_elem;
static int (*bpf_map_peek_elem)(void *map, void *value) = (void*) // NOLINT
BPF_FUNC_map_peek_elem;
static int (*bpf_msg_push_data)(void *skb, __u32 start, __u32 len, __u64 flags) = (void*) // NOLINT
BPF_FUNC_msg_push_data;
static int (*bpf_msg_pop_data)(void *msg, __u32 start, __u32 pop, __u64 flags) = (void*) // NOLINT
BPF_FUNC_msg_pop_data;
static int (*bpf_rc_pointer_rel)(void *ctx, __s32 rel_x, __s32 rel_y) = (void*) // NOLINT
BPF_FUNC_rc_pointer_rel;
static void (*bpf_spin_lock)(struct bpf_spin_lock *lock) = (void*) // NOLINT
BPF_FUNC_spin_lock;
static void (*bpf_spin_unlock)(struct bpf_spin_lock *lock) = (void*) // NOLINT
BPF_FUNC_spin_unlock;
static struct bpf_sock *(*bpf_sk_fullsock)(struct bpf_sock *sk) = (void*) // NOLINT
BPF_FUNC_sk_fullsock;
static struct bpf_tcp_sock *(*bpf_tcp_sock)(struct bpf_sock *sk) = (void*) // NOLINT
BPF_FUNC_tcp_sock;
static int (*bpf_skb_ecn_set_ce)(void *ctx) = (void*) // NOLINT
BPF_FUNC_skb_ecn_set_ce;
static struct bpf_sock *(*bpf_get_listener_sock)(struct bpf_sock *sk) = (void*) // NOLINT
BPF_FUNC_get_listener_sock;
static void *(*bpf_sk_storage_get)(void *map, struct bpf_sock *sk,
void *value, __u64 flags) = (void*) // NOLINT
BPF_FUNC_sk_storage_get;
static int (*bpf_sk_storage_delete)(void *map, struct bpf_sock *sk) = (void*) // NOLINT
BPF_FUNC_sk_storage_delete;
static int (*bpf_send_signal)(unsigned sig) = (void *) // NOLINT
BPF_FUNC_send_signal;
// Adjust the xdp_md.data by delta
// ctx: pointer to xdp_md
// delta: An positive/negative integer to be added to ctx.data
// Return: 0 on success or negative on error
static int (*bpf_xdp_adjust_head)(const void *ctx, int delta) = (void *) // NOLINT
BPF_FUNC_xdp_adjust_head;
// clang-format on
// printk() - kernel trace mechanism, like printf()
// To get trace (debug) messages:
// - Add #define DEBUG into your eBPF program before includes
// - $ sudo cat /sys/kernel/debug/tracing/trace
#ifdef DEBUG
#define bpf_printk(fmt, ...) \
({ \
char ____fmt[] = fmt; \
bpf_trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \
})
#else
#define bpf_printk(fmt, ...)
#endif
// Since BPF programs cannot perform any function calls other than
// those to BPF helpers, common library code needs to be implemented
// as inline functions. In addition, also LLVM provides some built-ins
// that can be used for constant sizes.
#define memset(dest, chr, n) __builtin_memset((dest), (chr), (n))
#define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n))
#define memmove(dest, src, n) __builtin_memmove((dest), (src), (n))
// Do not allow use printf()
#define printf(fmt, ...) do_not_use_printf_use_bpf_printk
// Macro to define BPF Map
#define BPF_MAP_DEF(name) struct bpf_map_def SEC("maps") name
#define BPF_MAP_ADD(x)
/* https://elixir.bootlin.com/linux/latest/source/include/uapi/linux/bpf.h#L4283 */
/* DIRECT: Skip the FIB rules and go to FIB table associated with device
* OUTPUT: Do lookup from egress perspective; default is ingress
*/
enum {
BPF_FIB_LOOKUP_DIRECT = (1U << 0),
BPF_FIB_LOOKUP_OUTPUT = (1U << 1),
};
enum {
BPF_FIB_LKUP_RET_SUCCESS, /* lookup successful */
BPF_FIB_LKUP_RET_BLACKHOLE, /* dest is blackholed; can be dropped */
BPF_FIB_LKUP_RET_UNREACHABLE, /* dest is unreachable; can be dropped */
BPF_FIB_LKUP_RET_PROHIBIT, /* dest not allowed; can be dropped */
BPF_FIB_LKUP_RET_NOT_FWDED, /* packet is not forwarded */
BPF_FIB_LKUP_RET_FWD_DISABLED, /* fwding is not enabled on ingress */
BPF_FIB_LKUP_RET_UNSUPP_LWT, /* fwd requires encapsulation */
BPF_FIB_LKUP_RET_NO_NEIGH, /* no neighbor entry for nh */
BPF_FIB_LKUP_RET_FRAG_NEEDED, /* fragmentation required to fwd */
};
struct bpf_fib_lookup {
/* input: network family for lookup (AF_INET, AF_INET6)
* output: network family of egress nexthop
*/
__u8 family;
/* set if lookup is to consider L4 data - e.g., FIB rules */
__u8 l4_protocol;
__be16 sport;
__be16 dport;
/* total length of packet from network header - used for MTU check */
__u16 tot_len;
/* input: L3 device index for lookup
* output: device index from FIB lookup
*/
__u32 ifindex;
union {
/* inputs to lookup */
__u8 tos; /* AF_INET */
__be32 flowinfo; /* AF_INET6, flow_label + priority */
/* output: metric of fib result (IPv4/IPv6 only) */
__u32 rt_metric;
};
union {
__be32 ipv4_src;
__u32 ipv6_src[4]; /* in6_addr; network order */
};
/* input to bpf_fib_lookup, ipv{4,6}_dst is destination address in
* network header. output: bpf_fib_lookup sets to gateway address
* if FIB lookup returns gateway route
*/
union {
__be32 ipv4_dst;
__u32 ipv6_dst[4]; /* in6_addr; network order */
};
/* output */
__be16 h_vlan_proto;
__be16 h_vlan_TCI;
__u8 smac[6]; /* ETH_ALEN */
__u8 dmac[6]; /* ETH_ALEN */
};
// offsetof gets the offset of a struct member
#ifndef offsetof
#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
#endif
///// end of __BPF__ /////
#else
//// All other platforms ////
// SEC() is useless for non eBPF - so just dummy
#define SEC(NAME)
// Functions must be inlined only for eBPF, so don't enforce it for *nix/mac.
// Also disable "unused function" warning -
// since eBPF programs define functions mostly in headers.
#define INLINE static __attribute__((unused))
// Disable warnings for "pragma unroll(all)"
#pragma GCC diagnostic ignored "-Wunknown-pragmas"
#include <assert.h>
#include <stdio.h>
#include <sys/queue.h>
#include <string.h>
// XDP metadata - defined twice because of real eBPF uses 32 bit pointers
// which are not acceptable for cross platform compilation.
struct xdp_md {
void *data;
void *data_end;
void *data_meta;
/* Below access go through struct xdp_rxq_info */
__u32 ingress_ifindex; /* rxq->dev->ifindex */
__u32 rx_queue_index; /* rxq->queue_index */
__u32 egress_ifindex; /* txq->dev->ifindex */
};
// Mock BPF map support:
// In order to automatically find all defined BPF maps from GO program we need
// to
// maintain linked list of maps (to be able to iterate and create them all)
// This could be easily and nicely done using __attribute__ ((constructor))
// Which is logically close to func init() int GO.
struct __create_map_def {
const char *name;
void *map_data; // Mock version only: holds head to single linked list of map
// items
struct bpf_map_def *map_def;
SLIST_ENTRY(__create_map_def) next;
};
// Declaration only. Definition held in mock_map package.
SLIST_HEAD(__maps_head_def, __create_map_def);
extern struct __maps_head_def *__maps_head;
#define BPF_MAP_DEF(x) static struct bpf_map_def x
#define BPF_MAP_ADD(x) \
static __attribute__((constructor)) void __bpf_map_##x() { \
static struct __create_map_def __bpf_map_entry_##x; \
__bpf_map_entry_##x.name = #x; \
__bpf_map_entry_##x.map_data = NULL; \
__bpf_map_entry_##x.map_def = &x; \
SLIST_INSERT_HEAD(__maps_head, &__bpf_map_entry_##x, next); \
}
// BPF helper prototypes - definition is up to mac/linux host program
void *bpf_map_lookup_elem(const void *map, const void *key);
int bpf_map_update_elem(const void *map, const void *key, const void *value,
__u64 flags);
int bpf_map_delete_elem(const void *map, const void *key);
// bpf_printk() is just printf()
#define bpf_printk(fmt, ...) \
printf(fmt, ##__VA_ARGS__); \
fflush(stdout);
// bpf_tail_call() is nothing: only relevant for BPF arch
#define bpf_tail_call(ctx, map, index)
// adjust_meta / ajdust_header are simple functions to move pointer
UNUSED static int bpf_xdp_adjust_meta(struct xdp_md *ctx, int offset) {
// For unittests only - function returns error if data_meta points to data_end
// which never the case in real world
if (ctx->data_meta == ctx->data_end) {
return 1;
}
ctx->data_meta = (__u8 *)ctx->data_meta + offset; // NOLINT
return 0;
}
UNUSED static int bpf_xdp_adjust_head(struct xdp_md *ctx, int offset) {
ctx->data = (__u8 *)ctx->data + offset; // NOLINT