History

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Alexei Starovoitov says: ==================== pull-request: bpf-next 2021-12-30 The following pull-request contains BPF updates for your *net-next* tree. We've added 72 non-merge commits during the last 20 day(s) which contain a total of 223 files changed, 3510 insertions(+), 1591 deletions(-). The main changes are: 1) Automatic setrlimit in libbpf when bpf is memcg's in the kernel, from Andrii. 2) Beautify and de-verbose verifier logs, from Christy. 3) Composable verifier types, from Hao. 4) bpf_strncmp helper, from Hou. 5) bpf.h header dependency cleanup, from Jakub. 6) get_func_[arg|ret|arg_cnt] helpers, from Jiri. 7) Sleepable local storage, from KP. 8) Extend kfunc with PTR_TO_CTX, PTR_TO_MEM argument support, from Kumar. ====================

David S. Miller committed 4 years ago

Commit e63a0232 parents ce2b6eb 9e6b19a221 files · +3506 −1591Browse files

Showing the changes to include (18 of 221 files). Show all

Unified · Split

18 files
  1. include/linux/bpf-cgroup-defs.h +70 −0
  2. include/linux/bpf-cgroup.h +1 −56
  3. include/linux/bpf-netns.h +1 −7
  4. include/linux/bpf.h +83 −24
  5. include/linux/bpf_local_storage.h +6 −0
  6. include/linux/bpf_verifier.h +27 −0
  7. include/linux/cgroup-defs.h +1 −1
  8. include/linux/dsa/loop.h +1 −0
  9. include/linux/filter.h +1 −1
  10. include/linux/perf_event.h +1 −0
  11. include/net/ip6_fib.h +1 −0
  12. include/net/ipv6.h +2 −0
  13. include/net/netns/bpf.h +8 −1
  14. include/net/route.h +1 −0
  15. include/net/sock.h +1 −1
  16. include/net/xdp_priv.h +0 −1
  17. include/net/xdp_sock.h +1 −0
  18. include/uapi/linux/bpf.h +39 −0
include/linux/bpf-cgroup-defs.h+70−0
@@ -0,0 +1,70 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+#ifndef _BPF_CGROUP_DEFS_H
3+#define _BPF_CGROUP_DEFS_H
4+
5+#ifdef CONFIG_CGROUP_BPF
6+
7+#include <linux/list.h>
8+#include <linux/percpu-refcount.h>
9+#include <linux/workqueue.h>
10+
11+struct bpf_prog_array;
12+
13+enum cgroup_bpf_attach_type {
14+ CGROUP_BPF_ATTACH_TYPE_INVALID = -1,
15+ CGROUP_INET_INGRESS = 0,
16+ CGROUP_INET_EGRESS,
17+ CGROUP_INET_SOCK_CREATE,
18+ CGROUP_SOCK_OPS,
19+ CGROUP_DEVICE,
20+ CGROUP_INET4_BIND,
21+ CGROUP_INET6_BIND,
22+ CGROUP_INET4_CONNECT,
23+ CGROUP_INET6_CONNECT,
24+ CGROUP_INET4_POST_BIND,
25+ CGROUP_INET6_POST_BIND,
26+ CGROUP_UDP4_SENDMSG,
27+ CGROUP_UDP6_SENDMSG,
28+ CGROUP_SYSCTL,
29+ CGROUP_UDP4_RECVMSG,
30+ CGROUP_UDP6_RECVMSG,
31+ CGROUP_GETSOCKOPT,
32+ CGROUP_SETSOCKOPT,
33+ CGROUP_INET4_GETPEERNAME,
34+ CGROUP_INET6_GETPEERNAME,
35+ CGROUP_INET4_GETSOCKNAME,
36+ CGROUP_INET6_GETSOCKNAME,
37+ CGROUP_INET_SOCK_RELEASE,
38+ MAX_CGROUP_BPF_ATTACH_TYPE
39+};
40+
41+struct cgroup_bpf {
42+ /* array of effective progs in this cgroup */
43+ struct bpf_prog_array __rcu *effective[MAX_CGROUP_BPF_ATTACH_TYPE];
44+
45+ /* attached progs to this cgroup and attach flags
46+ * when flags == 0 or BPF_F_ALLOW_OVERRIDE the progs list will
47+ * have either zero or one element
48+ * when BPF_F_ALLOW_MULTI the list can have up to BPF_CGROUP_MAX_PROGS
49+ */
50+ struct list_head progs[MAX_CGROUP_BPF_ATTACH_TYPE];
51+ u32 flags[MAX_CGROUP_BPF_ATTACH_TYPE];
52+
53+ /* list of cgroup shared storages */
54+ struct list_head storages;
55+
56+ /* temp storage for effective prog array used by prog_attach/detach */
57+ struct bpf_prog_array *inactive;
58+
59+ /* reference counter used to detach bpf programs after cgroup removal */
60+ struct percpu_ref refcnt;
61+
62+ /* cgroup_bpf is released using a work queue */
63+ struct work_struct release_work;
64+};
65+
66+#else /* CONFIG_CGROUP_BPF */
67+struct cgroup_bpf {};
68+#endif /* CONFIG_CGROUP_BPF */
69+
70+#endif
include/linux/bpf-cgroup.h+1−56
@@ -3,10 +3,10 @@ · 2 unmodified lines
33 #define _BPF_CGROUP_H
44
55 #include <linux/bpf.h>
6+#include <linux/bpf-cgroup-defs.h>
67 #include <linux/errno.h>
78 #include <linux/jump_label.h>
89 #include <linux/percpu.h>
9−#include <linux/percpu-refcount.h>
1010 #include <linux/rbtree.h>
1111 #include <uapi/linux/bpf.h>
1212
@@ -23,33 +23,6 @@ struct ctl_table_header; · 10 unmodified lines
2323 struct task_struct;
2424
2525 #ifdef CONFIG_CGROUP_BPF
26−enum cgroup_bpf_attach_type {
27− CGROUP_BPF_ATTACH_TYPE_INVALID = -1,
28− CGROUP_INET_INGRESS = 0,
29− CGROUP_INET_EGRESS,
30− CGROUP_INET_SOCK_CREATE,
31− CGROUP_SOCK_OPS,
32− CGROUP_DEVICE,
33− CGROUP_INET4_BIND,
34− CGROUP_INET6_BIND,
35− CGROUP_INET4_CONNECT,
36− CGROUP_INET6_CONNECT,
37− CGROUP_INET4_POST_BIND,
38− CGROUP_INET6_POST_BIND,
39− CGROUP_UDP4_SENDMSG,
40− CGROUP_UDP6_SENDMSG,
41− CGROUP_SYSCTL,
42− CGROUP_UDP4_RECVMSG,
43− CGROUP_UDP6_RECVMSG,
44− CGROUP_GETSOCKOPT,
45− CGROUP_SETSOCKOPT,
46− CGROUP_INET4_GETPEERNAME,
47− CGROUP_INET6_GETPEERNAME,
48− CGROUP_INET4_GETSOCKNAME,
49− CGROUP_INET6_GETSOCKNAME,
50− CGROUP_INET_SOCK_RELEASE,
51− MAX_CGROUP_BPF_ATTACH_TYPE
52−};
5326
5427 #define CGROUP_ATYPE(type) \
5528 case BPF_##type: return type
@@ -127,33 +100,6 @@ struct bpf_prog_list { · 71 unmodified lines
127100 struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE];
128101 };
129102
130−struct bpf_prog_array;
131−
132−struct cgroup_bpf {
133− /* array of effective progs in this cgroup */
134− struct bpf_prog_array __rcu *effective[MAX_CGROUP_BPF_ATTACH_TYPE];
135−
136− /* attached progs to this cgroup and attach flags
137− * when flags == 0 or BPF_F_ALLOW_OVERRIDE the progs list will
138− * have either zero or one element
139− * when BPF_F_ALLOW_MULTI the list can have up to BPF_CGROUP_MAX_PROGS
140− */
141− struct list_head progs[MAX_CGROUP_BPF_ATTACH_TYPE];
142− u32 flags[MAX_CGROUP_BPF_ATTACH_TYPE];
143−
144− /* list of cgroup shared storages */
145− struct list_head storages;
146−
147− /* temp storage for effective prog array used by prog_attach/detach */
148− struct bpf_prog_array *inactive;
149−
150− /* reference counter used to detach bpf programs after cgroup removal */
151− struct percpu_ref refcnt;
152−
153− /* cgroup_bpf is released using a work queue */
154− struct work_struct release_work;
155−};
156−
157103 int cgroup_bpf_inherit(struct cgroup *cgrp);
158104 void cgroup_bpf_offline(struct cgroup *cgrp);
159105
@@ -451,7 +397,6 @@ int cgroup_bpf_prog_query(const union bpf_attr *attr, · 291 unmodified lines
451397 union bpf_attr __user *uattr);
452398 #else
453399
454−struct cgroup_bpf {};
455400 static inline int cgroup_bpf_inherit(struct cgroup *cgrp) { return 0; }
456401 static inline void cgroup_bpf_offline(struct cgroup *cgrp) {}
457402
include/linux/bpf-netns.h+1−7
@@ -3,15 +3,9 @@ · 2 unmodified lines
33 #define _BPF_NETNS_H
44
55 #include <linux/mutex.h>
6+#include <net/netns/bpf.h>
67 #include <uapi/linux/bpf.h>
78
8−enum netns_bpf_attach_type {
9− NETNS_BPF_INVALID = -1,
10− NETNS_BPF_FLOW_DISSECTOR = 0,
11− NETNS_BPF_SK_LOOKUP,
12− MAX_NETNS_BPF_ATTACH_TYPE
13−};
14−
159 static inline enum netns_bpf_attach_type
1610 to_netns_bpf_attach_type(enum bpf_attach_type attach_type)
1711 {
include/linux/bpf.h+83−24
@@ -297,6 +297,34 @@ bool bpf_map_meta_equal(const struct bpf_map *meta0, · 296 unmodified lines
297297
298298 extern const struct bpf_map_ops bpf_map_offload_ops;
299299
300+/* bpf_type_flag contains a set of flags that are applicable to the values of
301+ * arg_type, ret_type and reg_type. For example, a pointer value may be null,
302+ * or a memory is read-only. We classify types into two categories: base types
303+ * and extended types. Extended types are base types combined with a type flag.
304+ *
305+ * Currently there are no more than 32 base types in arg_type, ret_type and
306+ * reg_types.
307+ */
308+#define BPF_BASE_TYPE_BITS 8
309+
310+enum bpf_type_flag {
311+ /* PTR may be NULL. */
312+ PTR_MAYBE_NULL = BIT(0 + BPF_BASE_TYPE_BITS),
313+
314+ /* MEM is read-only. When applied on bpf_arg, it indicates the arg is
315+ * compatible with both mutable and immutable memory.
316+ */
317+ MEM_RDONLY = BIT(1 + BPF_BASE_TYPE_BITS),
318+
319+ __BPF_TYPE_LAST_FLAG = MEM_RDONLY,
320+};
321+
322+/* Max number of base types. */
323+#define BPF_BASE_TYPE_LIMIT (1UL << BPF_BASE_TYPE_BITS)
324+
325+/* Max number of all types. */
326+#define BPF_TYPE_LIMIT (__BPF_TYPE_LAST_FLAG | (__BPF_TYPE_LAST_FLAG - 1))
327+
300328 /* function argument constraints */
301329 enum bpf_arg_type {
302330 ARG_DONTCARE = 0, /* unused argument in helper function */
@@ -308,13 +336,11 @@ enum bpf_arg_type { · 5 unmodified lines
308336 ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */
309337 ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */
310338 ARG_PTR_TO_UNINIT_MAP_VALUE, /* pointer to valid memory used to store a map value */
311− ARG_PTR_TO_MAP_VALUE_OR_NULL, /* pointer to stack used as map value or NULL */
312339
313340 /* the following constraints used to prototype bpf_memcmp() and other
314341 * functions that access data on eBPF program stack
315342 */
316343 ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */
317− ARG_PTR_TO_MEM_OR_NULL, /* pointer to valid memory or NULL */
318344 ARG_PTR_TO_UNINIT_MEM, /* pointer to memory does not need to be initialized,
319345 * helper function must fill all bytes or clear
320346 * them in error case.
@@ -324,42 +350,65 @@ enum bpf_arg_type { · 3 unmodified lines
324350 ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */
325351
326352 ARG_PTR_TO_CTX, /* pointer to context */
327− ARG_PTR_TO_CTX_OR_NULL, /* pointer to context or NULL */
328353 ARG_ANYTHING, /* any (initialized) argument is ok */
329354 ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */
330355 ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */
331356 ARG_PTR_TO_INT, /* pointer to int */
332357 ARG_PTR_TO_LONG, /* pointer to long */
333358 ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */
334− ARG_PTR_TO_SOCKET_OR_NULL, /* pointer to bpf_sock (fullsock) or NULL */
335359 ARG_PTR_TO_BTF_ID, /* pointer to in-kernel struct */
336360 ARG_PTR_TO_ALLOC_MEM, /* pointer to dynamically allocated memory */
337− ARG_PTR_TO_ALLOC_MEM_OR_NULL, /* pointer to dynamically allocated memory or NULL */
338361 ARG_CONST_ALLOC_SIZE_OR_ZERO, /* number of allocated bytes requested */
339362 ARG_PTR_TO_BTF_ID_SOCK_COMMON, /* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */
340363 ARG_PTR_TO_PERCPU_BTF_ID, /* pointer to in-kernel percpu type */
341364 ARG_PTR_TO_FUNC, /* pointer to a bpf program function */
342− ARG_PTR_TO_STACK_OR_NULL, /* pointer to stack or NULL */
365+ ARG_PTR_TO_STACK, /* pointer to stack */
343366 ARG_PTR_TO_CONST_STR, /* pointer to a null terminated read-only string */
344367 ARG_PTR_TO_TIMER, /* pointer to bpf_timer */
345368 __BPF_ARG_TYPE_MAX,
369+
370+ /* Extended arg_types. */
371+ ARG_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_MAP_VALUE,
372+ ARG_PTR_TO_MEM_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_MEM,
373+ ARG_PTR_TO_CTX_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_CTX,
374+ ARG_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_SOCKET,
375+ ARG_PTR_TO_ALLOC_MEM_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_ALLOC_MEM,
376+ ARG_PTR_TO_STACK_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_STACK,
377+
378+ /* This must be the last entry. Its purpose is to ensure the enum is
379+ * wide enough to hold the higher bits reserved for bpf_type_flag.
380+ */
381+ __BPF_ARG_TYPE_LIMIT = BPF_TYPE_LIMIT,
346382 };
383+static_assert(__BPF_ARG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
347384
348385 /* type of values returned from helper functions */
349386 enum bpf_return_type {
350387 RET_INTEGER, /* function returns integer */
351388 RET_VOID, /* function doesn't return anything */
352389 RET_PTR_TO_MAP_VALUE, /* returns a pointer to map elem value */
353− RET_PTR_TO_MAP_VALUE_OR_NULL, /* returns a pointer to map elem value or NULL */
354− RET_PTR_TO_SOCKET_OR_NULL, /* returns a pointer to a socket or NULL */
355− RET_PTR_TO_TCP_SOCK_OR_NULL, /* returns a pointer to a tcp_sock or NULL */
356− RET_PTR_TO_SOCK_COMMON_OR_NULL, /* returns a pointer to a sock_common or NULL */
357− RET_PTR_TO_ALLOC_MEM_OR_NULL, /* returns a pointer to dynamically allocated memory or NULL */
358− RET_PTR_TO_BTF_ID_OR_NULL, /* returns a pointer to a btf_id or NULL */
359− RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL, /* returns a pointer to a valid memory or a btf_id or NULL */
390+ RET_PTR_TO_SOCKET, /* returns a pointer to a socket */
391+ RET_PTR_TO_TCP_SOCK, /* returns a pointer to a tcp_sock */
392+ RET_PTR_TO_SOCK_COMMON, /* returns a pointer to a sock_common */
393+ RET_PTR_TO_ALLOC_MEM, /* returns a pointer to dynamically allocated memory */
360394 RET_PTR_TO_MEM_OR_BTF_ID, /* returns a pointer to a valid memory or a btf_id */
361395 RET_PTR_TO_BTF_ID, /* returns a pointer to a btf_id */
396+ __BPF_RET_TYPE_MAX,
397+
398+ /* Extended ret_types. */
399+ RET_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_MAP_VALUE,
400+ RET_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_SOCKET,
401+ RET_PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_TCP_SOCK,
402+ RET_PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_SOCK_COMMON,
403+ RET_PTR_TO_ALLOC_MEM_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_ALLOC_MEM,
404+ RET_PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_BTF_ID,
405+
406+ /* This must be the last entry. Its purpose is to ensure the enum is
407+ * wide enough to hold the higher bits reserved for bpf_type_flag.
408+ */
409+ __BPF_RET_TYPE_LIMIT = BPF_TYPE_LIMIT,
362410 };
411+static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
363412
364413 /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs
365414 * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL
@@ -421,18 +470,15 @@ enum bpf_reg_type { · 55 unmodified lines
421470 PTR_TO_CTX, /* reg points to bpf_context */
422471 CONST_PTR_TO_MAP, /* reg points to struct bpf_map */
423472 PTR_TO_MAP_VALUE, /* reg points to map element value */
424− PTR_TO_MAP_VALUE_OR_NULL,/* points to map elem value or NULL */
473+ PTR_TO_MAP_KEY, /* reg points to a map element key */
425474 PTR_TO_STACK, /* reg == frame_pointer + offset */
426475 PTR_TO_PACKET_META, /* skb->data - meta_len */
427476 PTR_TO_PACKET, /* reg points to skb->data */
428477 PTR_TO_PACKET_END, /* skb->data + headlen */
429478 PTR_TO_FLOW_KEYS, /* reg points to bpf_flow_keys */
430479 PTR_TO_SOCKET, /* reg points to struct bpf_sock */
431− PTR_TO_SOCKET_OR_NULL, /* reg points to struct bpf_sock or NULL */
432480 PTR_TO_SOCK_COMMON, /* reg points to sock_common */
433− PTR_TO_SOCK_COMMON_OR_NULL, /* reg points to sock_common or NULL */
434481 PTR_TO_TCP_SOCK, /* reg points to struct tcp_sock */
435− PTR_TO_TCP_SOCK_OR_NULL, /* reg points to struct tcp_sock or NULL */
436482 PTR_TO_TP_BUFFER, /* reg points to a writable raw tp's buffer */
437483 PTR_TO_XDP_SOCK, /* reg points to struct xdp_sock */
438484 /* PTR_TO_BTF_ID points to a kernel struct that does not need
@@ -450,18 +496,25 @@ enum bpf_reg_type { · 11 unmodified lines
450496 * been checked for null. Used primarily to inform the verifier
451497 * an explicit null check is required for this struct.
452498 */
453− PTR_TO_BTF_ID_OR_NULL,
454499 PTR_TO_MEM, /* reg points to valid memory region */
455− PTR_TO_MEM_OR_NULL, /* reg points to valid memory region or NULL */
456− PTR_TO_RDONLY_BUF, /* reg points to a readonly buffer */
457− PTR_TO_RDONLY_BUF_OR_NULL, /* reg points to a readonly buffer or NULL */
458− PTR_TO_RDWR_BUF, /* reg points to a read/write buffer */
459− PTR_TO_RDWR_BUF_OR_NULL, /* reg points to a read/write buffer or NULL */
500+ PTR_TO_BUF, /* reg points to a read/write buffer */
460501 PTR_TO_PERCPU_BTF_ID, /* reg points to a percpu kernel variable */
461502 PTR_TO_FUNC, /* reg points to a bpf program function */
462− PTR_TO_MAP_KEY, /* reg points to a map element key */
463503 __BPF_REG_TYPE_MAX,
504+
505+ /* Extended reg_types. */
506+ PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | PTR_TO_MAP_VALUE,
507+ PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCKET,
508+ PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCK_COMMON,
509+ PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | PTR_TO_TCP_SOCK,
510+ PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | PTR_TO_BTF_ID,
511+
512+ /* This must be the last entry. Its purpose is to ensure the enum is
513+ * wide enough to hold the higher bits reserved for bpf_type_flag.
514+ */
515+ __BPF_REG_TYPE_LIMIT = BPF_TYPE_LIMIT,
464516 };
517+static_assert(__BPF_REG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
465518
466519 /* The information passed from prog-specific *_is_valid_access
467520 * back to the verifier.
@@ -777,6 +830,7 @@ void bpf_ksym_add(struct bpf_ksym *ksym); · 309 unmodified lines
777830 void bpf_ksym_del(struct bpf_ksym *ksym);
778831 int bpf_jit_charge_modmem(u32 pages);
779832 void bpf_jit_uncharge_modmem(u32 pages);
833+bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
780834 #else
781835 static inline int bpf_trampoline_link_prog(struct bpf_prog *prog,
782836 struct bpf_trampoline *tr)
@@ -805,6 +859,10 @@ static inline bool is_bpf_image_address(unsigned long address) · 22 unmodified lines
805859 {
806860 return false;
807861 }
862+static inline bool bpf_prog_has_trampoline(const struct bpf_prog *prog)
863+{
864+ return false;
865+}
808866 #endif
809867
810868 struct bpf_func_info_aux {
@@ -2163,6 +2221,7 @@ extern const struct bpf_func_proto bpf_sk_getsockopt_proto; · 1352 unmodified lines
21632221 extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;
21642222 extern const struct bpf_func_proto bpf_find_vma_proto;
21652223 extern const struct bpf_func_proto bpf_loop_proto;
2224+extern const struct bpf_func_proto bpf_strncmp_proto;
21662225
21672226 const struct bpf_func_proto *tracing_prog_func_proto(
21682227 enum bpf_func_id func_id, const struct bpf_prog *prog);
include/linux/bpf_local_storage.h+6−0
@@ -8,6 +8,7 @@ · 7 unmodified lines
88 #define _BPF_LOCAL_STORAGE_H
99
1010 #include <linux/bpf.h>
11+#include <linux/filter.h>
1112 #include <linux/rculist.h>
1213 #include <linux/list.h>
1314 #include <linux/hash.h>
@@ -16,6 +17,9 @@ · 2 unmodified lines
1617
1718 #define BPF_LOCAL_STORAGE_CACHE_SIZE 16
1819
20+#define bpf_rcu_lock_held() \
21+ (rcu_read_lock_held() || rcu_read_lock_trace_held() || \
22+ rcu_read_lock_bh_held())
1923 struct bpf_local_storage_map_bucket {
2024 struct hlist_head list;
2125 raw_spinlock_t lock;
@@ -161,4 +165,6 @@ struct bpf_local_storage_data * · 139 unmodified lines
161165 bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
162166 void *value, u64 map_flags);
163167
168+void bpf_local_storage_free_rcu(struct rcu_head *rcu);
169+
164170 #endif /* _BPF_LOCAL_STORAGE_H */
include/linux/bpf_verifier.h+27−0
@@ -18,6 +18,8 @@ · 17 unmodified lines
1818 * that converting umax_value to int cannot overflow.
1919 */
2020 #define BPF_MAX_VAR_SIZ (1 << 29)
21+/* size of type_str_buf in bpf_verifier. */
22+#define TYPE_STR_BUF_LEN 64
2123
2224 /* Liveness marks, used for registers and spilled-regs (in stack slots).
2325 * Read marks propagate upwards until they find a write mark; they record that
@@ -388,6 +390,8 @@ static inline bool bpf_verifier_log_full(const struct bpf_verifier_log *log) · 364 unmodified lines
388390 #define BPF_LOG_LEVEL (BPF_LOG_LEVEL1 | BPF_LOG_LEVEL2)
389391 #define BPF_LOG_MASK (BPF_LOG_LEVEL | BPF_LOG_STATS)
390392 #define BPF_LOG_KERNEL (BPF_LOG_MASK + 1) /* kernel internal flag */
393+#define BPF_LOG_MIN_ALIGNMENT 8U
394+#define BPF_LOG_ALIGNMENT 40U
391395
392396 static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log)
393397 {
@@ -474,6 +478,16 @@ struct bpf_verifier_env { · 80 unmodified lines
474478 /* longest register parentage chain walked for liveness marking */
475479 u32 longest_mark_read_walk;
476480 bpfptr_t fd_array;
481+
482+ /* bit mask to keep track of whether a register has been accessed
483+ * since the last time the function state was printed
484+ */
485+ u32 scratched_regs;
486+ /* Same as scratched_regs but for stack slots */
487+ u64 scratched_stack_slots;
488+ u32 prev_log_len, prev_insn_print_len;
489+ /* buffer used in reg_type_str() to generate reg_type string */
490+ char type_str_buf[TYPE_STR_BUF_LEN];
477491 };
478492
479493 __printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log,
@@ -536,5 +550,18 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, · 56 unmodified lines
536550 struct bpf_attach_target_info *tgt_info);
537551 void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab);
538552
553+#define BPF_BASE_TYPE_MASK GENMASK(BPF_BASE_TYPE_BITS - 1, 0)
554+
555+/* extract base type from bpf_{arg, return, reg}_type. */
556+static inline u32 base_type(u32 type)
557+{
558+ return type & BPF_BASE_TYPE_MASK;
559+}
560+
561+/* extract flags from an extended type. See bpf_type_flag in bpf.h. */
562+static inline u32 type_flag(u32 type)
563+{
564+ return type & ~BPF_BASE_TYPE_MASK;
565+}
539566
540567 #endif /* _LINUX_BPF_VERIFIER_H */
include/linux/cgroup-defs.h+1−1
@@ -19,7 +19,7 @@ · 18 unmodified lines
1919 #include <linux/percpu-rwsem.h>
2020 #include <linux/u64_stats_sync.h>
2121 #include <linux/workqueue.h>
22−#include <linux/bpf-cgroup.h>
22+#include <linux/bpf-cgroup-defs.h>
2323 #include <linux/psi_types.h>
2424
2525 #ifdef CONFIG_CGROUPS
include/linux/dsa/loop.h+1−0
@@ -2,6 +2,7 @@ · 1 unmodified line
22 #ifndef DSA_LOOP_H
33 #define DSA_LOOP_H
44
5+#include <linux/if_vlan.h>
56 #include <linux/types.h>
67 #include <linux/ethtool.h>
78 #include <net/dsa.h>
include/linux/filter.h+1−1
@@ -1027,7 +1027,7 @@ void xdp_do_flush(void); · 1026 unmodified lines
10271027 */
10281028 #define xdp_do_flush_map xdp_do_flush
10291029
1030−void bpf_warn_invalid_xdp_action(u32 act);
1030+void bpf_warn_invalid_xdp_action(struct net_device *dev, struct bpf_prog *prog, u32 act);
10311031
10321032 #ifdef CONFIG_INET
10331033 struct sock *bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk,
include/linux/perf_event.h+1−0
@@ -611,6 +611,7 @@ struct swevent_hlist { · 610 unmodified lines
611611 #define PERF_ATTACH_SCHED_CB 0x20
612612 #define PERF_ATTACH_CHILD 0x40
613613
614+struct bpf_prog;
614615 struct perf_cgroup;
615616 struct perf_buffer;
616617
include/net/ip6_fib.h+1−0
@@ -20,6 +20,7 @@ · 19 unmodified lines
2020 #include <net/inetpeer.h>
2121 #include <net/fib_notifier.h>
2222 #include <linux/indirect_call_wrapper.h>
23+#include <uapi/linux/bpf.h>
2324
2425 #ifdef CONFIG_IPV6_MULTIPLE_TABLES
2526 #define FIB6_TABLE_HASHSZ 256
include/net/ipv6.h+2−0
@@ -21,6 +21,8 @@ · 20 unmodified lines
2121 #include <net/snmp.h>
2222 #include <net/netns/hash.h>
2323
24+struct ip_tunnel_info;
25+
2426 #define SIN6_LEN_RFC2133 24
2527
2628 #define IPV6_MAXPLEN 65535
include/net/netns/bpf.h+8−1
@@ -6,11 +6,18 @@ · 5 unmodified lines
66 #ifndef __NETNS_BPF_H__
77 #define __NETNS_BPF_H__
88
9−#include <linux/bpf-netns.h>
9+#include <linux/list.h>
1010
1111 struct bpf_prog;
1212 struct bpf_prog_array;
1313
14+enum netns_bpf_attach_type {
15+ NETNS_BPF_INVALID = -1,
16+ NETNS_BPF_FLOW_DISSECTOR = 0,
17+ NETNS_BPF_SK_LOOKUP,
18+ MAX_NETNS_BPF_ATTACH_TYPE
19+};
20+
1421 struct netns_bpf {
1522 /* Array of programs to run compiled from progs or links */
1623 struct bpf_prog_array __rcu *run_array[MAX_NETNS_BPF_ATTACH_TYPE];
include/net/route.h+1−0
@@ -43,6 +43,7 @@ · 42 unmodified lines
4343 #define RT_CONN_FLAGS(sk) (RT_TOS(inet_sk(sk)->tos) | sock_flag(sk, SOCK_LOCALROUTE))
4444 #define RT_CONN_FLAGS_TOS(sk,tos) (RT_TOS(tos) | sock_flag(sk, SOCK_LOCALROUTE))
4545
46+struct ip_tunnel_info;
4647 struct fib_nh;
4748 struct fib_info;
4849 struct uncached_list;
include/net/sock.h+1−1
@@ -56,7 +56,6 @@ · 55 unmodified lines
5656 #include <linux/wait.h>
5757 #include <linux/cgroup-defs.h>
5858 #include <linux/rbtree.h>
59−#include <linux/filter.h>
6059 #include <linux/rculist_nulls.h>
6160 #include <linux/poll.h>
6261 #include <linux/sockptr.h>
@@ -249,6 +248,7 @@ struct sock_common { · 186 unmodified lines
249248 };
250249
251250 struct bpf_local_storage;
251+struct sk_filter;
252252
253253 /**
254254 * struct sock - network layer representation of sockets
include/net/xdp_priv.h+0−1
@@ -10,7 +10,6 @@ struct xdp_mem_allocator { · 9 unmodified lines
1010 union {
1111 void *allocator;
1212 struct page_pool *page_pool;
13− struct zero_copy_allocator *zc_alloc;
1413 };
1514 struct rhash_head node;
1615 struct rcu_head rcu;
include/net/xdp_sock.h+1−0
@@ -6,6 +6,7 @@ · 5 unmodified lines
66 #ifndef _LINUX_XDP_SOCK_H
77 #define _LINUX_XDP_SOCK_H
88
9+#include <linux/bpf.h>
910 #include <linux/workqueue.h>
1011 #include <linux/if_xdp.h>
1112 #include <linux/mutex.h>
include/uapi/linux/bpf.h+39−0
@@ -4983,6 +4983,41 @@ union bpf_attr { · 4982 unmodified lines
49834983 * Return
49844984 * The number of loops performed, **-EINVAL** for invalid **flags**,
49854985 * **-E2BIG** if **nr_loops** exceeds the maximum number of loops.
4986+ *
4987+ * long bpf_strncmp(const char *s1, u32 s1_sz, const char *s2)
4988+ * Description
4989+ * Do strncmp() between **s1** and **s2**. **s1** doesn't need
4990+ * to be null-terminated and **s1_sz** is the maximum storage
4991+ * size of **s1**. **s2** must be a read-only string.
4992+ * Return
4993+ * An integer less than, equal to, or greater than zero
4994+ * if the first **s1_sz** bytes of **s1** is found to be
4995+ * less than, to match, or be greater than **s2**.
4996+ *
4997+ * long bpf_get_func_arg(void *ctx, u32 n, u64 *value)
4998+ * Description
4999+ * Get **n**-th argument (zero based) of the traced function (for tracing programs)
5000+ * returned in **value**.
5001+ *
5002+ * Return
5003+ * 0 on success.
5004+ * **-EINVAL** if n >= arguments count of traced function.
5005+ *
5006+ * long bpf_get_func_ret(void *ctx, u64 *value)
5007+ * Description
5008+ * Get return value of the traced function (for tracing programs)
5009+ * in **value**.
5010+ *
5011+ * Return
5012+ * 0 on success.
5013+ * **-EOPNOTSUPP** for tracing programs other than BPF_TRACE_FEXIT or BPF_MODIFY_RETURN.
5014+ *
5015+ * long bpf_get_func_arg_cnt(void *ctx)
5016+ * Description
5017+ * Get number of arguments of the traced function (for tracing programs).
5018+ *
5019+ * Return
5020+ * The number of arguments of the traced function.
49865021 */
49875022 #define __BPF_FUNC_MAPPER(FN) \
49885023 FN(unspec), \
@@ -5167,6 +5202,10 @@ union bpf_attr { · 178 unmodified lines
51675202 FN(kallsyms_lookup_name), \
51685203 FN(find_vma), \
51695204 FN(loop), \
5205+ FN(strncmp), \
5206+ FN(get_func_arg), \
5207+ FN(get_func_ret), \
5208+ FN(get_func_arg_cnt), \
51705209 /* */
51715210
51725211 /* integer value in 'imm' field of BPF_CALL instruction selects which helper