History

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Alexei Starovoitov says: ==================== pull-request: bpf-next 2021-12-30 The following pull-request contains BPF updates for your *net-next* tree. We've added 72 non-merge commits during the last 20 day(s) which contain a total of 223 files changed, 3510 insertions(+), 1591 deletions(-). The main changes are: 1) Automatic setrlimit in libbpf when bpf is memcg's in the kernel, from Andrii. 2) Beautify and de-verbose verifier logs, from Christy. 3) Composable verifier types, from Hao. 4) bpf_strncmp helper, from Hou. 5) bpf.h header dependency cleanup, from Jakub. 6) get_func_[arg|ret|arg_cnt] helpers, from Jiri. 7) Sleepable local storage, from KP. 8) Extend kfunc with PTR_TO_CTX, PTR_TO_MEM argument support, from Kumar. ====================

David S. Miller committed 4 years ago

Commit e63a0232 parents ce2b6eb 9e6b19a221 files · +3506 −1591Browse files

Showing the changes to include (18 of 221 files). Show all

Unified · Split

18 files
  1. include/linux/bpf-cgroup-defs.h +70 −0
  2. include/linux/bpf-cgroup.h +1 −56
  3. include/linux/bpf-netns.h +1 −7
  4. include/linux/bpf.h +83 −24
  5. include/linux/bpf_local_storage.h +6 −0
  6. include/linux/bpf_verifier.h +27 −0
  7. include/linux/cgroup-defs.h +1 −1
  8. include/linux/dsa/loop.h +1 −0
  9. include/linux/filter.h +1 −1
  10. include/linux/perf_event.h +1 −0
  11. include/net/ip6_fib.h +1 −0
  12. include/net/ipv6.h +2 −0
  13. include/net/netns/bpf.h +8 −1
  14. include/net/route.h +1 −0
  15. include/net/sock.h +1 −1
  16. include/net/xdp_priv.h +0 −1
  17. include/net/xdp_sock.h +1 −0
  18. include/uapi/linux/bpf.h +39 −0
include/linux/bpf-cgroup-defs.h+70−0
@@ -0,0 +1,70 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _BPF_CGROUP_DEFS_H
3#define _BPF_CGROUP_DEFS_H
4
5#ifdef CONFIG_CGROUP_BPF
6
7#include <linux/list.h>
8#include <linux/percpu-refcount.h>
9#include <linux/workqueue.h>
10
11struct bpf_prog_array;
12
13enum cgroup_bpf_attach_type {
14 CGROUP_BPF_ATTACH_TYPE_INVALID = -1,
15 CGROUP_INET_INGRESS = 0,
16 CGROUP_INET_EGRESS,
17 CGROUP_INET_SOCK_CREATE,
18 CGROUP_SOCK_OPS,
19 CGROUP_DEVICE,
20 CGROUP_INET4_BIND,
21 CGROUP_INET6_BIND,
22 CGROUP_INET4_CONNECT,
23 CGROUP_INET6_CONNECT,
24 CGROUP_INET4_POST_BIND,
25 CGROUP_INET6_POST_BIND,
26 CGROUP_UDP4_SENDMSG,
27 CGROUP_UDP6_SENDMSG,
28 CGROUP_SYSCTL,
29 CGROUP_UDP4_RECVMSG,
30 CGROUP_UDP6_RECVMSG,
31 CGROUP_GETSOCKOPT,
32 CGROUP_SETSOCKOPT,
33 CGROUP_INET4_GETPEERNAME,
34 CGROUP_INET6_GETPEERNAME,
35 CGROUP_INET4_GETSOCKNAME,
36 CGROUP_INET6_GETSOCKNAME,
37 CGROUP_INET_SOCK_RELEASE,
38 MAX_CGROUP_BPF_ATTACH_TYPE
39};
40
41struct cgroup_bpf {
42 /* array of effective progs in this cgroup */
43 struct bpf_prog_array __rcu *effective[MAX_CGROUP_BPF_ATTACH_TYPE];
44
45 /* attached progs to this cgroup and attach flags
46 * when flags == 0 or BPF_F_ALLOW_OVERRIDE the progs list will
47 * have either zero or one element
48 * when BPF_F_ALLOW_MULTI the list can have up to BPF_CGROUP_MAX_PROGS
49 */
50 struct list_head progs[MAX_CGROUP_BPF_ATTACH_TYPE];
51 u32 flags[MAX_CGROUP_BPF_ATTACH_TYPE];
52
53 /* list of cgroup shared storages */
54 struct list_head storages;
55
56 /* temp storage for effective prog array used by prog_attach/detach */
57 struct bpf_prog_array *inactive;
58
59 /* reference counter used to detach bpf programs after cgroup removal */
60 struct percpu_ref refcnt;
61
62 /* cgroup_bpf is released using a work queue */
63 struct work_struct release_work;
64};
65
66#else /* CONFIG_CGROUP_BPF */
67struct cgroup_bpf {};
68#endif /* CONFIG_CGROUP_BPF */
69
70#endif
include/linux/bpf-cgroup.h+1−56
@@ -3,10 +3,10 @@ · 2 unmodified lines
3#define _BPF_CGROUP_H3#define _BPF_CGROUP_H
4 4
5#include <linux/bpf.h>5#include <linux/bpf.h>
6#include <linux/bpf-cgroup-defs.h>
6#include <linux/errno.h>7#include <linux/errno.h>
7#include <linux/jump_label.h>8#include <linux/jump_label.h>
8#include <linux/percpu.h>9#include <linux/percpu.h>
9#include <linux/percpu-refcount.h>
10#include <linux/rbtree.h>10#include <linux/rbtree.h>
11#include <uapi/linux/bpf.h>11#include <uapi/linux/bpf.h>
12 12
@@ -23,33 +23,6 @@ struct ctl_table_header; · 10 unmodified lines
23struct task_struct;23struct task_struct;
24 24
25#ifdef CONFIG_CGROUP_BPF25#ifdef CONFIG_CGROUP_BPF
26enum cgroup_bpf_attach_type {
27 CGROUP_BPF_ATTACH_TYPE_INVALID = -1,
28 CGROUP_INET_INGRESS = 0,
29 CGROUP_INET_EGRESS,
30 CGROUP_INET_SOCK_CREATE,
31 CGROUP_SOCK_OPS,
32 CGROUP_DEVICE,
33 CGROUP_INET4_BIND,
34 CGROUP_INET6_BIND,
35 CGROUP_INET4_CONNECT,
36 CGROUP_INET6_CONNECT,
37 CGROUP_INET4_POST_BIND,
38 CGROUP_INET6_POST_BIND,
39 CGROUP_UDP4_SENDMSG,
40 CGROUP_UDP6_SENDMSG,
41 CGROUP_SYSCTL,
42 CGROUP_UDP4_RECVMSG,
43 CGROUP_UDP6_RECVMSG,
44 CGROUP_GETSOCKOPT,
45 CGROUP_SETSOCKOPT,
46 CGROUP_INET4_GETPEERNAME,
47 CGROUP_INET6_GETPEERNAME,
48 CGROUP_INET4_GETSOCKNAME,
49 CGROUP_INET6_GETSOCKNAME,
50 CGROUP_INET_SOCK_RELEASE,
51 MAX_CGROUP_BPF_ATTACH_TYPE
52};
53 26
54#define CGROUP_ATYPE(type) \27#define CGROUP_ATYPE(type) \
55 case BPF_##type: return type28 case BPF_##type: return type
@@ -127,33 +100,6 @@ struct bpf_prog_list { · 71 unmodified lines
127 struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE];100 struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE];
128};101};
129 102
130struct bpf_prog_array;
131
132struct cgroup_bpf {
133 /* array of effective progs in this cgroup */
134 struct bpf_prog_array __rcu *effective[MAX_CGROUP_BPF_ATTACH_TYPE];
135
136 /* attached progs to this cgroup and attach flags
137 * when flags == 0 or BPF_F_ALLOW_OVERRIDE the progs list will
138 * have either zero or one element
139 * when BPF_F_ALLOW_MULTI the list can have up to BPF_CGROUP_MAX_PROGS
140 */
141 struct list_head progs[MAX_CGROUP_BPF_ATTACH_TYPE];
142 u32 flags[MAX_CGROUP_BPF_ATTACH_TYPE];
143
144 /* list of cgroup shared storages */
145 struct list_head storages;
146
147 /* temp storage for effective prog array used by prog_attach/detach */
148 struct bpf_prog_array *inactive;
149
150 /* reference counter used to detach bpf programs after cgroup removal */
151 struct percpu_ref refcnt;
152
153 /* cgroup_bpf is released using a work queue */
154 struct work_struct release_work;
155};
156
157int cgroup_bpf_inherit(struct cgroup *cgrp);103int cgroup_bpf_inherit(struct cgroup *cgrp);
158void cgroup_bpf_offline(struct cgroup *cgrp);104void cgroup_bpf_offline(struct cgroup *cgrp);
159 105
@@ -451,7 +397,6 @@ int cgroup_bpf_prog_query(const union bpf_attr *attr, · 291 unmodified lines
451 union bpf_attr __user *uattr);397 union bpf_attr __user *uattr);
452#else398#else
453 399
454struct cgroup_bpf {};
455static inline int cgroup_bpf_inherit(struct cgroup *cgrp) { return 0; }400static inline int cgroup_bpf_inherit(struct cgroup *cgrp) { return 0; }
456static inline void cgroup_bpf_offline(struct cgroup *cgrp) {}401static inline void cgroup_bpf_offline(struct cgroup *cgrp) {}
457 402
include/linux/bpf-netns.h+1−7
@@ -3,15 +3,9 @@ · 2 unmodified lines
3#define _BPF_NETNS_H3#define _BPF_NETNS_H
4 4
5#include <linux/mutex.h>5#include <linux/mutex.h>
6#include <net/netns/bpf.h>
6#include <uapi/linux/bpf.h>7#include <uapi/linux/bpf.h>
7 8
8enum netns_bpf_attach_type {
9 NETNS_BPF_INVALID = -1,
10 NETNS_BPF_FLOW_DISSECTOR = 0,
11 NETNS_BPF_SK_LOOKUP,
12 MAX_NETNS_BPF_ATTACH_TYPE
13};
14
15static inline enum netns_bpf_attach_type9static inline enum netns_bpf_attach_type
16to_netns_bpf_attach_type(enum bpf_attach_type attach_type)10to_netns_bpf_attach_type(enum bpf_attach_type attach_type)
17{11{
include/linux/bpf.h+83−24
@@ -297,6 +297,34 @@ bool bpf_map_meta_equal(const struct bpf_map *meta0, · 296 unmodified lines
297 297
298extern const struct bpf_map_ops bpf_map_offload_ops;298extern const struct bpf_map_ops bpf_map_offload_ops;
299 299
300/* bpf_type_flag contains a set of flags that are applicable to the values of
301 * arg_type, ret_type and reg_type. For example, a pointer value may be null,
302 * or a memory is read-only. We classify types into two categories: base types
303 * and extended types. Extended types are base types combined with a type flag.
304 *
305 * Currently there are no more than 32 base types in arg_type, ret_type and
306 * reg_types.
307 */
308#define BPF_BASE_TYPE_BITS 8
309
310enum bpf_type_flag {
311 /* PTR may be NULL. */
312 PTR_MAYBE_NULL = BIT(0 + BPF_BASE_TYPE_BITS),
313
314 /* MEM is read-only. When applied on bpf_arg, it indicates the arg is
315 * compatible with both mutable and immutable memory.
316 */
317 MEM_RDONLY = BIT(1 + BPF_BASE_TYPE_BITS),
318
319 __BPF_TYPE_LAST_FLAG = MEM_RDONLY,
320};
321
322/* Max number of base types. */
323#define BPF_BASE_TYPE_LIMIT (1UL << BPF_BASE_TYPE_BITS)
324
325/* Max number of all types. */
326#define BPF_TYPE_LIMIT (__BPF_TYPE_LAST_FLAG | (__BPF_TYPE_LAST_FLAG - 1))
327
300/* function argument constraints */328/* function argument constraints */
301enum bpf_arg_type {329enum bpf_arg_type {
302 ARG_DONTCARE = 0, /* unused argument in helper function */330 ARG_DONTCARE = 0, /* unused argument in helper function */
@@ -308,13 +336,11 @@ enum bpf_arg_type { · 5 unmodified lines
308 ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */336 ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */
309 ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */337 ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */
310 ARG_PTR_TO_UNINIT_MAP_VALUE, /* pointer to valid memory used to store a map value */338 ARG_PTR_TO_UNINIT_MAP_VALUE, /* pointer to valid memory used to store a map value */
311 ARG_PTR_TO_MAP_VALUE_OR_NULL, /* pointer to stack used as map value or NULL */
312 339
313 /* the following constraints used to prototype bpf_memcmp() and other340 /* the following constraints used to prototype bpf_memcmp() and other
314 * functions that access data on eBPF program stack341 * functions that access data on eBPF program stack
315 */342 */
316 ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */343 ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */
317 ARG_PTR_TO_MEM_OR_NULL, /* pointer to valid memory or NULL */
318 ARG_PTR_TO_UNINIT_MEM, /* pointer to memory does not need to be initialized,344 ARG_PTR_TO_UNINIT_MEM, /* pointer to memory does not need to be initialized,
319 * helper function must fill all bytes or clear345 * helper function must fill all bytes or clear
320 * them in error case.346 * them in error case.
@@ -324,42 +350,65 @@ enum bpf_arg_type { · 3 unmodified lines
324 ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */350 ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */
325 351
326 ARG_PTR_TO_CTX, /* pointer to context */352 ARG_PTR_TO_CTX, /* pointer to context */
327 ARG_PTR_TO_CTX_OR_NULL, /* pointer to context or NULL */
328 ARG_ANYTHING, /* any (initialized) argument is ok */353 ARG_ANYTHING, /* any (initialized) argument is ok */
329 ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */354 ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */
330 ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */355 ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */
331 ARG_PTR_TO_INT, /* pointer to int */356 ARG_PTR_TO_INT, /* pointer to int */
332 ARG_PTR_TO_LONG, /* pointer to long */357 ARG_PTR_TO_LONG, /* pointer to long */
333 ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */358 ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */
334 ARG_PTR_TO_SOCKET_OR_NULL, /* pointer to bpf_sock (fullsock) or NULL */
335 ARG_PTR_TO_BTF_ID, /* pointer to in-kernel struct */359 ARG_PTR_TO_BTF_ID, /* pointer to in-kernel struct */
336 ARG_PTR_TO_ALLOC_MEM, /* pointer to dynamically allocated memory */360 ARG_PTR_TO_ALLOC_MEM, /* pointer to dynamically allocated memory */
337 ARG_PTR_TO_ALLOC_MEM_OR_NULL, /* pointer to dynamically allocated memory or NULL */
338 ARG_CONST_ALLOC_SIZE_OR_ZERO, /* number of allocated bytes requested */361 ARG_CONST_ALLOC_SIZE_OR_ZERO, /* number of allocated bytes requested */
339 ARG_PTR_TO_BTF_ID_SOCK_COMMON, /* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */362 ARG_PTR_TO_BTF_ID_SOCK_COMMON, /* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */
340 ARG_PTR_TO_PERCPU_BTF_ID, /* pointer to in-kernel percpu type */363 ARG_PTR_TO_PERCPU_BTF_ID, /* pointer to in-kernel percpu type */
341 ARG_PTR_TO_FUNC, /* pointer to a bpf program function */364 ARG_PTR_TO_FUNC, /* pointer to a bpf program function */
342 ARG_PTR_TO_STACK_OR_NULL, /* pointer to stack or NULL */365 ARG_PTR_TO_STACK, /* pointer to stack */
343 ARG_PTR_TO_CONST_STR, /* pointer to a null terminated read-only string */366 ARG_PTR_TO_CONST_STR, /* pointer to a null terminated read-only string */
344 ARG_PTR_TO_TIMER, /* pointer to bpf_timer */367 ARG_PTR_TO_TIMER, /* pointer to bpf_timer */
345 __BPF_ARG_TYPE_MAX,368 __BPF_ARG_TYPE_MAX,
369
370 /* Extended arg_types. */
371 ARG_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_MAP_VALUE,
372 ARG_PTR_TO_MEM_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_MEM,
373 ARG_PTR_TO_CTX_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_CTX,
374 ARG_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_SOCKET,
375 ARG_PTR_TO_ALLOC_MEM_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_ALLOC_MEM,
376 ARG_PTR_TO_STACK_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_STACK,
377
378 /* This must be the last entry. Its purpose is to ensure the enum is
379 * wide enough to hold the higher bits reserved for bpf_type_flag.
380 */
381 __BPF_ARG_TYPE_LIMIT = BPF_TYPE_LIMIT,
346};382};
383static_assert(__BPF_ARG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
347 384
348/* type of values returned from helper functions */385/* type of values returned from helper functions */
349enum bpf_return_type {386enum bpf_return_type {
350 RET_INTEGER, /* function returns integer */387 RET_INTEGER, /* function returns integer */
351 RET_VOID, /* function doesn't return anything */388 RET_VOID, /* function doesn't return anything */
352 RET_PTR_TO_MAP_VALUE, /* returns a pointer to map elem value */389 RET_PTR_TO_MAP_VALUE, /* returns a pointer to map elem value */
353 RET_PTR_TO_MAP_VALUE_OR_NULL, /* returns a pointer to map elem value or NULL */390 RET_PTR_TO_SOCKET, /* returns a pointer to a socket */
354 RET_PTR_TO_SOCKET_OR_NULL, /* returns a pointer to a socket or NULL */391 RET_PTR_TO_TCP_SOCK, /* returns a pointer to a tcp_sock */
355 RET_PTR_TO_TCP_SOCK_OR_NULL, /* returns a pointer to a tcp_sock or NULL */392 RET_PTR_TO_SOCK_COMMON, /* returns a pointer to a sock_common */
356 RET_PTR_TO_SOCK_COMMON_OR_NULL, /* returns a pointer to a sock_common or NULL */393 RET_PTR_TO_ALLOC_MEM, /* returns a pointer to dynamically allocated memory */
357 RET_PTR_TO_ALLOC_MEM_OR_NULL, /* returns a pointer to dynamically allocated memory or NULL */
358 RET_PTR_TO_BTF_ID_OR_NULL, /* returns a pointer to a btf_id or NULL */
359 RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL, /* returns a pointer to a valid memory or a btf_id or NULL */
360 RET_PTR_TO_MEM_OR_BTF_ID, /* returns a pointer to a valid memory or a btf_id */394 RET_PTR_TO_MEM_OR_BTF_ID, /* returns a pointer to a valid memory or a btf_id */
361 RET_PTR_TO_BTF_ID, /* returns a pointer to a btf_id */395 RET_PTR_TO_BTF_ID, /* returns a pointer to a btf_id */
396 __BPF_RET_TYPE_MAX,
397
398 /* Extended ret_types. */
399 RET_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_MAP_VALUE,
400 RET_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_SOCKET,
401 RET_PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_TCP_SOCK,
402 RET_PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_SOCK_COMMON,
403 RET_PTR_TO_ALLOC_MEM_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_ALLOC_MEM,
404 RET_PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_BTF_ID,
405
406 /* This must be the last entry. Its purpose is to ensure the enum is
407 * wide enough to hold the higher bits reserved for bpf_type_flag.
408 */
409 __BPF_RET_TYPE_LIMIT = BPF_TYPE_LIMIT,
362};410};
411static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
363 412
364/* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs413/* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs
365 * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL414 * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL
@@ -421,18 +470,15 @@ enum bpf_reg_type { · 55 unmodified lines
421 PTR_TO_CTX, /* reg points to bpf_context */470 PTR_TO_CTX, /* reg points to bpf_context */
422 CONST_PTR_TO_MAP, /* reg points to struct bpf_map */471 CONST_PTR_TO_MAP, /* reg points to struct bpf_map */
423 PTR_TO_MAP_VALUE, /* reg points to map element value */472 PTR_TO_MAP_VALUE, /* reg points to map element value */
424 PTR_TO_MAP_VALUE_OR_NULL,/* points to map elem value or NULL */473 PTR_TO_MAP_KEY, /* reg points to a map element key */
425 PTR_TO_STACK, /* reg == frame_pointer + offset */474 PTR_TO_STACK, /* reg == frame_pointer + offset */
426 PTR_TO_PACKET_META, /* skb->data - meta_len */475 PTR_TO_PACKET_META, /* skb->data - meta_len */
427 PTR_TO_PACKET, /* reg points to skb->data */476 PTR_TO_PACKET, /* reg points to skb->data */
428 PTR_TO_PACKET_END, /* skb->data + headlen */477 PTR_TO_PACKET_END, /* skb->data + headlen */
429 PTR_TO_FLOW_KEYS, /* reg points to bpf_flow_keys */478 PTR_TO_FLOW_KEYS, /* reg points to bpf_flow_keys */
430 PTR_TO_SOCKET, /* reg points to struct bpf_sock */479 PTR_TO_SOCKET, /* reg points to struct bpf_sock */
431 PTR_TO_SOCKET_OR_NULL, /* reg points to struct bpf_sock or NULL */
432 PTR_TO_SOCK_COMMON, /* reg points to sock_common */480 PTR_TO_SOCK_COMMON, /* reg points to sock_common */
433 PTR_TO_SOCK_COMMON_OR_NULL, /* reg points to sock_common or NULL */
434 PTR_TO_TCP_SOCK, /* reg points to struct tcp_sock */481 PTR_TO_TCP_SOCK, /* reg points to struct tcp_sock */
435 PTR_TO_TCP_SOCK_OR_NULL, /* reg points to struct tcp_sock or NULL */
436 PTR_TO_TP_BUFFER, /* reg points to a writable raw tp's buffer */482 PTR_TO_TP_BUFFER, /* reg points to a writable raw tp's buffer */
437 PTR_TO_XDP_SOCK, /* reg points to struct xdp_sock */483 PTR_TO_XDP_SOCK, /* reg points to struct xdp_sock */
438 /* PTR_TO_BTF_ID points to a kernel struct that does not need484 /* PTR_TO_BTF_ID points to a kernel struct that does not need
@@ -450,18 +496,25 @@ enum bpf_reg_type { · 11 unmodified lines
450 * been checked for null. Used primarily to inform the verifier496 * been checked for null. Used primarily to inform the verifier
451 * an explicit null check is required for this struct.497 * an explicit null check is required for this struct.
452 */498 */
453 PTR_TO_BTF_ID_OR_NULL,
454 PTR_TO_MEM, /* reg points to valid memory region */499 PTR_TO_MEM, /* reg points to valid memory region */
455 PTR_TO_MEM_OR_NULL, /* reg points to valid memory region or NULL */500 PTR_TO_BUF, /* reg points to a read/write buffer */
456 PTR_TO_RDONLY_BUF, /* reg points to a readonly buffer */
457 PTR_TO_RDONLY_BUF_OR_NULL, /* reg points to a readonly buffer or NULL */
458 PTR_TO_RDWR_BUF, /* reg points to a read/write buffer */
459 PTR_TO_RDWR_BUF_OR_NULL, /* reg points to a read/write buffer or NULL */
460 PTR_TO_PERCPU_BTF_ID, /* reg points to a percpu kernel variable */501 PTR_TO_PERCPU_BTF_ID, /* reg points to a percpu kernel variable */
461 PTR_TO_FUNC, /* reg points to a bpf program function */502 PTR_TO_FUNC, /* reg points to a bpf program function */
462 PTR_TO_MAP_KEY, /* reg points to a map element key */
463 __BPF_REG_TYPE_MAX,503 __BPF_REG_TYPE_MAX,
504
505 /* Extended reg_types. */
506 PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | PTR_TO_MAP_VALUE,
507 PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCKET,
508 PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCK_COMMON,
509 PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | PTR_TO_TCP_SOCK,
510 PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | PTR_TO_BTF_ID,
511
512 /* This must be the last entry. Its purpose is to ensure the enum is
513 * wide enough to hold the higher bits reserved for bpf_type_flag.
514 */
515 __BPF_REG_TYPE_LIMIT = BPF_TYPE_LIMIT,
464};516};
517static_assert(__BPF_REG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
465 518
466/* The information passed from prog-specific *_is_valid_access519/* The information passed from prog-specific *_is_valid_access
467 * back to the verifier.520 * back to the verifier.
@@ -777,6 +830,7 @@ void bpf_ksym_add(struct bpf_ksym *ksym); · 309 unmodified lines
777void bpf_ksym_del(struct bpf_ksym *ksym);830void bpf_ksym_del(struct bpf_ksym *ksym);
778int bpf_jit_charge_modmem(u32 pages);831int bpf_jit_charge_modmem(u32 pages);
779void bpf_jit_uncharge_modmem(u32 pages);832void bpf_jit_uncharge_modmem(u32 pages);
833bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
780#else834#else
781static inline int bpf_trampoline_link_prog(struct bpf_prog *prog,835static inline int bpf_trampoline_link_prog(struct bpf_prog *prog,
782 struct bpf_trampoline *tr)836 struct bpf_trampoline *tr)
@@ -805,6 +859,10 @@ static inline bool is_bpf_image_address(unsigned long address) · 22 unmodified lines
805{859{
806 return false;860 return false;
807}861}
862static inline bool bpf_prog_has_trampoline(const struct bpf_prog *prog)
863{
864 return false;
865}
808#endif866#endif
809 867
810struct bpf_func_info_aux {868struct bpf_func_info_aux {
@@ -2163,6 +2221,7 @@ extern const struct bpf_func_proto bpf_sk_getsockopt_proto; · 1352 unmodified lines
2163extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;2221extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;
2164extern const struct bpf_func_proto bpf_find_vma_proto;2222extern const struct bpf_func_proto bpf_find_vma_proto;
2165extern const struct bpf_func_proto bpf_loop_proto;2223extern const struct bpf_func_proto bpf_loop_proto;
2224extern const struct bpf_func_proto bpf_strncmp_proto;
2166 2225
2167const struct bpf_func_proto *tracing_prog_func_proto(2226const struct bpf_func_proto *tracing_prog_func_proto(
2168 enum bpf_func_id func_id, const struct bpf_prog *prog);2227 enum bpf_func_id func_id, const struct bpf_prog *prog);
include/linux/bpf_local_storage.h+6−0
@@ -8,6 +8,7 @@ · 7 unmodified lines
8#define _BPF_LOCAL_STORAGE_H8#define _BPF_LOCAL_STORAGE_H
9 9
10#include <linux/bpf.h>10#include <linux/bpf.h>
11#include <linux/filter.h>
11#include <linux/rculist.h>12#include <linux/rculist.h>
12#include <linux/list.h>13#include <linux/list.h>
13#include <linux/hash.h>14#include <linux/hash.h>
@@ -16,6 +17,9 @@ · 2 unmodified lines
16 17
17#define BPF_LOCAL_STORAGE_CACHE_SIZE 1618#define BPF_LOCAL_STORAGE_CACHE_SIZE 16
18 19
20#define bpf_rcu_lock_held() \
21 (rcu_read_lock_held() || rcu_read_lock_trace_held() || \
22 rcu_read_lock_bh_held())
19struct bpf_local_storage_map_bucket {23struct bpf_local_storage_map_bucket {
20 struct hlist_head list;24 struct hlist_head list;
21 raw_spinlock_t lock;25 raw_spinlock_t lock;
@@ -161,4 +165,6 @@ struct bpf_local_storage_data * · 139 unmodified lines
161bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,165bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
162 void *value, u64 map_flags);166 void *value, u64 map_flags);
163 167
168void bpf_local_storage_free_rcu(struct rcu_head *rcu);
169
164#endif /* _BPF_LOCAL_STORAGE_H */170#endif /* _BPF_LOCAL_STORAGE_H */
include/linux/bpf_verifier.h+27−0
@@ -18,6 +18,8 @@ · 17 unmodified lines
18 * that converting umax_value to int cannot overflow.18 * that converting umax_value to int cannot overflow.
19 */19 */
20#define BPF_MAX_VAR_SIZ (1 << 29)20#define BPF_MAX_VAR_SIZ (1 << 29)
21/* size of type_str_buf in bpf_verifier. */
22#define TYPE_STR_BUF_LEN 64
21 23
22/* Liveness marks, used for registers and spilled-regs (in stack slots).24/* Liveness marks, used for registers and spilled-regs (in stack slots).
23 * Read marks propagate upwards until they find a write mark; they record that25 * Read marks propagate upwards until they find a write mark; they record that
@@ -388,6 +390,8 @@ static inline bool bpf_verifier_log_full(const struct bpf_verifier_log *log) · 364 unmodified lines
388#define BPF_LOG_LEVEL (BPF_LOG_LEVEL1 | BPF_LOG_LEVEL2)390#define BPF_LOG_LEVEL (BPF_LOG_LEVEL1 | BPF_LOG_LEVEL2)
389#define BPF_LOG_MASK (BPF_LOG_LEVEL | BPF_LOG_STATS)391#define BPF_LOG_MASK (BPF_LOG_LEVEL | BPF_LOG_STATS)
390#define BPF_LOG_KERNEL (BPF_LOG_MASK + 1) /* kernel internal flag */392#define BPF_LOG_KERNEL (BPF_LOG_MASK + 1) /* kernel internal flag */
393#define BPF_LOG_MIN_ALIGNMENT 8U
394#define BPF_LOG_ALIGNMENT 40U
391 395
392static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log)396static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log)
393{397{
@@ -474,6 +478,16 @@ struct bpf_verifier_env { · 80 unmodified lines
474 /* longest register parentage chain walked for liveness marking */478 /* longest register parentage chain walked for liveness marking */
475 u32 longest_mark_read_walk;479 u32 longest_mark_read_walk;
476 bpfptr_t fd_array;480 bpfptr_t fd_array;
481
482 /* bit mask to keep track of whether a register has been accessed
483 * since the last time the function state was printed
484 */
485 u32 scratched_regs;
486 /* Same as scratched_regs but for stack slots */
487 u64 scratched_stack_slots;
488 u32 prev_log_len, prev_insn_print_len;
489 /* buffer used in reg_type_str() to generate reg_type string */
490 char type_str_buf[TYPE_STR_BUF_LEN];
477};491};
478 492
479__printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log,493__printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log,
@@ -536,5 +550,18 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, · 56 unmodified lines
536 struct bpf_attach_target_info *tgt_info);550 struct bpf_attach_target_info *tgt_info);
537void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab);551void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab);
538 552
553#define BPF_BASE_TYPE_MASK GENMASK(BPF_BASE_TYPE_BITS - 1, 0)
554
555/* extract base type from bpf_{arg, return, reg}_type. */
556static inline u32 base_type(u32 type)
557{
558 return type & BPF_BASE_TYPE_MASK;
559}
560
561/* extract flags from an extended type. See bpf_type_flag in bpf.h. */
562static inline u32 type_flag(u32 type)
563{
564 return type & ~BPF_BASE_TYPE_MASK;
565}
539 566
540#endif /* _LINUX_BPF_VERIFIER_H */567#endif /* _LINUX_BPF_VERIFIER_H */
include/linux/cgroup-defs.h+1−1
@@ -19,7 +19,7 @@ · 18 unmodified lines
19#include <linux/percpu-rwsem.h>19#include <linux/percpu-rwsem.h>
20#include <linux/u64_stats_sync.h>20#include <linux/u64_stats_sync.h>
21#include <linux/workqueue.h>21#include <linux/workqueue.h>
22#include <linux/bpf-cgroup.h>22#include <linux/bpf-cgroup-defs.h>
23#include <linux/psi_types.h>23#include <linux/psi_types.h>
24 24
25#ifdef CONFIG_CGROUPS25#ifdef CONFIG_CGROUPS
include/linux/dsa/loop.h+1−0
@@ -2,6 +2,7 @@ · 1 unmodified line
2#ifndef DSA_LOOP_H2#ifndef DSA_LOOP_H
3#define DSA_LOOP_H3#define DSA_LOOP_H
4 4
5#include <linux/if_vlan.h>
5#include <linux/types.h>6#include <linux/types.h>
6#include <linux/ethtool.h>7#include <linux/ethtool.h>
7#include <net/dsa.h>8#include <net/dsa.h>
include/linux/filter.h+1−1
@@ -1027,7 +1027,7 @@ void xdp_do_flush(void); · 1026 unmodified lines
1027 */1027 */
1028#define xdp_do_flush_map xdp_do_flush1028#define xdp_do_flush_map xdp_do_flush
1029 1029
1030void bpf_warn_invalid_xdp_action(u32 act);1030void bpf_warn_invalid_xdp_action(struct net_device *dev, struct bpf_prog *prog, u32 act);
1031 1031
1032#ifdef CONFIG_INET1032#ifdef CONFIG_INET
1033struct sock *bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk,1033struct sock *bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk,
include/linux/perf_event.h+1−0
@@ -611,6 +611,7 @@ struct swevent_hlist { · 610 unmodified lines
611#define PERF_ATTACH_SCHED_CB 0x20611#define PERF_ATTACH_SCHED_CB 0x20
612#define PERF_ATTACH_CHILD 0x40612#define PERF_ATTACH_CHILD 0x40
613 613
614struct bpf_prog;
614struct perf_cgroup;615struct perf_cgroup;
615struct perf_buffer;616struct perf_buffer;
616 617
include/net/ip6_fib.h+1−0
@@ -20,6 +20,7 @@ · 19 unmodified lines
20#include <net/inetpeer.h>20#include <net/inetpeer.h>
21#include <net/fib_notifier.h>21#include <net/fib_notifier.h>
22#include <linux/indirect_call_wrapper.h>22#include <linux/indirect_call_wrapper.h>
23#include <uapi/linux/bpf.h>
23 24
24#ifdef CONFIG_IPV6_MULTIPLE_TABLES25#ifdef CONFIG_IPV6_MULTIPLE_TABLES
25#define FIB6_TABLE_HASHSZ 25626#define FIB6_TABLE_HASHSZ 256
include/net/ipv6.h+2−0
@@ -21,6 +21,8 @@ · 20 unmodified lines
21#include <net/snmp.h>21#include <net/snmp.h>
22#include <net/netns/hash.h>22#include <net/netns/hash.h>
23 23
24struct ip_tunnel_info;
25
24#define SIN6_LEN_RFC2133 2426#define SIN6_LEN_RFC2133 24
25 27
26#define IPV6_MAXPLEN 6553528#define IPV6_MAXPLEN 65535
include/net/netns/bpf.h+8−1
@@ -6,11 +6,18 @@ · 5 unmodified lines
6#ifndef __NETNS_BPF_H__6#ifndef __NETNS_BPF_H__
7#define __NETNS_BPF_H__7#define __NETNS_BPF_H__
8 8
9#include <linux/bpf-netns.h>9#include <linux/list.h>
10 10
11struct bpf_prog;11struct bpf_prog;
12struct bpf_prog_array;12struct bpf_prog_array;
13 13
14enum netns_bpf_attach_type {
15 NETNS_BPF_INVALID = -1,
16 NETNS_BPF_FLOW_DISSECTOR = 0,
17 NETNS_BPF_SK_LOOKUP,
18 MAX_NETNS_BPF_ATTACH_TYPE
19};
20
14struct netns_bpf {21struct netns_bpf {
15 /* Array of programs to run compiled from progs or links */22 /* Array of programs to run compiled from progs or links */
16 struct bpf_prog_array __rcu *run_array[MAX_NETNS_BPF_ATTACH_TYPE];23 struct bpf_prog_array __rcu *run_array[MAX_NETNS_BPF_ATTACH_TYPE];
include/net/route.h+1−0
@@ -43,6 +43,7 @@ · 42 unmodified lines
43#define RT_CONN_FLAGS(sk) (RT_TOS(inet_sk(sk)->tos) | sock_flag(sk, SOCK_LOCALROUTE))43#define RT_CONN_FLAGS(sk) (RT_TOS(inet_sk(sk)->tos) | sock_flag(sk, SOCK_LOCALROUTE))
44#define RT_CONN_FLAGS_TOS(sk,tos) (RT_TOS(tos) | sock_flag(sk, SOCK_LOCALROUTE))44#define RT_CONN_FLAGS_TOS(sk,tos) (RT_TOS(tos) | sock_flag(sk, SOCK_LOCALROUTE))
45 45
46struct ip_tunnel_info;
46struct fib_nh;47struct fib_nh;
47struct fib_info;48struct fib_info;
48struct uncached_list;49struct uncached_list;
include/net/sock.h+1−1
@@ -56,7 +56,6 @@ · 55 unmodified lines
56#include <linux/wait.h>56#include <linux/wait.h>
57#include <linux/cgroup-defs.h>57#include <linux/cgroup-defs.h>
58#include <linux/rbtree.h>58#include <linux/rbtree.h>
59#include <linux/filter.h>
60#include <linux/rculist_nulls.h>59#include <linux/rculist_nulls.h>
61#include <linux/poll.h>60#include <linux/poll.h>
62#include <linux/sockptr.h>61#include <linux/sockptr.h>
@@ -249,6 +248,7 @@ struct sock_common { · 186 unmodified lines
249};248};
250 249
251struct bpf_local_storage;250struct bpf_local_storage;
251struct sk_filter;
252 252
253/**253/**
254 * struct sock - network layer representation of sockets254 * struct sock - network layer representation of sockets
include/net/xdp_priv.h+0−1
@@ -10,7 +10,6 @@ struct xdp_mem_allocator { · 9 unmodified lines
10 union {10 union {
11 void *allocator;11 void *allocator;
12 struct page_pool *page_pool;12 struct page_pool *page_pool;
13 struct zero_copy_allocator *zc_alloc;
14 };13 };
15 struct rhash_head node;14 struct rhash_head node;
16 struct rcu_head rcu;15 struct rcu_head rcu;
include/net/xdp_sock.h+1−0
@@ -6,6 +6,7 @@ · 5 unmodified lines
6#ifndef _LINUX_XDP_SOCK_H6#ifndef _LINUX_XDP_SOCK_H
7#define _LINUX_XDP_SOCK_H7#define _LINUX_XDP_SOCK_H
8 8
9#include <linux/bpf.h>
9#include <linux/workqueue.h>10#include <linux/workqueue.h>
10#include <linux/if_xdp.h>11#include <linux/if_xdp.h>
11#include <linux/mutex.h>12#include <linux/mutex.h>
include/uapi/linux/bpf.h+39−0
@@ -4983,6 +4983,41 @@ union bpf_attr { · 4982 unmodified lines
4983 * Return4983 * Return
4984 * The number of loops performed, **-EINVAL** for invalid **flags**,4984 * The number of loops performed, **-EINVAL** for invalid **flags**,
4985 * **-E2BIG** if **nr_loops** exceeds the maximum number of loops.4985 * **-E2BIG** if **nr_loops** exceeds the maximum number of loops.
4986 *
4987 * long bpf_strncmp(const char *s1, u32 s1_sz, const char *s2)
4988 * Description
4989 * Do strncmp() between **s1** and **s2**. **s1** doesn't need
4990 * to be null-terminated and **s1_sz** is the maximum storage
4991 * size of **s1**. **s2** must be a read-only string.
4992 * Return
4993 * An integer less than, equal to, or greater than zero
4994 * if the first **s1_sz** bytes of **s1** is found to be
4995 * less than, to match, or be greater than **s2**.
4996 *
4997 * long bpf_get_func_arg(void *ctx, u32 n, u64 *value)
4998 * Description
4999 * Get **n**-th argument (zero based) of the traced function (for tracing programs)
5000 * returned in **value**.
5001 *
5002 * Return
5003 * 0 on success.
5004 * **-EINVAL** if n >= arguments count of traced function.
5005 *
5006 * long bpf_get_func_ret(void *ctx, u64 *value)
5007 * Description
5008 * Get return value of the traced function (for tracing programs)
5009 * in **value**.
5010 *
5011 * Return
5012 * 0 on success.
5013 * **-EOPNOTSUPP** for tracing programs other than BPF_TRACE_FEXIT or BPF_MODIFY_RETURN.
5014 *
5015 * long bpf_get_func_arg_cnt(void *ctx)
5016 * Description
5017 * Get number of arguments of the traced function (for tracing programs).
5018 *
5019 * Return
5020 * The number of arguments of the traced function.
4986 */5021 */
4987#define __BPF_FUNC_MAPPER(FN) \5022#define __BPF_FUNC_MAPPER(FN) \
4988 FN(unspec), \5023 FN(unspec), \
@@ -5167,6 +5202,10 @@ union bpf_attr { · 178 unmodified lines
5167 FN(kallsyms_lookup_name), \5202 FN(kallsyms_lookup_name), \
5168 FN(find_vma), \5203 FN(find_vma), \
5169 FN(loop), \5204 FN(loop), \
5205 FN(strncmp), \
5206 FN(get_func_arg), \
5207 FN(get_func_ret), \
5208 FN(get_func_arg_cnt), \
5170 /* */5209 /* */
5171 5210
5172/* integer value in 'imm' field of BPF_CALL instruction selects which helper5211/* integer value in 'imm' field of BPF_CALL instruction selects which helper