History

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Daniel Borkmann says: ==================== pull-request: bpf-next 2021-02-16 The following pull-request contains BPF updates for your *net-next* tree. There's a small merge conflict between 7eeba1706eba ("tcp: Add receive timestamp support for receive zerocopy.") from net-next tree and 9cacf81f8161 ("bpf: Remove extra lock_sock for TCP_ZEROCOPY_RECEIVE") from bpf-next tree. Resolve as follows: [...] lock_sock(sk); err = tcp_zerocopy_receive(sk, &zc, &tss); err = BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sk, level, optname, &zc, &len, err); release_sock(sk); [...] We've added 116 non-merge commits during the last 27 day(s) which contain a total of 156 files changed, 5662 insertions(+), 1489 deletions(-). The main changes are: 1) Adds support of pointers to types with known size among global function args to overcome the limit on max # of allowed args, from Dmitrii Banshchikov. 2) Add bpf_iter for task_vma which can be used to generate information similar to /proc/pid/maps, from Song Liu. 3) Enable bpf_{g,s}etsockopt() from all sock_addr related program hooks. Allow rewriting bind user ports from BPF side below the ip_unprivileged_port_start range, both from Stanislav Fomichev. 4) Prevent recursion on fentry/fexit & sleepable programs and allow map-in-map as well as per-cpu maps for the latter, from Alexei Starovoitov. 5) Add selftest script to run BPF CI locally. Also enable BPF ringbuffer for sleepable programs, both from KP Singh. 6) Extend verifier to enable variable offset read/write access to the BPF program stack, from Andrei Matei. 7) Improve tc & XDP MTU handling and add a new bpf_check_mtu() helper to query device MTU from programs, from Jesper Dangaard Brouer. 8) Allow bpf_get_socket_cookie() helper also be called from [sleepable] BPF tracing programs, from Florent Revest. 9) Extend x86 JIT to pad JMPs with NOPs for helping image to converge when otherwise too many passes are required, from Gary Lin. 10) Verifier fixes on atomics with BPF_FETCH as well as function-by-function verification both related to zero-extension handling, from Ilya Leoshkevich. 11) Better kernel build integration of resolve_btfids tool, from Jiri Olsa. 12) Batch of AF_XDP selftest cleanups and small performance improvement for libbpf's xsk map redirect for newer kernels, from Björn Töpel. 13) Follow-up BPF doc and verifier improvements around atomics with BPF_FETCH, from Brendan Jackman. 14) Permit zero-sized data sections e.g. if ELF .rodata section contains read-only data from local variables, from Yonghong Song. 15) veth driver skb bulk-allocation for ndo_xdp_xmit, from Lorenzo Bianconi. ====================

David S. Miller committed 5 years ago

Commit b8af4172 parents 9ec5eea 45159b2156 files · +5662 −1489Browse files

Showing the changes to kernel (15 of 156 files). Show all

Unified · Split

15 files
  1. kernel/bpf/bpf_iter.c +1 −1
  2. kernel/bpf/bpf_lru_list.c +4 −3
  3. kernel/bpf/btf.c +55 −21
  4. kernel/bpf/cgroup.c +103 −17
  5. kernel/bpf/core.c +14 −4
  6. kernel/bpf/cpumap.c +2 −44
  7. kernel/bpf/devmap.c +1 −3
  8. kernel/bpf/disasm.c +1 −1
  9. kernel/bpf/hashtab.c +2 −2
  10. kernel/bpf/helpers.c +4 −8
  11. kernel/bpf/syscall.c +11 −5
  12. kernel/bpf/task_iter.c +266 −1
  13. kernel/bpf/trampoline.c +61 −16
  14. kernel/bpf/verifier.c +662 −215
  15. kernel/trace/bpf_trace.c +6 −0
kernel/bpf/bpf_iter.c+1−1
@@ -287,7 +287,7 @@ int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info) · 286 unmodified lines
287287 {
288288 struct bpf_iter_target_info *tinfo;
289289
290− tinfo = kmalloc(sizeof(*tinfo), GFP_KERNEL);
290+ tinfo = kzalloc(sizeof(*tinfo), GFP_KERNEL);
291291 if (!tinfo)
292292 return -ENOMEM;
293293
kernel/bpf/bpf_lru_list.c+4−3
@@ -502,13 +502,14 @@ struct bpf_lru_node *bpf_lru_pop_free(struct bpf_lru *lru, u32 hash) · 501 unmodified lines
502502 static void bpf_common_lru_push_free(struct bpf_lru *lru,
503503 struct bpf_lru_node *node)
504504 {
505+ u8 node_type = READ_ONCE(node->type);
505506 unsigned long flags;
506507
507− if (WARN_ON_ONCE(node->type == BPF_LRU_LIST_T_FREE) ||
508− WARN_ON_ONCE(node->type == BPF_LRU_LOCAL_LIST_T_FREE))
508+ if (WARN_ON_ONCE(node_type == BPF_LRU_LIST_T_FREE) ||
509+ WARN_ON_ONCE(node_type == BPF_LRU_LOCAL_LIST_T_FREE))
509510 return;
510511
511− if (node->type == BPF_LRU_LOCAL_LIST_T_PENDING) {
512+ if (node_type == BPF_LRU_LOCAL_LIST_T_PENDING) {
512513 struct bpf_lru_locallist *loc_l;
513514
514515 loc_l = per_cpu_ptr(lru->common_lru.local_list, node->cpu);
kernel/bpf/btf.c+55−21
@@ -3540,11 +3540,6 @@ static s32 btf_datasec_check_meta(struct btf_verifier_env *env, · 3539 unmodified lines
35403540 return -EINVAL;
35413541 }
35423542
3543− if (!btf_type_vlen(t)) {
3544− btf_verifier_log_type(env, t, "vlen == 0");
3545− return -EINVAL;
3546− }
3547−
35483543 if (!t->size) {
35493544 btf_verifier_log_type(env, t, "size == 0");
35503545 return -EINVAL;
@@ -5296,15 +5291,16 @@ int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *pr · 1745 unmodified lines
52965291 * Only PTR_TO_CTX and SCALAR_VALUE states are recognized.
52975292 */
52985293 int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
5299− struct bpf_reg_state *reg)
5294+ struct bpf_reg_state *regs)
53005295 {
53015296 struct bpf_verifier_log *log = &env->log;
53025297 struct bpf_prog *prog = env->prog;
53035298 struct btf *btf = prog->aux->btf;
53045299 const struct btf_param *args;
5305− const struct btf_type *t;
5306− u32 i, nargs, btf_id;
5300+ const struct btf_type *t, *ref_t;
5301+ u32 i, nargs, btf_id, type_size;
53075302 const char *tname;
5303+ bool is_global;
53085304
53095305 if (!prog->aux->func_info)
53105306 return -EINVAL;
@@ -5338,38 +5334,57 @@ int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, · 27 unmodified lines
53385334 bpf_log(log, "Function %s has %d > 5 args\n", tname, nargs);
53395335 goto out;
53405336 }
5337+
5338+ is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
53415339 /* check that BTF function arguments match actual types that the
53425340 * verifier sees.
53435341 */
53445342 for (i = 0; i < nargs; i++) {
5343+ struct bpf_reg_state *reg = &regs[i + 1];
5344+
53455345 t = btf_type_by_id(btf, args[i].type);
53465346 while (btf_type_is_modifier(t))
53475347 t = btf_type_by_id(btf, t->type);
53485348 if (btf_type_is_int(t) || btf_type_is_enum(t)) {
5349− if (reg[i + 1].type == SCALAR_VALUE)
5349+ if (reg->type == SCALAR_VALUE)
53505350 continue;
53515351 bpf_log(log, "R%d is not a scalar\n", i + 1);
53525352 goto out;
53535353 }
53545354 if (btf_type_is_ptr(t)) {
5355− if (reg[i + 1].type == SCALAR_VALUE) {
5356− bpf_log(log, "R%d is not a pointer\n", i + 1);
5357− goto out;
5358− }
53595355 /* If function expects ctx type in BTF check that caller
53605356 * is passing PTR_TO_CTX.
53615357 */
53625358 if (btf_get_prog_ctx_type(log, btf, t, prog->type, i)) {
5363− if (reg[i + 1].type != PTR_TO_CTX) {
5359+ if (reg->type != PTR_TO_CTX) {
53645360 bpf_log(log,
53655361 "arg#%d expected pointer to ctx, but got %s\n",
53665362 i, btf_kind_str[BTF_INFO_KIND(t->info)]);
53675363 goto out;
53685364 }
5369− if (check_ctx_reg(env, &reg[i + 1], i + 1))
5365+ if (check_ctx_reg(env, reg, i + 1))
53705366 goto out;
53715367 continue;
53725368 }
5369+
5370+ if (!is_global)
5371+ goto out;
5372+
5373+ t = btf_type_skip_modifiers(btf, t->type, NULL);
5374+
5375+ ref_t = btf_resolve_size(btf, t, &type_size);
5376+ if (IS_ERR(ref_t)) {
5377+ bpf_log(log,
5378+ "arg#%d reference type('%s %s') size cannot be determined: %ld\n",
5379+ i, btf_type_str(t), btf_name_by_offset(btf, t->name_off),
5380+ PTR_ERR(ref_t));
5381+ goto out;
5382+ }
5383+
5384+ if (check_mem_reg(env, reg, i + 1, type_size))
5385+ goto out;
5386+
5387+ continue;
53735388 }
53745389 bpf_log(log, "Unrecognized arg#%d type %s\n",
53755390 i, btf_kind_str[BTF_INFO_KIND(t->info)]);
@@ -5393,14 +5408,14 @@ out: · 17 unmodified lines
53935408 * (either PTR_TO_CTX or SCALAR_VALUE).
53945409 */
53955410 int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
5396− struct bpf_reg_state *reg)
5411+ struct bpf_reg_state *regs)
53975412 {
53985413 struct bpf_verifier_log *log = &env->log;
53995414 struct bpf_prog *prog = env->prog;
54005415 enum bpf_prog_type prog_type = prog->type;
54015416 struct btf *btf = prog->aux->btf;
54025417 const struct btf_param *args;
5403− const struct btf_type *t;
5418+ const struct btf_type *t, *ref_t;
54045419 u32 i, nargs, btf_id;
54055420 const char *tname;
54065421
@@ -5464,16 +5479,35 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, · 57 unmodified lines
54645479 * Only PTR_TO_CTX and SCALAR are supported atm.
54655480 */
54665481 for (i = 0; i < nargs; i++) {
5482+ struct bpf_reg_state *reg = &regs[i + 1];
5483+
54675484 t = btf_type_by_id(btf, args[i].type);
54685485 while (btf_type_is_modifier(t))
54695486 t = btf_type_by_id(btf, t->type);
54705487 if (btf_type_is_int(t) || btf_type_is_enum(t)) {
5471− reg[i + 1].type = SCALAR_VALUE;
5488+ reg->type = SCALAR_VALUE;
54725489 continue;
54735490 }
5474− if (btf_type_is_ptr(t) &&
5475− btf_get_prog_ctx_type(log, btf, t, prog_type, i)) {
5476− reg[i + 1].type = PTR_TO_CTX;
5491+ if (btf_type_is_ptr(t)) {
5492+ if (btf_get_prog_ctx_type(log, btf, t, prog_type, i)) {
5493+ reg->type = PTR_TO_CTX;
5494+ continue;
5495+ }
5496+
5497+ t = btf_type_skip_modifiers(btf, t->type, NULL);
5498+
5499+ ref_t = btf_resolve_size(btf, t, &reg->mem_size);
5500+ if (IS_ERR(ref_t)) {
5501+ bpf_log(log,
5502+ "arg#%d reference type('%s %s') size cannot be determined: %ld\n",
5503+ i, btf_type_str(t), btf_name_by_offset(btf, t->name_off),
5504+ PTR_ERR(ref_t));
5505+ return -EINVAL;
5506+ }
5507+
5508+ reg->type = PTR_TO_MEM_OR_NULL;
5509+ reg->id = ++env->id_gen;
5510+
54775511 continue;
54785512 }
54795513 bpf_log(log, "Arg#%d type %s in %s() is not supported yet.\n",
kernel/bpf/cgroup.c+103−17
@@ -19,7 +19,7 @@ · 18 unmodified lines
1919
2020 #include "../cgroup/cgroup-internal.h"
2121
22−DEFINE_STATIC_KEY_FALSE(cgroup_bpf_enabled_key);
22+DEFINE_STATIC_KEY_ARRAY_FALSE(cgroup_bpf_enabled_key, MAX_BPF_ATTACH_TYPE);
2323 EXPORT_SYMBOL(cgroup_bpf_enabled_key);
2424
2525 void cgroup_bpf_offline(struct cgroup *cgrp)
@@ -128,7 +128,7 @@ static void cgroup_bpf_release(struct work_struct *work) · 102 unmodified lines
128128 if (pl->link)
129129 bpf_cgroup_link_auto_detach(pl->link);
130130 kfree(pl);
131− static_branch_dec(&cgroup_bpf_enabled_key);
131+ static_branch_dec(&cgroup_bpf_enabled_key[type]);
132132 }
133133 old_array = rcu_dereference_protected(
134134 cgrp->bpf.effective[type],
@@ -499,7 +499,7 @@ int __cgroup_bpf_attach(struct cgroup *cgrp, · 364 unmodified lines
499499 if (old_prog)
500500 bpf_prog_put(old_prog);
501501 else
502− static_branch_inc(&cgroup_bpf_enabled_key);
502+ static_branch_inc(&cgroup_bpf_enabled_key[type]);
503503 bpf_cgroup_storages_link(new_storage, cgrp, type);
504504 return 0;
505505
@@ -698,7 +698,7 @@ int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, · 192 unmodified lines
698698 cgrp->bpf.flags[type] = 0;
699699 if (old_prog)
700700 bpf_prog_put(old_prog);
701− static_branch_dec(&cgroup_bpf_enabled_key);
701+ static_branch_dec(&cgroup_bpf_enabled_key[type]);
702702 return 0;
703703
704704 cleanup:
@@ -1055,6 +1055,8 @@ EXPORT_SYMBOL(__cgroup_bpf_run_filter_sk); · 350 unmodified lines
10551055 * @uaddr: sockaddr struct provided by user
10561056 * @type: The type of program to be exectuted
10571057 * @t_ctx: Pointer to attach type specific context
1058+ * @flags: Pointer to u32 which contains higher bits of BPF program
1059+ * return value (OR'ed together).
10581060 *
10591061 * socket is expected to be of type INET or INET6.
10601062 *
@@ -1064,7 +1066,8 @@ EXPORT_SYMBOL(__cgroup_bpf_run_filter_sk); · 3 unmodified lines
10641066 int __cgroup_bpf_run_filter_sock_addr(struct sock *sk,
10651067 struct sockaddr *uaddr,
10661068 enum bpf_attach_type type,
1067− void *t_ctx)
1069+ void *t_ctx,
1070+ u32 *flags)
10681071 {
10691072 struct bpf_sock_addr_kern ctx = {
10701073 .sk = sk,
@@ -1087,7 +1090,8 @@ int __cgroup_bpf_run_filter_sock_addr(struct sock *sk, · 16 unmodified lines
10871090 }
10881091
10891092 cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
1090− ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[type], &ctx, BPF_PROG_RUN);
1093+ ret = BPF_PROG_RUN_ARRAY_FLAGS(cgrp->bpf.effective[type], &ctx,
1094+ BPF_PROG_RUN, flags);
10911095
10921096 return ret == 1 ? 0 : -EPERM;
10931097 }
@@ -1298,7 +1302,8 @@ static bool __cgroup_bpf_prog_array_is_empty(struct cgroup *cgrp, · 204 unmodified lines
12981302 return empty;
12991303 }
13001304
1301−static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen)
1305+static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen,
1306+ struct bpf_sockopt_buf *buf)
13021307 {
13031308 if (unlikely(max_optlen < 0))
13041309 return -EINVAL;
@@ -1310,6 +1315,15 @@ static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen) · 5 unmodified lines
13101315 max_optlen = PAGE_SIZE;
13111316 }
13121317
1318+ if (max_optlen <= sizeof(buf->data)) {
1319+ /* When the optval fits into BPF_SOCKOPT_KERN_BUF_SIZE
1320+ * bytes avoid the cost of kzalloc.
1321+ */
1322+ ctx->optval = buf->data;
1323+ ctx->optval_end = ctx->optval + max_optlen;
1324+ return max_optlen;
1325+ }
1326+
13131327 ctx->optval = kzalloc(max_optlen, GFP_USER);
13141328 if (!ctx->optval)
13151329 return -ENOMEM;
@@ -1319,16 +1333,26 @@ static int sockopt_alloc_buf(struct bpf_sockopt_kern *ctx, int max_optlen) · 3 unmodified lines
13191333 return max_optlen;
13201334 }
13211335
1322−static void sockopt_free_buf(struct bpf_sockopt_kern *ctx)
1336+static void sockopt_free_buf(struct bpf_sockopt_kern *ctx,
1337+ struct bpf_sockopt_buf *buf)
13231338 {
1339+ if (ctx->optval == buf->data)
1340+ return;
13241341 kfree(ctx->optval);
13251342 }
13261343
1344+static bool sockopt_buf_allocated(struct bpf_sockopt_kern *ctx,
1345+ struct bpf_sockopt_buf *buf)
1346+{
1347+ return ctx->optval != buf->data;
1348+}
1349+
13271350 int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level,
13281351 int *optname, char __user *optval,
13291352 int *optlen, char **kernel_optval)
13301353 {
13311354 struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
1355+ struct bpf_sockopt_buf buf = {};
13321356 struct bpf_sockopt_kern ctx = {
13331357 .sk = sk,
13341358 .level = *level,
@@ -1340,8 +1364,7 @@ int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level, · 5 unmodified lines
13401364 * attached to the hook so we don't waste time allocating
13411365 * memory and locking the socket.
13421366 */
1343− if (!cgroup_bpf_enabled ||
1344− __cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_SETSOCKOPT))
1367+ if (__cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_SETSOCKOPT))
13451368 return 0;
13461369
13471370 /* Allocate a bit more than the initial user buffer for
@@ -1350,7 +1373,7 @@ int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level, · 2 unmodified lines
13501373 */
13511374 max_optlen = max_t(int, 16, *optlen);
13521375
1353− max_optlen = sockopt_alloc_buf(&ctx, max_optlen);
1376+ max_optlen = sockopt_alloc_buf(&ctx, max_optlen, &buf);
13541377 if (max_optlen < 0)
13551378 return max_optlen;
13561379
@@ -1390,14 +1413,31 @@ int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level, · 33 unmodified lines
13901413 */
13911414 if (ctx.optlen != 0) {
13921415 *optlen = ctx.optlen;
1393− *kernel_optval = ctx.optval;
1416+ /* We've used bpf_sockopt_kern->buf as an intermediary
1417+ * storage, but the BPF program indicates that we need
1418+ * to pass this data to the kernel setsockopt handler.
1419+ * No way to export on-stack buf, have to allocate a
1420+ * new buffer.
1421+ */
1422+ if (!sockopt_buf_allocated(&ctx, &buf)) {
1423+ void *p = kmalloc(ctx.optlen, GFP_USER);
1424+
1425+ if (!p) {
1426+ ret = -ENOMEM;
1427+ goto out;
1428+ }
1429+ memcpy(p, ctx.optval, ctx.optlen);
1430+ *kernel_optval = p;
1431+ } else {
1432+ *kernel_optval = ctx.optval;
1433+ }
13941434 /* export and don't free sockopt buf */
13951435 return 0;
13961436 }
13971437 }
13981438
13991439 out:
1400− sockopt_free_buf(&ctx);
1440+ sockopt_free_buf(&ctx, &buf);
14011441 return ret;
14021442 }
14031443
@@ -1407,6 +1447,7 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, · 3 unmodified lines
14071447 int retval)
14081448 {
14091449 struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
1450+ struct bpf_sockopt_buf buf = {};
14101451 struct bpf_sockopt_kern ctx = {
14111452 .sk = sk,
14121453 .level = level,
@@ -1419,13 +1460,12 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, · 6 unmodified lines
14191460 * attached to the hook so we don't waste time allocating
14201461 * memory and locking the socket.
14211462 */
1422− if (!cgroup_bpf_enabled ||
1423− __cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_GETSOCKOPT))
1463+ if (__cgroup_bpf_prog_array_is_empty(cgrp, BPF_CGROUP_GETSOCKOPT))
14241464 return retval;
14251465
14261466 ctx.optlen = max_optlen;
14271467
1428− max_optlen = sockopt_alloc_buf(&ctx, max_optlen);
1468+ max_optlen = sockopt_alloc_buf(&ctx, max_optlen, &buf);
14291469 if (max_optlen < 0)
14301470 return max_optlen;
14311471
@@ -1488,9 +1528,55 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, · 56 unmodified lines
14881528 ret = ctx.retval;
14891529
14901530 out:
1491− sockopt_free_buf(&ctx);
1531+ sockopt_free_buf(&ctx, &buf);
14921532 return ret;
14931533 }
1534+
1535+int __cgroup_bpf_run_filter_getsockopt_kern(struct sock *sk, int level,
1536+ int optname, void *optval,
1537+ int *optlen, int retval)
1538+{
1539+ struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
1540+ struct bpf_sockopt_kern ctx = {
1541+ .sk = sk,
1542+ .level = level,
1543+ .optname = optname,
1544+ .retval = retval,
1545+ .optlen = *optlen,
1546+ .optval = optval,
1547+ .optval_end = optval + *optlen,
1548+ };
1549+ int ret;
1550+
1551+ /* Note that __cgroup_bpf_run_filter_getsockopt doesn't copy
1552+ * user data back into BPF buffer when reval != 0. This is
1553+ * done as an optimization to avoid extra copy, assuming
1554+ * kernel won't populate the data in case of an error.
1555+ * Here we always pass the data and memset() should
1556+ * be called if that data shouldn't be "exported".
1557+ */
1558+
1559+ ret = BPF_PROG_RUN_ARRAY(cgrp->bpf.effective[BPF_CGROUP_GETSOCKOPT],
1560+ &ctx, BPF_PROG_RUN);
1561+ if (!ret)
1562+ return -EPERM;
1563+
1564+ if (ctx.optlen > *optlen)
1565+ return -EFAULT;
1566+
1567+ /* BPF programs only allowed to set retval to 0, not some
1568+ * arbitrary value.
1569+ */
1570+ if (ctx.retval != 0 && ctx.retval != retval)
1571+ return -EFAULT;
1572+
1573+ /* BPF programs can shrink the buffer, export the modifications.
1574+ */
1575+ if (ctx.optlen != 0)
1576+ *optlen = ctx.optlen;
1577+
1578+ return ctx.retval;
1579+}
14941580 #endif
14951581
14961582 static ssize_t sysctl_cpy_dir(const struct ctl_dir *dir, char **bufp,
kernel/bpf/core.c+14−4
@@ -91,6 +91,12 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag · 90 unmodified lines
9191 vfree(fp);
9292 return NULL;
9393 }
94+ fp->active = alloc_percpu_gfp(int, GFP_KERNEL_ACCOUNT | gfp_extra_flags);
95+ if (!fp->active) {
96+ vfree(fp);
97+ kfree(aux);
98+ return NULL;
99+ }
94100
95101 fp->pages = size / PAGE_SIZE;
96102 fp->aux = aux;
@@ -114,8 +120,9 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) · 17 unmodified lines
114120 if (!prog)
115121 return NULL;
116122
117− prog->aux->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags);
118− if (!prog->aux->stats) {
123+ prog->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags);
124+ if (!prog->stats) {
125+ free_percpu(prog->active);
119126 kfree(prog->aux);
120127 vfree(prog);
121128 return NULL;
@@ -124,7 +131,7 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) · 2 unmodified lines
124131 for_each_possible_cpu(cpu) {
125132 struct bpf_prog_stats *pstats;
126133
127− pstats = per_cpu_ptr(prog->aux->stats, cpu);
134+ pstats = per_cpu_ptr(prog->stats, cpu);
128135 u64_stats_init(&pstats->syncp);
129136 }
130137 return prog;
@@ -238,6 +245,8 @@ struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, · 107 unmodified lines
238245 * reallocated structure.
239246 */
240247 fp_old->aux = NULL;
248+ fp_old->stats = NULL;
249+ fp_old->active = NULL;
241250 __bpf_prog_free(fp_old);
242251 }
243252
@@ -249,10 +258,11 @@ void __bpf_prog_free(struct bpf_prog *fp) · 5 unmodified lines
249258 if (fp->aux) {
250259 mutex_destroy(&fp->aux->used_maps_mutex);
251260 mutex_destroy(&fp->aux->dst_mutex);
252− free_percpu(fp->aux->stats);
253261 kfree(fp->aux->poke_tab);
254262 kfree(fp->aux);
255263 }
264+ free_percpu(fp->stats);
265+ free_percpu(fp->active);
256266 vfree(fp);
257267 }
258268
kernel/bpf/cpumap.c+2−44
@@ -141,49 +141,6 @@ static void cpu_map_kthread_stop(struct work_struct *work) · 140 unmodified lines
141141 kthread_stop(rcpu->kthread);
142142 }
143143
144−static struct sk_buff *cpu_map_build_skb(struct xdp_frame *xdpf,
145− struct sk_buff *skb)
146−{
147− unsigned int hard_start_headroom;
148− unsigned int frame_size;
149− void *pkt_data_start;
150−
151− /* Part of headroom was reserved to xdpf */
152− hard_start_headroom = sizeof(struct xdp_frame) + xdpf->headroom;
153−
154− /* Memory size backing xdp_frame data already have reserved
155− * room for build_skb to place skb_shared_info in tailroom.
156− */
157− frame_size = xdpf->frame_sz;
158−
159− pkt_data_start = xdpf->data - hard_start_headroom;
160− skb = build_skb_around(skb, pkt_data_start, frame_size);
161− if (unlikely(!skb))
162− return NULL;
163−
164− skb_reserve(skb, hard_start_headroom);
165− __skb_put(skb, xdpf->len);
166− if (xdpf->metasize)
167− skb_metadata_set(skb, xdpf->metasize);
168−
169− /* Essential SKB info: protocol and skb->dev */
170− skb->protocol = eth_type_trans(skb, xdpf->dev_rx);
171−
172− /* Optional SKB info, currently missing:
173− * - HW checksum info (skb->ip_summed)
174− * - HW RX hash (skb_set_hash)
175− * - RX ring dev queue index (skb_record_rx_queue)
176− */
177−
178− /* Until page_pool get SKB return path, release DMA here */
179− xdp_release_frame(xdpf);
180−
181− /* Allow SKB to reuse area used by xdp_frame */
182− xdp_scrub_frame(xdpf);
183−
184− return skb;
185−}
186−
187144 static void __cpu_map_ring_cleanup(struct ptr_ring *ring)
188145 {
189146 /* The tear-down procedure should have made sure that queue is
@@ -350,7 +307,8 @@ static int cpu_map_kthread_run(void *data) · 160 unmodified lines
350307 struct sk_buff *skb = skbs[i];
351308 int ret;
352309
353− skb = cpu_map_build_skb(xdpf, skb);
310+ skb = __xdp_build_skb_from_frame(xdpf, skb,
311+ xdpf->dev_rx);
354312 if (!skb) {
355313 xdp_return_frame(xdpf);
356314 continue;
kernel/bpf/devmap.c+1−3
@@ -802,9 +802,7 @@ static int dev_map_notification(struct notifier_block *notifier, · 801 unmodified lines
802802 break;
803803
804804 /* will be freed in free_netdev() */
805− netdev->xdp_bulkq =
806− __alloc_percpu_gfp(sizeof(struct xdp_dev_bulk_queue),
807− sizeof(void *), GFP_ATOMIC);
805+ netdev->xdp_bulkq = alloc_percpu(struct xdp_dev_bulk_queue);
808806 if (!netdev->xdp_bulkq)
809807 return NOTIFY_BAD;
810808
kernel/bpf/disasm.c+1−1
@@ -161,7 +161,7 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, · 160 unmodified lines
161161 insn->dst_reg,
162162 insn->off, insn->src_reg);
163163 else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
164− (insn->imm == BPF_ADD || insn->imm == BPF_ADD ||
164+ (insn->imm == BPF_ADD || insn->imm == BPF_AND ||
165165 insn->imm == BPF_OR || insn->imm == BPF_XOR)) {
166166 verbose(cbs->private_data, "(%02x) lock *(%s *)(r%d %+d) %s r%d\n",
167167 insn->code,
kernel/bpf/hashtab.c+2−2
@@ -1148,7 +1148,7 @@ static int __htab_percpu_map_update_elem(struct bpf_map *map, void *key, · 1147 unmodified lines
11481148 /* unknown flags */
11491149 return -EINVAL;
11501150
1151− WARN_ON_ONCE(!rcu_read_lock_held());
1151+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
11521152
11531153 key_size = map->key_size;
11541154
@@ -1202,7 +1202,7 @@ static int __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, · 47 unmodified lines
12021202 /* unknown flags */
12031203 return -EINVAL;
12041204
1205− WARN_ON_ONCE(!rcu_read_lock_held());
1205+ WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
12061206
12071207 key_size = map->key_size;
12081208
kernel/bpf/helpers.c+4−8
@@ -720,14 +720,6 @@ bpf_base_func_proto(enum bpf_func_id func_id) · 719 unmodified lines
720720 return &bpf_spin_lock_proto;
721721 case BPF_FUNC_spin_unlock:
722722 return &bpf_spin_unlock_proto;
723− case BPF_FUNC_trace_printk:
724− if (!perfmon_capable())
725− return NULL;
726− return bpf_get_trace_printk_proto();
727− case BPF_FUNC_snprintf_btf:
728− if (!perfmon_capable())
729− return NULL;
730− return &bpf_snprintf_btf_proto;
731723 case BPF_FUNC_jiffies64:
732724 return &bpf_jiffies64_proto;
733725 case BPF_FUNC_per_cpu_ptr:
@@ -742,6 +734,8 @@ bpf_base_func_proto(enum bpf_func_id func_id) · 8 unmodified lines
742734 return NULL;
743735
744736 switch (func_id) {
737+ case BPF_FUNC_trace_printk:
738+ return bpf_get_trace_printk_proto();
745739 case BPF_FUNC_get_current_task:
746740 return &bpf_get_current_task_proto;
747741 case BPF_FUNC_probe_read_user:
@@ -752,6 +746,8 @@ bpf_base_func_proto(enum bpf_func_id func_id) · 4 unmodified lines
752746 return &bpf_probe_read_user_str_proto;
753747 case BPF_FUNC_probe_read_kernel_str:
754748 return &bpf_probe_read_kernel_str_proto;
749+ case BPF_FUNC_snprintf_btf:
750+ return &bpf_snprintf_btf_proto;
755751 default:
756752 return NULL;
757753 }
kernel/bpf/syscall.c+11−5
@@ -1731,25 +1731,28 @@ static int bpf_prog_release(struct inode *inode, struct file *filp) · 1730 unmodified lines
17311731 static void bpf_prog_get_stats(const struct bpf_prog *prog,
17321732 struct bpf_prog_stats *stats)
17331733 {
1734− u64 nsecs = 0, cnt = 0;
1734+ u64 nsecs = 0, cnt = 0, misses = 0;
17351735 int cpu;
17361736
17371737 for_each_possible_cpu(cpu) {
17381738 const struct bpf_prog_stats *st;
17391739 unsigned int start;
1740− u64 tnsecs, tcnt;
1740+ u64 tnsecs, tcnt, tmisses;
17411741
1742− st = per_cpu_ptr(prog->aux->stats, cpu);
1742+ st = per_cpu_ptr(prog->stats, cpu);
17431743 do {
17441744 start = u64_stats_fetch_begin_irq(&st->syncp);
17451745 tnsecs = st->nsecs;
17461746 tcnt = st->cnt;
1747+ tmisses = st->misses;
17471748 } while (u64_stats_fetch_retry_irq(&st->syncp, start));
17481749 nsecs += tnsecs;
17491750 cnt += tcnt;
1751+ misses += tmisses;
17501752 }
17511753 stats->nsecs = nsecs;
17521754 stats->cnt = cnt;
1755+ stats->misses = misses;
17531756 }
17541757
17551758 #ifdef CONFIG_PROC_FS
@@ -1768,14 +1771,16 @@ static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp) · 12 unmodified lines
17681771 "memlock:\t%llu\n"
17691772 "prog_id:\t%u\n"
17701773 "run_time_ns:\t%llu\n"
1771− "run_cnt:\t%llu\n",
1774+ "run_cnt:\t%llu\n"
1775+ "recursion_misses:\t%llu\n",
17721776 prog->type,
17731777 prog->jited,
17741778 prog_tag,
17751779 prog->pages * 1ULL << PAGE_SHIFT,
17761780 prog->aux->id,
17771781 stats.nsecs,
1778− stats.cnt);
1782+ stats.cnt,
1783+ stats.misses);
17791784 }
17801785 #endif
17811786
@@ -3438,6 +3443,7 @@ static int bpf_prog_get_info_by_fd(struct file *file, · 1656 unmodified lines
34383443 bpf_prog_get_stats(prog, &stats);
34393444 info.run_time_ns = stats.nsecs;
34403445 info.run_cnt = stats.cnt;
3446+ info.recursion_misses = stats.misses;
34413447
34423448 if (!bpf_capable()) {
34433449 info.jited_prog_len = 0;
kernel/bpf/task_iter.c+266−1
@@ -286,9 +286,248 @@ static const struct seq_operations task_file_seq_ops = { · 285 unmodified lines
286286 .show = task_file_seq_show,
287287 };
288288
289+struct bpf_iter_seq_task_vma_info {
290+ /* The first field must be struct bpf_iter_seq_task_common.
291+ * this is assumed by {init, fini}_seq_pidns() callback functions.
292+ */
293+ struct bpf_iter_seq_task_common common;
294+ struct task_struct *task;
295+ struct vm_area_struct *vma;
296+ u32 tid;
297+ unsigned long prev_vm_start;
298+ unsigned long prev_vm_end;
299+};
300+
301+enum bpf_task_vma_iter_find_op {
302+ task_vma_iter_first_vma, /* use mm->mmap */
303+ task_vma_iter_next_vma, /* use curr_vma->vm_next */
304+ task_vma_iter_find_vma, /* use find_vma() to find next vma */
305+};
306+
307+static struct vm_area_struct *
308+task_vma_seq_get_next(struct bpf_iter_seq_task_vma_info *info)
309+{
310+ struct pid_namespace *ns = info->common.ns;
311+ enum bpf_task_vma_iter_find_op op;
312+ struct vm_area_struct *curr_vma;
313+ struct task_struct *curr_task;
314+ u32 curr_tid = info->tid;
315+
316+ /* If this function returns a non-NULL vma, it holds a reference to
317+ * the task_struct, and holds read lock on vma->mm->mmap_lock.
318+ * If this function returns NULL, it does not hold any reference or
319+ * lock.
320+ */
321+ if (info->task) {
322+ curr_task = info->task;
323+ curr_vma = info->vma;
324+ /* In case of lock contention, drop mmap_lock to unblock
325+ * the writer.
326+ *
327+ * After relock, call find(mm, prev_vm_end - 1) to find
328+ * new vma to process.
329+ *
330+ * +------+------+-----------+
331+ * | VMA1 | VMA2 | VMA3 |
332+ * +------+------+-----------+
333+ * | | | |
334+ * 4k 8k 16k 400k
335+ *
336+ * For example, curr_vma == VMA2. Before unlock, we set
337+ *
338+ * prev_vm_start = 8k
339+ * prev_vm_end = 16k
340+ *
341+ * There are a few cases:
342+ *
343+ * 1) VMA2 is freed, but VMA3 exists.
344+ *
345+ * find_vma() will return VMA3, just process VMA3.
346+ *
347+ * 2) VMA2 still exists.
348+ *
349+ * find_vma() will return VMA2, process VMA2->next.
350+ *
351+ * 3) no more vma in this mm.
352+ *
353+ * Process the next task.
354+ *
355+ * 4) find_vma() returns a different vma, VMA2'.
356+ *
357+ * 4.1) If VMA2 covers same range as VMA2', skip VMA2',
358+ * because we already covered the range;
359+ * 4.2) VMA2 and VMA2' covers different ranges, process
360+ * VMA2'.
361+ */
362+ if (mmap_lock_is_contended(curr_task->mm)) {
363+ info->prev_vm_start = curr_vma->vm_start;
364+ info->prev_vm_end = curr_vma->vm_end;
365+ op = task_vma_iter_find_vma;
366+ mmap_read_unlock(curr_task->mm);
367+ if (mmap_read_lock_killable(curr_task->mm))
368+ goto finish;
369+ } else {
370+ op = task_vma_iter_next_vma;
371+ }
372+ } else {
373+again:
374+ curr_task = task_seq_get_next(ns, &curr_tid, true);
375+ if (!curr_task) {
376+ info->tid = curr_tid + 1;
377+ goto finish;
378+ }
379+
380+ if (curr_tid != info->tid) {
381+ info->tid = curr_tid;
382+ /* new task, process the first vma */
383+ op = task_vma_iter_first_vma;
384+ } else {
385+ /* Found the same tid, which means the user space
386+ * finished data in previous buffer and read more.
387+ * We dropped mmap_lock before returning to user
388+ * space, so it is necessary to use find_vma() to
389+ * find the next vma to process.
390+ */
391+ op = task_vma_iter_find_vma;
392+ }
393+
394+ if (!curr_task->mm)
395+ goto next_task;
396+
397+ if (mmap_read_lock_killable(curr_task->mm))
398+ goto finish;
399+ }
400+
401+ switch (op) {
402+ case task_vma_iter_first_vma:
403+ curr_vma = curr_task->mm->mmap;
404+ break;
405+ case task_vma_iter_next_vma:
406+ curr_vma = curr_vma->vm_next;
407+ break;
408+ case task_vma_iter_find_vma:
409+ /* We dropped mmap_lock so it is necessary to use find_vma
410+ * to find the next vma. This is similar to the mechanism
411+ * in show_smaps_rollup().
412+ */
413+ curr_vma = find_vma(curr_task->mm, info->prev_vm_end - 1);
414+ /* case 1) and 4.2) above just use curr_vma */
415+
416+ /* check for case 2) or case 4.1) above */
417+ if (curr_vma &&
418+ curr_vma->vm_start == info->prev_vm_start &&
419+ curr_vma->vm_end == info->prev_vm_end)
420+ curr_vma = curr_vma->vm_next;
421+ break;
422+ }
423+ if (!curr_vma) {
424+ /* case 3) above, or case 2) 4.1) with vma->next == NULL */
425+ mmap_read_unlock(curr_task->mm);
426+ goto next_task;
427+ }
428+ info->task = curr_task;
429+ info->vma = curr_vma;
430+ return curr_vma;
431+
432+next_task:
433+ put_task_struct(curr_task);
434+ info->task = NULL;
435+ curr_tid++;
436+ goto again;
437+
438+finish:
439+ if (curr_task)
440+ put_task_struct(curr_task);
441+ info->task = NULL;
442+ info->vma = NULL;
443+ return NULL;
444+}
445+
446+static void *task_vma_seq_start(struct seq_file *seq, loff_t *pos)
447+{
448+ struct bpf_iter_seq_task_vma_info *info = seq->private;
449+ struct vm_area_struct *vma;
450+
451+ vma = task_vma_seq_get_next(info);
452+ if (vma && *pos == 0)
453+ ++*pos;
454+
455+ return vma;
456+}
457+
458+static void *task_vma_seq_next(struct seq_file *seq, void *v, loff_t *pos)
459+{
460+ struct bpf_iter_seq_task_vma_info *info = seq->private;
461+
462+ ++*pos;
463+ return task_vma_seq_get_next(info);
464+}
465+
466+struct bpf_iter__task_vma {
467+ __bpf_md_ptr(struct bpf_iter_meta *, meta);
468+ __bpf_md_ptr(struct task_struct *, task);
469+ __bpf_md_ptr(struct vm_area_struct *, vma);
470+};
471+
472+DEFINE_BPF_ITER_FUNC(task_vma, struct bpf_iter_meta *meta,
473+ struct task_struct *task, struct vm_area_struct *vma)
474+
475+static int __task_vma_seq_show(struct seq_file *seq, bool in_stop)
476+{
477+ struct bpf_iter_seq_task_vma_info *info = seq->private;
478+ struct bpf_iter__task_vma ctx;
479+ struct bpf_iter_meta meta;
480+ struct bpf_prog *prog;
481+
482+ meta.seq = seq;
483+ prog = bpf_iter_get_info(&meta, in_stop);
484+ if (!prog)
485+ return 0;
486+
487+ ctx.meta = &meta;
488+ ctx.task = info->task;
489+ ctx.vma = info->vma;
490+ return bpf_iter_run_prog(prog, &ctx);
491+}
492+
493+static int task_vma_seq_show(struct seq_file *seq, void *v)
494+{
495+ return __task_vma_seq_show(seq, false);
496+}
497+
498+static void task_vma_seq_stop(struct seq_file *seq, void *v)
499+{
500+ struct bpf_iter_seq_task_vma_info *info = seq->private;
501+
502+ if (!v) {
503+ (void)__task_vma_seq_show(seq, true);
504+ } else {
505+ /* info->vma has not been seen by the BPF program. If the
506+ * user space reads more, task_vma_seq_get_next should
507+ * return this vma again. Set prev_vm_start to ~0UL,
508+ * so that we don't skip the vma returned by the next
509+ * find_vma() (case task_vma_iter_find_vma in
510+ * task_vma_seq_get_next()).
511+ */
512+ info->prev_vm_start = ~0UL;
513+ info->prev_vm_end = info->vma->vm_end;
514+ mmap_read_unlock(info->task->mm);
515+ put_task_struct(info->task);
516+ info->task = NULL;
517+ }
518+}
519+
520+static const struct seq_operations task_vma_seq_ops = {
521+ .start = task_vma_seq_start,
522+ .next = task_vma_seq_next,
523+ .stop = task_vma_seq_stop,
524+ .show = task_vma_seq_show,
525+};
526+
289527 BTF_ID_LIST(btf_task_file_ids)
290528 BTF_ID(struct, task_struct)
291529 BTF_ID(struct, file)
530+BTF_ID(struct, vm_area_struct)
292531
293532 static const struct bpf_iter_seq_info task_seq_info = {
294533 .seq_ops = &task_seq_ops,
@@ -328,6 +567,26 @@ static struct bpf_iter_reg task_file_reg_info = { · 33 unmodified lines
328567 .seq_info = &task_file_seq_info,
329568 };
330569
570+static const struct bpf_iter_seq_info task_vma_seq_info = {
571+ .seq_ops = &task_vma_seq_ops,
572+ .init_seq_private = init_seq_pidns,
573+ .fini_seq_private = fini_seq_pidns,
574+ .seq_priv_size = sizeof(struct bpf_iter_seq_task_vma_info),
575+};
576+
577+static struct bpf_iter_reg task_vma_reg_info = {
578+ .target = "task_vma",
579+ .feature = BPF_ITER_RESCHED,
580+ .ctx_arg_info_size = 2,
581+ .ctx_arg_info = {
582+ { offsetof(struct bpf_iter__task_vma, task),
583+ PTR_TO_BTF_ID_OR_NULL },
584+ { offsetof(struct bpf_iter__task_vma, vma),
585+ PTR_TO_BTF_ID_OR_NULL },
586+ },
587+ .seq_info = &task_vma_seq_info,
588+};
589+
331590 static int __init task_iter_init(void)
332591 {
333592 int ret;
@@ -339,6 +598,12 @@ static int __init task_iter_init(void) · 5 unmodified lines
339598
340599 task_file_reg_info.ctx_arg_info[0].btf_id = btf_task_file_ids[0];
341600 task_file_reg_info.ctx_arg_info[1].btf_id = btf_task_file_ids[1];
342− return bpf_iter_reg_target(&task_file_reg_info);
601+ ret = bpf_iter_reg_target(&task_file_reg_info);
602+ if (ret)
603+ return ret;
604+
605+ task_vma_reg_info.ctx_arg_info[0].btf_id = btf_task_file_ids[0];
606+ task_vma_reg_info.ctx_arg_info[1].btf_id = btf_task_file_ids[2];
607+ return bpf_iter_reg_target(&task_vma_reg_info);
343608 }
344609 late_initcall(task_iter_init);
kernel/bpf/trampoline.c+61−16
@@ -381,55 +381,100 @@ out: · 380 unmodified lines
381381 mutex_unlock(&trampoline_mutex);
382382 }
383383
384+#define NO_START_TIME 1
385+static u64 notrace bpf_prog_start_time(void)
386+{
387+ u64 start = NO_START_TIME;
388+
389+ if (static_branch_unlikely(&bpf_stats_enabled_key)) {
390+ start = sched_clock();
391+ if (unlikely(!start))
392+ start = NO_START_TIME;
393+ }
394+ return start;
395+}
396+
397+static void notrace inc_misses_counter(struct bpf_prog *prog)
398+{
399+ struct bpf_prog_stats *stats;
400+
401+ stats = this_cpu_ptr(prog->stats);
402+ u64_stats_update_begin(&stats->syncp);
403+ stats->misses++;
404+ u64_stats_update_end(&stats->syncp);
405+}
406+
384407 /* The logic is similar to BPF_PROG_RUN, but with an explicit
385408 * rcu_read_lock() and migrate_disable() which are required
386409 * for the trampoline. The macro is split into
387− * call _bpf_prog_enter
410+ * call __bpf_prog_enter
388411 * call prog->bpf_func
389412 * call __bpf_prog_exit
413+ *
414+ * __bpf_prog_enter returns:
415+ * 0 - skip execution of the bpf prog
416+ * 1 - execute bpf prog
417+ * [2..MAX_U64] - excute bpf prog and record execution time.
418+ * This is start time.
390419 */
391−u64 notrace __bpf_prog_enter(void)
420+u64 notrace __bpf_prog_enter(struct bpf_prog *prog)
392421 __acquires(RCU)
393422 {
394− u64 start = 0;
395−
396423 rcu_read_lock();
397424 migrate_disable();
398− if (static_branch_unlikely(&bpf_stats_enabled_key))
399− start = sched_clock();
400− return start;
425+ if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) {
426+ inc_misses_counter(prog);
427+ return 0;
428+ }
429+ return bpf_prog_start_time();
401430 }
402431
403−void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start)
404− __releases(RCU)
432+static void notrace update_prog_stats(struct bpf_prog *prog,
433+ u64 start)
405434 {
406435 struct bpf_prog_stats *stats;
407436
408437 if (static_branch_unlikely(&bpf_stats_enabled_key) &&
409− /* static_key could be enabled in __bpf_prog_enter
410− * and disabled in __bpf_prog_exit.
438+ /* static_key could be enabled in __bpf_prog_enter*
439+ * and disabled in __bpf_prog_exit*.
411440 * And vice versa.
412− * Hence check that 'start' is not zero.
441+ * Hence check that 'start' is valid.
413442 */
414− start) {
415− stats = this_cpu_ptr(prog->aux->stats);
443+ start > NO_START_TIME) {
444+ stats = this_cpu_ptr(prog->stats);
416445 u64_stats_update_begin(&stats->syncp);
417446 stats->cnt++;
418447 stats->nsecs += sched_clock() - start;
419448 u64_stats_update_end(&stats->syncp);
420449 }
450+}
451+
452+void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start)
453+ __releases(RCU)
454+{
455+ update_prog_stats(prog, start);
456+ __this_cpu_dec(*(prog->active));
421457 migrate_enable();
422458 rcu_read_unlock();
423459 }
424460
425−void notrace __bpf_prog_enter_sleepable(void)
461+u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog)
426462 {
427463 rcu_read_lock_trace();
464+ migrate_disable();
428465 might_fault();
466+ if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) {
467+ inc_misses_counter(prog);
468+ return 0;
469+ }
470+ return bpf_prog_start_time();
429471 }
430472
431−void notrace __bpf_prog_exit_sleepable(void)
473+void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start)
432474 {
475+ update_prog_stats(prog, start);
476+ __this_cpu_dec(*(prog->active));
477+ migrate_enable();
433478 rcu_read_unlock_trace();
434479 }
435480
kernel/bpf/verifier.c+662−215
@@ -228,6 +228,12 @@ static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state) · 227 unmodified lines
228228 (poisoned ? BPF_MAP_KEY_POISON : 0ULL);
229229 }
230230
231+static bool bpf_pseudo_call(const struct bpf_insn *insn)
232+{
233+ return insn->code == (BPF_JMP | BPF_CALL) &&
234+ insn->src_reg == BPF_PSEUDO_CALL;
235+}
236+
231237 struct bpf_call_arg_meta {
232238 struct bpf_map *map_ptr;
233239 bool raw_mode;
@@ -1073,6 +1079,51 @@ static void mark_reg_known_zero(struct bpf_verifier_env *env, · 839 unmodified lines
10731079 __mark_reg_known_zero(regs + regno);
10741080 }
10751081
1082+static void mark_ptr_not_null_reg(struct bpf_reg_state *reg)
1083+{
1084+ switch (reg->type) {
1085+ case PTR_TO_MAP_VALUE_OR_NULL: {
1086+ const struct bpf_map *map = reg->map_ptr;
1087+
1088+ if (map->inner_map_meta) {
1089+ reg->type = CONST_PTR_TO_MAP;
1090+ reg->map_ptr = map->inner_map_meta;
1091+ } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) {
1092+ reg->type = PTR_TO_XDP_SOCK;
1093+ } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP ||
1094+ map->map_type == BPF_MAP_TYPE_SOCKHASH) {
1095+ reg->type = PTR_TO_SOCKET;
1096+ } else {
1097+ reg->type = PTR_TO_MAP_VALUE;
1098+ }
1099+ break;
1100+ }
1101+ case PTR_TO_SOCKET_OR_NULL:
1102+ reg->type = PTR_TO_SOCKET;
1103+ break;
1104+ case PTR_TO_SOCK_COMMON_OR_NULL:
1105+ reg->type = PTR_TO_SOCK_COMMON;
1106+ break;
1107+ case PTR_TO_TCP_SOCK_OR_NULL:
1108+ reg->type = PTR_TO_TCP_SOCK;
1109+ break;
1110+ case PTR_TO_BTF_ID_OR_NULL:
1111+ reg->type = PTR_TO_BTF_ID;
1112+ break;
1113+ case PTR_TO_MEM_OR_NULL:
1114+ reg->type = PTR_TO_MEM;
1115+ break;
1116+ case PTR_TO_RDONLY_BUF_OR_NULL:
1117+ reg->type = PTR_TO_RDONLY_BUF;
1118+ break;
1119+ case PTR_TO_RDWR_BUF_OR_NULL:
1120+ reg->type = PTR_TO_RDWR_BUF;
1121+ break;
1122+ default:
1123+ WARN_ON("unknown nullable register type");
1124+ }
1125+}
1126+
10761127 static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)
10771128 {
10781129 return type_is_pkt_pointer(reg->type);
@@ -1486,9 +1537,7 @@ static int check_subprogs(struct bpf_verifier_env *env) · 407 unmodified lines
14861537
14871538 /* determine subprog starts. The end is one before the next starts */
14881539 for (i = 0; i < insn_cnt; i++) {
1489− if (insn[i].code != (BPF_JMP | BPF_CALL))
1490− continue;
1491− if (insn[i].src_reg != BPF_PSEUDO_CALL)
1540+ if (!bpf_pseudo_call(insn + i))
14921541 continue;
14931542 if (!env->bpf_capable) {
14941543 verbose(env,
@@ -2271,12 +2320,14 @@ static void save_register_state(struct bpf_func_state *state, · 776 unmodified lines
22712320 state->stack[spi].slot_type[i] = STACK_SPILL;
22722321 }
22732322
2274−/* check_stack_read/write functions track spill/fill of registers,
2323+/* check_stack_{read,write}_fixed_off functions track spill/fill of registers,
22752324 * stack boundary and alignment are checked in check_mem_access()
22762325 */
2277−static int check_stack_write(struct bpf_verifier_env *env,
2278− struct bpf_func_state *state, /* func where register points to */
2279− int off, int size, int value_regno, int insn_idx)
2326+static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
2327+ /* stack frame we're writing to */
2328+ struct bpf_func_state *state,
2329+ int off, int size, int value_regno,
2330+ int insn_idx)
22802331 {
22812332 struct bpf_func_state *cur; /* state of the current function */
22822333 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err;
@@ -2402,9 +2453,175 @@ static int check_stack_write(struct bpf_verifier_env *env, · 119 unmodified lines
24022453 return 0;
24032454 }
24042455
2405−static int check_stack_read(struct bpf_verifier_env *env,
2406− struct bpf_func_state *reg_state /* func where register points to */,
2407− int off, int size, int value_regno)
2456+/* Write the stack: 'stack[ptr_regno + off] = value_regno'. 'ptr_regno' is
2457+ * known to contain a variable offset.
2458+ * This function checks whether the write is permitted and conservatively
2459+ * tracks the effects of the write, considering that each stack slot in the
2460+ * dynamic range is potentially written to.
2461+ *
2462+ * 'off' includes 'regno->off'.
2463+ * 'value_regno' can be -1, meaning that an unknown value is being written to
2464+ * the stack.
2465+ *
2466+ * Spilled pointers in range are not marked as written because we don't know
2467+ * what's going to be actually written. This means that read propagation for
2468+ * future reads cannot be terminated by this write.
2469+ *
2470+ * For privileged programs, uninitialized stack slots are considered
2471+ * initialized by this write (even though we don't know exactly what offsets
2472+ * are going to be written to). The idea is that we don't want the verifier to
2473+ * reject future reads that access slots written to through variable offsets.
2474+ */
2475+static int check_stack_write_var_off(struct bpf_verifier_env *env,
2476+ /* func where register points to */
2477+ struct bpf_func_state *state,
2478+ int ptr_regno, int off, int size,
2479+ int value_regno, int insn_idx)
2480+{
2481+ struct bpf_func_state *cur; /* state of the current function */
2482+ int min_off, max_off;
2483+ int i, err;
2484+ struct bpf_reg_state *ptr_reg = NULL, *value_reg = NULL;
2485+ bool writing_zero = false;
2486+ /* set if the fact that we're writing a zero is used to let any
2487+ * stack slots remain STACK_ZERO
2488+ */
2489+ bool zero_used = false;
2490+
2491+ cur = env->cur_state->frame[env->cur_state->curframe];
2492+ ptr_reg = &cur->regs[ptr_regno];
2493+ min_off = ptr_reg->smin_value + off;
2494+ max_off = ptr_reg->smax_value + off + size;
2495+ if (value_regno >= 0)
2496+ value_reg = &cur->regs[value_regno];
2497+ if (value_reg && register_is_null(value_reg))
2498+ writing_zero = true;
2499+
2500+ err = realloc_func_state(state, round_up(-min_off, BPF_REG_SIZE),
2501+ state->acquired_refs, true);
2502+ if (err)
2503+ return err;
2504+
2505+
2506+ /* Variable offset writes destroy any spilled pointers in range. */
2507+ for (i = min_off; i < max_off; i++) {
2508+ u8 new_type, *stype;
2509+ int slot, spi;
2510+
2511+ slot = -i - 1;
2512+ spi = slot / BPF_REG_SIZE;
2513+ stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
2514+
2515+ if (!env->allow_ptr_leaks
2516+ && *stype != NOT_INIT
2517+ && *stype != SCALAR_VALUE) {
2518+ /* Reject the write if there's are spilled pointers in
2519+ * range. If we didn't reject here, the ptr status
2520+ * would be erased below (even though not all slots are
2521+ * actually overwritten), possibly opening the door to
2522+ * leaks.
2523+ */
2524+ verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d",
2525+ insn_idx, i);
2526+ return -EINVAL;
2527+ }
2528+
2529+ /* Erase all spilled pointers. */
2530+ state->stack[spi].spilled_ptr.type = NOT_INIT;
2531+
2532+ /* Update the slot type. */
2533+ new_type = STACK_MISC;
2534+ if (writing_zero && *stype == STACK_ZERO) {
2535+ new_type = STACK_ZERO;
2536+ zero_used = true;
2537+ }
2538+ /* If the slot is STACK_INVALID, we check whether it's OK to
2539+ * pretend that it will be initialized by this write. The slot
2540+ * might not actually be written to, and so if we mark it as
2541+ * initialized future reads might leak uninitialized memory.
2542+ * For privileged programs, we will accept such reads to slots
2543+ * that may or may not be written because, if we're reject
2544+ * them, the error would be too confusing.
2545+ */
2546+ if (*stype == STACK_INVALID && !env->allow_uninit_stack) {
2547+ verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d",
2548+ insn_idx, i);
2549+ return -EINVAL;
2550+ }
2551+ *stype = new_type;
2552+ }
2553+ if (zero_used) {
2554+ /* backtracking doesn't work for STACK_ZERO yet. */
2555+ err = mark_chain_precision(env, value_regno);
2556+ if (err)
2557+ return err;
2558+ }
2559+ return 0;
2560+}
2561+
2562+/* When register 'dst_regno' is assigned some values from stack[min_off,
2563+ * max_off), we set the register's type according to the types of the
2564+ * respective stack slots. If all the stack values are known to be zeros, then
2565+ * so is the destination reg. Otherwise, the register is considered to be
2566+ * SCALAR. This function does not deal with register filling; the caller must
2567+ * ensure that all spilled registers in the stack range have been marked as
2568+ * read.
2569+ */
2570+static void mark_reg_stack_read(struct bpf_verifier_env *env,
2571+ /* func where src register points to */
2572+ struct bpf_func_state *ptr_state,
2573+ int min_off, int max_off, int dst_regno)
2574+{
2575+ struct bpf_verifier_state *vstate = env->cur_state;
2576+ struct bpf_func_state *state = vstate->frame[vstate->curframe];
2577+ int i, slot, spi;
2578+ u8 *stype;
2579+ int zeros = 0;
2580+
2581+ for (i = min_off; i < max_off; i++) {
2582+ slot = -i - 1;
2583+ spi = slot / BPF_REG_SIZE;
2584+ stype = ptr_state->stack[spi].slot_type;
2585+ if (stype[slot % BPF_REG_SIZE] != STACK_ZERO)
2586+ break;
2587+ zeros++;
2588+ }
2589+ if (zeros == max_off - min_off) {
2590+ /* any access_size read into register is zero extended,
2591+ * so the whole register == const_zero
2592+ */
2593+ __mark_reg_const_zero(&state->regs[dst_regno]);
2594+ /* backtracking doesn't support STACK_ZERO yet,
2595+ * so mark it precise here, so that later
2596+ * backtracking can stop here.
2597+ * Backtracking may not need this if this register
2598+ * doesn't participate in pointer adjustment.
2599+ * Forward propagation of precise flag is not
2600+ * necessary either. This mark is only to stop
2601+ * backtracking. Any register that contributed
2602+ * to const 0 was marked precise before spill.
2603+ */
2604+ state->regs[dst_regno].precise = true;
2605+ } else {
2606+ /* have read misc data from the stack */
2607+ mark_reg_unknown(env, state->regs, dst_regno);
2608+ }
2609+ state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
2610+}
2611+
2612+/* Read the stack at 'off' and put the results into the register indicated by
2613+ * 'dst_regno'. It handles reg filling if the addressed stack slot is a
2614+ * spilled reg.
2615+ *
2616+ * 'dst_regno' can be -1, meaning that the read value is not going to a
2617+ * register.
2618+ *
2619+ * The access is assumed to be within the current stack bounds.
2620+ */
2621+static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
2622+ /* func where src register points to */
2623+ struct bpf_func_state *reg_state,
2624+ int off, int size, int dst_regno)
24082625 {
24092626 struct bpf_verifier_state *vstate = env->cur_state;
24102627 struct bpf_func_state *state = vstate->frame[vstate->curframe];
@@ -2412,11 +2629,6 @@ static int check_stack_read(struct bpf_verifier_env *env, · 1 unmodified line
24122629 struct bpf_reg_state *reg;
24132630 u8 *stype;
24142631
2415− if (reg_state->allocated_stack <= slot) {
2416− verbose(env, "invalid read from stack off %d+0 size %d\n",
2417− off, size);
2418− return -EACCES;
2419− }
24202632 stype = reg_state->stack[spi].slot_type;
24212633 reg = &reg_state->stack[spi].spilled_ptr;
24222634
@@ -2427,9 +2639,9 @@ static int check_stack_read(struct bpf_verifier_env *env, · 4 unmodified lines
24272639 verbose(env, "invalid size of register fill\n");
24282640 return -EACCES;
24292641 }
2430− if (value_regno >= 0) {
2431− mark_reg_unknown(env, state->regs, value_regno);
2432− state->regs[value_regno].live |= REG_LIVE_WRITTEN;
2642+ if (dst_regno >= 0) {
2643+ mark_reg_unknown(env, state->regs, dst_regno);
2644+ state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
24332645 }
24342646 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
24352647 return 0;
@@ -2441,16 +2653,16 @@ static int check_stack_read(struct bpf_verifier_env *env, · 5 unmodified lines
24412653 }
24422654 }
24432655
2444− if (value_regno >= 0) {
2656+ if (dst_regno >= 0) {
24452657 /* restore register state from stack */
2446− state->regs[value_regno] = *reg;
2658+ state->regs[dst_regno] = *reg;
24472659 /* mark reg as written since spilled pointer state likely
24482660 * has its liveness marks cleared by is_state_visited()
24492661 * which resets stack/reg liveness for state transitions
24502662 */
2451− state->regs[value_regno].live |= REG_LIVE_WRITTEN;
2663+ state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
24522664 } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) {
2453− /* If value_regno==-1, the caller is asking us whether
2665+ /* If dst_regno==-1, the caller is asking us whether
24542666 * it is acceptable to use this value as a SCALAR_VALUE
24552667 * (e.g. for XADD).
24562668 * We must not allow unprivileged callers to do that
@@ -2462,70 +2674,167 @@ static int check_stack_read(struct bpf_verifier_env *env, · 5 unmodified lines
24622674 }
24632675 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
24642676 } else {
2465− int zeros = 0;
2677+ u8 type;
24662678
24672679 for (i = 0; i < size; i++) {
2468− if (stype[(slot - i) % BPF_REG_SIZE] == STACK_MISC)
2680+ type = stype[(slot - i) % BPF_REG_SIZE];
2681+ if (type == STACK_MISC)
24692682 continue;
2470− if (stype[(slot - i) % BPF_REG_SIZE] == STACK_ZERO) {
2471− zeros++;
2683+ if (type == STACK_ZERO)
24722684 continue;
2473− }
24742685 verbose(env, "invalid read from stack off %d+%d size %d\n",
24752686 off, i, size);
24762687 return -EACCES;
24772688 }
24782689 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
2479− if (value_regno >= 0) {
2480− if (zeros == size) {
2481− /* any size read into register is zero extended,
2482− * so the whole register == const_zero
2483− */
2484− __mark_reg_const_zero(&state->regs[value_regno]);
2485− /* backtracking doesn't support STACK_ZERO yet,
2486− * so mark it precise here, so that later
2487− * backtracking can stop here.
2488− * Backtracking may not need this if this register
2489− * doesn't participate in pointer adjustment.
2490− * Forward propagation of precise flag is not
2491− * necessary either. This mark is only to stop
2492− * backtracking. Any register that contributed
2493− * to const 0 was marked precise before spill.
2494− */
2495− state->regs[value_regno].precise = true;
2496− } else {
2497− /* have read misc data from the stack */
2498− mark_reg_unknown(env, state->regs, value_regno);
2499− }
2500− state->regs[value_regno].live |= REG_LIVE_WRITTEN;
2501− }
2690+ if (dst_regno >= 0)
2691+ mark_reg_stack_read(env, reg_state, off, off + size, dst_regno);
25022692 }
25032693 return 0;
25042694 }
25052695
2506−static int check_stack_access(struct bpf_verifier_env *env,
2507− const struct bpf_reg_state *reg,
2508− int off, int size)
2696+enum stack_access_src {
2697+ ACCESS_DIRECT = 1, /* the access is performed by an instruction */
2698+ ACCESS_HELPER = 2, /* the access is performed by a helper */
2699+};
2700+
2701+static int check_stack_range_initialized(struct bpf_verifier_env *env,
2702+ int regno, int off, int access_size,
2703+ bool zero_size_allowed,
2704+ enum stack_access_src type,
2705+ struct bpf_call_arg_meta *meta);
2706+
2707+static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)
2708+{
2709+ return cur_regs(env) + regno;
2710+}
2711+
2712+/* Read the stack at 'ptr_regno + off' and put the result into the register
2713+ * 'dst_regno'.
2714+ * 'off' includes the pointer register's fixed offset(i.e. 'ptr_regno.off'),
2715+ * but not its variable offset.
2716+ * 'size' is assumed to be <= reg size and the access is assumed to be aligned.
2717+ *
2718+ * As opposed to check_stack_read_fixed_off, this function doesn't deal with
2719+ * filling registers (i.e. reads of spilled register cannot be detected when
2720+ * the offset is not fixed). We conservatively mark 'dst_regno' as containing
2721+ * SCALAR_VALUE. That's why we assert that the 'ptr_regno' has a variable
2722+ * offset; for a fixed offset check_stack_read_fixed_off should be used
2723+ * instead.
2724+ */
2725+static int check_stack_read_var_off(struct bpf_verifier_env *env,
2726+ int ptr_regno, int off, int size, int dst_regno)
25092727 {
2510− /* Stack accesses must be at a fixed offset, so that we
2511− * can determine what type of data were returned. See
2512− * check_stack_read().
2728+ /* The state of the source register. */
2729+ struct bpf_reg_state *reg = reg_state(env, ptr_regno);
2730+ struct bpf_func_state *ptr_state = func(env, reg);
2731+ int err;
2732+ int min_off, max_off;
2733+
2734+ /* Note that we pass a NULL meta, so raw access will not be permitted.
25132735 */
2514− if (!tnum_is_const(reg->var_off)) {
2736+ err = check_stack_range_initialized(env, ptr_regno, off, size,
2737+ false, ACCESS_DIRECT, NULL);
2738+ if (err)
2739+ return err;
2740+
2741+ min_off = reg->smin_value + off;
2742+ max_off = reg->smax_value + off;
2743+ mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno);
2744+ return 0;
2745+}
2746+
2747+/* check_stack_read dispatches to check_stack_read_fixed_off or
2748+ * check_stack_read_var_off.
2749+ *
2750+ * The caller must ensure that the offset falls within the allocated stack
2751+ * bounds.
2752+ *
2753+ * 'dst_regno' is a register which will receive the value from the stack. It
2754+ * can be -1, meaning that the read value is not going to a register.
2755+ */
2756+static int check_stack_read(struct bpf_verifier_env *env,
2757+ int ptr_regno, int off, int size,
2758+ int dst_regno)
2759+{
2760+ struct bpf_reg_state *reg = reg_state(env, ptr_regno);
2761+ struct bpf_func_state *state = func(env, reg);
2762+ int err;
2763+ /* Some accesses are only permitted with a static offset. */
2764+ bool var_off = !tnum_is_const(reg->var_off);
2765+
2766+ /* The offset is required to be static when reads don't go to a
2767+ * register, in order to not leak pointers (see
2768+ * check_stack_read_fixed_off).
2769+ */
2770+ if (dst_regno < 0 && var_off) {
25152771 char tn_buf[48];
25162772
25172773 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
2518− verbose(env, "variable stack access var_off=%s off=%d size=%d\n",
2774+ verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n",
25192775 tn_buf, off, size);
25202776 return -EACCES;
25212777 }
2778+ /* Variable offset is prohibited for unprivileged mode for simplicity
2779+ * since it requires corresponding support in Spectre masking for stack
2780+ * ALU. See also retrieve_ptr_limit().
2781+ */
2782+ if (!env->bypass_spec_v1 && var_off) {
2783+ char tn_buf[48];
25222784
2523− if (off >= 0 || off < -MAX_BPF_STACK) {
2524− verbose(env, "invalid stack off=%d size=%d\n", off, size);
2785+ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
2786+ verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n",
2787+ ptr_regno, tn_buf);
25252788 return -EACCES;
25262789 }
25272790
2528− return 0;
2791+ if (!var_off) {
2792+ off += reg->var_off.value;
2793+ err = check_stack_read_fixed_off(env, state, off, size,
2794+ dst_regno);
2795+ } else {
2796+ /* Variable offset stack reads need more conservative handling
2797+ * than fixed offset ones. Note that dst_regno >= 0 on this
2798+ * branch.
2799+ */
2800+ err = check_stack_read_var_off(env, ptr_regno, off, size,
2801+ dst_regno);
2802+ }
2803+ return err;
2804+}
2805+
2806+
2807+/* check_stack_write dispatches to check_stack_write_fixed_off or
2808+ * check_stack_write_var_off.
2809+ *
2810+ * 'ptr_regno' is the register used as a pointer into the stack.
2811+ * 'off' includes 'ptr_regno->off', but not its variable offset (if any).
2812+ * 'value_regno' is the register whose value we're writing to the stack. It can
2813+ * be -1, meaning that we're not writing from a register.
2814+ *
2815+ * The caller must ensure that the offset falls within the maximum stack size.
2816+ */
2817+static int check_stack_write(struct bpf_verifier_env *env,
2818+ int ptr_regno, int off, int size,
2819+ int value_regno, int insn_idx)
2820+{
2821+ struct bpf_reg_state *reg = reg_state(env, ptr_regno);
2822+ struct bpf_func_state *state = func(env, reg);
2823+ int err;
2824+
2825+ if (tnum_is_const(reg->var_off)) {
2826+ off += reg->var_off.value;
2827+ err = check_stack_write_fixed_off(env, state, off, size,
2828+ value_regno, insn_idx);
2829+ } else {
2830+ /* Variable offset stack reads need more conservative handling
2831+ * than fixed offset ones.
2832+ */
2833+ err = check_stack_write_var_off(env, state,
2834+ ptr_regno, off, size,
2835+ value_regno, insn_idx);
2836+ }
2837+ return err;
25292838 }
25302839
25312840 static int check_map_access_type(struct bpf_verifier_env *env, u32 regno,
@@ -2858,11 +3167,6 @@ static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, · 326 unmodified lines
28583167 return -EACCES;
28593168 }
28603169
2861−static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)
2862−{
2863− return cur_regs(env) + regno;
2864−}
2865−
28663170 static bool is_pointer_value(struct bpf_verifier_env *env, int regno)
28673171 {
28683172 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno));
@@ -2981,8 +3285,8 @@ static int check_ptr_alignment(struct bpf_verifier_env *env, · 112 unmodified lines
29813285 break;
29823286 case PTR_TO_STACK:
29833287 pointer_desc = "stack ";
2984− /* The stack spill tracking logic in check_stack_write()
2985− * and check_stack_read() relies on stack accesses being
3288+ /* The stack spill tracking logic in check_stack_write_fixed_off()
3289+ * and check_stack_read_fixed_off() relies on stack accesses being
29863290 * aligned.
29873291 */
29883292 strict = true;
@@ -3074,9 +3378,7 @@ process_func: · 85 unmodified lines
30743378 continue_func:
30753379 subprog_end = subprog[idx + 1].start;
30763380 for (; i < subprog_end; i++) {
3077− if (insn[i].code != (BPF_JMP | BPF_CALL))
3078− continue;
3079− if (insn[i].src_reg != BPF_PSEUDO_CALL)
3381+ if (!bpf_pseudo_call(insn + i))
30803382 continue;
30813383 /* remember insn and function to return to */
30823384 ret_insn[frame] = i + 1;
@@ -3400,6 +3702,91 @@ static int check_ptr_to_map_access(struct bpf_verifier_env *env, · 317 unmodified lines
34003702 return 0;
34013703 }
34023704
3705+/* Check that the stack access at the given offset is within bounds. The
3706+ * maximum valid offset is -1.
3707+ *
3708+ * The minimum valid offset is -MAX_BPF_STACK for writes, and
3709+ * -state->allocated_stack for reads.
3710+ */
3711+static int check_stack_slot_within_bounds(int off,
3712+ struct bpf_func_state *state,
3713+ enum bpf_access_type t)
3714+{
3715+ int min_valid_off;
3716+
3717+ if (t == BPF_WRITE)
3718+ min_valid_off = -MAX_BPF_STACK;
3719+ else
3720+ min_valid_off = -state->allocated_stack;
3721+
3722+ if (off < min_valid_off || off > -1)
3723+ return -EACCES;
3724+ return 0;
3725+}
3726+
3727+/* Check that the stack access at 'regno + off' falls within the maximum stack
3728+ * bounds.
3729+ *
3730+ * 'off' includes `regno->offset`, but not its dynamic part (if any).
3731+ */
3732+static int check_stack_access_within_bounds(
3733+ struct bpf_verifier_env *env,
3734+ int regno, int off, int access_size,
3735+ enum stack_access_src src, enum bpf_access_type type)
3736+{
3737+ struct bpf_reg_state *regs = cur_regs(env);
3738+ struct bpf_reg_state *reg = regs + regno;
3739+ struct bpf_func_state *state = func(env, reg);
3740+ int min_off, max_off;
3741+ int err;
3742+ char *err_extra;
3743+
3744+ if (src == ACCESS_HELPER)
3745+ /* We don't know if helpers are reading or writing (or both). */
3746+ err_extra = " indirect access to";
3747+ else if (type == BPF_READ)
3748+ err_extra = " read from";
3749+ else
3750+ err_extra = " write to";
3751+
3752+ if (tnum_is_const(reg->var_off)) {
3753+ min_off = reg->var_off.value + off;
3754+ if (access_size > 0)
3755+ max_off = min_off + access_size - 1;
3756+ else
3757+ max_off = min_off;
3758+ } else {
3759+ if (reg->smax_value >= BPF_MAX_VAR_OFF ||
3760+ reg->smin_value <= -BPF_MAX_VAR_OFF) {
3761+ verbose(env, "invalid unbounded variable-offset%s stack R%d\n",
3762+ err_extra, regno);
3763+ return -EACCES;
3764+ }
3765+ min_off = reg->smin_value + off;
3766+ if (access_size > 0)
3767+ max_off = reg->smax_value + off + access_size - 1;
3768+ else
3769+ max_off = min_off;
3770+ }
3771+
3772+ err = check_stack_slot_within_bounds(min_off, state, type);
3773+ if (!err)
3774+ err = check_stack_slot_within_bounds(max_off, state, type);
3775+
3776+ if (err) {
3777+ if (tnum_is_const(reg->var_off)) {
3778+ verbose(env, "invalid%s stack R%d off=%d size=%d\n",
3779+ err_extra, regno, off, access_size);
3780+ } else {
3781+ char tn_buf[48];
3782+
3783+ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3784+ verbose(env, "invalid variable-offset%s stack R%d var_off=%s size=%d\n",
3785+ err_extra, regno, tn_buf, access_size);
3786+ }
3787+ }
3788+ return err;
3789+}
34033790
34043791 /* check whether memory at (regno + off) is accessible for t = (read | write)
34053792 * if t==write, value_regno is a register which value is stored into memory
@@ -3515,8 +3902,8 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 109 unmodified lines
35153902 }
35163903
35173904 } else if (reg->type == PTR_TO_STACK) {
3518− off += reg->var_off.value;
3519− err = check_stack_access(env, reg, off, size);
3905+ /* Basic bounds checks. */
3906+ err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t);
35203907 if (err)
35213908 return err;
35223909
@@ -3525,12 +3912,12 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 2 unmodified lines
35253912 if (err)
35263913 return err;
35273914
3528− if (t == BPF_WRITE)
3529− err = check_stack_write(env, state, off, size,
3530− value_regno, insn_idx);
3531− else
3532− err = check_stack_read(env, state, off, size,
3915+ if (t == BPF_READ)
3916+ err = check_stack_read(env, regno, off, size,
35333917 value_regno);
3918+ else
3919+ err = check_stack_write(env, regno, off, size,
3920+ value_regno, insn_idx);
35343921 } else if (reg_is_pkt_pointer(reg)) {
35353922 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) {
35363923 verbose(env, "cannot write into packet\n");
@@ -3665,9 +4052,26 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i · 128 unmodified lines
36654052 return -EACCES;
36664053 }
36674054
4055+ if (insn->imm & BPF_FETCH) {
4056+ if (insn->imm == BPF_CMPXCHG)
4057+ load_reg = BPF_REG_0;
4058+ else
4059+ load_reg = insn->src_reg;
4060+
4061+ /* check and record load of old value */
4062+ err = check_reg_arg(env, load_reg, DST_OP);
4063+ if (err)
4064+ return err;
4065+ } else {
4066+ /* This instruction accesses a memory location but doesn't
4067+ * actually load it into a register.
4068+ */
4069+ load_reg = -1;
4070+ }
4071+
36684072 /* check whether we can read the memory */
36694073 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
3670− BPF_SIZE(insn->code), BPF_READ, -1, true);
4074+ BPF_SIZE(insn->code), BPF_READ, load_reg, true);
36714075 if (err)
36724076 return err;
36734077
@@ -3677,65 +4081,56 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i · 3 unmodified lines
36774081 if (err)
36784082 return err;
36794083
3680− if (!(insn->imm & BPF_FETCH))
3681− return 0;
3682−
3683− if (insn->imm == BPF_CMPXCHG)
3684− load_reg = BPF_REG_0;
3685− else
3686− load_reg = insn->src_reg;
3687−
3688− /* check and record load of old value */
3689− err = check_reg_arg(env, load_reg, DST_OP);
3690− if (err)
3691− return err;
3692−
36934084 return 0;
36944085 }
36954086
3696−static int __check_stack_boundary(struct bpf_verifier_env *env, u32 regno,
3697− int off, int access_size,
3698− bool zero_size_allowed)
4087+/* When register 'regno' is used to read the stack (either directly or through
4088+ * a helper function) make sure that it's within stack boundary and, depending
4089+ * on the access type, that all elements of the stack are initialized.
4090+ *
4091+ * 'off' includes 'regno->off', but not its dynamic part (if any).
4092+ *
4093+ * All registers that have been spilled on the stack in the slots within the
4094+ * read offsets are marked as read.
4095+ */
4096+static int check_stack_range_initialized(
4097+ struct bpf_verifier_env *env, int regno, int off,
4098+ int access_size, bool zero_size_allowed,
4099+ enum stack_access_src type, struct bpf_call_arg_meta *meta)
36994100 {
37004101 struct bpf_reg_state *reg = reg_state(env, regno);
4102+ struct bpf_func_state *state = func(env, reg);
4103+ int err, min_off, max_off, i, j, slot, spi;
4104+ char *err_extra = type == ACCESS_HELPER ? " indirect" : "";
4105+ enum bpf_access_type bounds_check_type;
4106+ /* Some accesses can write anything into the stack, others are
4107+ * read-only.
4108+ */
4109+ bool clobber = false;
37014110
3702− if (off >= 0 || off < -MAX_BPF_STACK || off + access_size > 0 ||
3703− access_size < 0 || (access_size == 0 && !zero_size_allowed)) {
3704− if (tnum_is_const(reg->var_off)) {
3705− verbose(env, "invalid stack type R%d off=%d access_size=%d\n",
3706− regno, off, access_size);
3707− } else {
3708− char tn_buf[48];
3709−
3710− tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3711− verbose(env, "invalid stack type R%d var_off=%s access_size=%d\n",
3712− regno, tn_buf, access_size);
3713− }
4111+ if (access_size == 0 && !zero_size_allowed) {
4112+ verbose(env, "invalid zero-sized read\n");
37144113 return -EACCES;
37154114 }
3716− return 0;
3717−}
37184115
3719−/* when register 'regno' is passed into function that will read 'access_size'
3720− * bytes from that pointer, make sure that it's within stack boundary
3721− * and all elements of stack are initialized.
3722− * Unlike most pointer bounds-checking functions, this one doesn't take an
3723− * 'off' argument, so it has to add in reg->off itself.
3724− */
3725−static int check_stack_boundary(struct bpf_verifier_env *env, int regno,
3726− int access_size, bool zero_size_allowed,
3727− struct bpf_call_arg_meta *meta)
3728−{
3729− struct bpf_reg_state *reg = reg_state(env, regno);
3730− struct bpf_func_state *state = func(env, reg);
3731− int err, min_off, max_off, i, j, slot, spi;
4116+ if (type == ACCESS_HELPER) {
4117+ /* The bounds checks for writes are more permissive than for
4118+ * reads. However, if raw_mode is not set, we'll do extra
4119+ * checks below.
4120+ */
4121+ bounds_check_type = BPF_WRITE;
4122+ clobber = true;
4123+ } else {
4124+ bounds_check_type = BPF_READ;
4125+ }
4126+ err = check_stack_access_within_bounds(env, regno, off, access_size,
4127+ type, bounds_check_type);
4128+ if (err)
4129+ return err;
4130+
37324131
37334132 if (tnum_is_const(reg->var_off)) {
3734− min_off = max_off = reg->var_off.value + reg->off;
3735− err = __check_stack_boundary(env, regno, min_off, access_size,
3736− zero_size_allowed);
3737− if (err)
3738− return err;
4133+ min_off = max_off = reg->var_off.value + off;
37394134 } else {
37404135 /* Variable offset is prohibited for unprivileged mode for
37414136 * simplicity since it requires corresponding support in
@@ -3746,8 +4141,8 @@ static int check_stack_boundary(struct bpf_verifier_env *env, int regno, · 4 unmodified lines
37464141 char tn_buf[48];
37474142
37484143 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3749− verbose(env, "R%d indirect variable offset stack access prohibited for !root, var_off=%s\n",
3750− regno, tn_buf);
4144+ verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n",
4145+ regno, err_extra, tn_buf);
37514146 return -EACCES;
37524147 }
37534148 /* Only initialized buffer on stack is allowed to be accessed
@@ -3759,28 +4154,8 @@ static int check_stack_boundary(struct bpf_verifier_env *env, int regno, · 5 unmodified lines
37594154 if (meta && meta->raw_mode)
37604155 meta = NULL;
37614156
3762− if (reg->smax_value >= BPF_MAX_VAR_OFF ||
3763− reg->smax_value <= -BPF_MAX_VAR_OFF) {
3764− verbose(env, "R%d unbounded indirect variable offset stack access\n",
3765− regno);
3766− return -EACCES;
3767− }
3768− min_off = reg->smin_value + reg->off;
3769− max_off = reg->smax_value + reg->off;
3770− err = __check_stack_boundary(env, regno, min_off, access_size,
3771− zero_size_allowed);
3772− if (err) {
3773− verbose(env, "R%d min value is outside of stack bound\n",
3774− regno);
3775− return err;
3776− }
3777− err = __check_stack_boundary(env, regno, max_off, access_size,
3778− zero_size_allowed);
3779− if (err) {
3780− verbose(env, "R%d max value is outside of stack bound\n",
3781− regno);
3782− return err;
3783− }
4157+ min_off = reg->smin_value + off;
4158+ max_off = reg->smax_value + off;
37844159 }
37854160
37864161 if (meta && meta->raw_mode) {
@@ -3800,8 +4175,10 @@ static int check_stack_boundary(struct bpf_verifier_env *env, int regno, · 13 unmodified lines
38004175 if (*stype == STACK_MISC)
38014176 goto mark;
38024177 if (*stype == STACK_ZERO) {
3803− /* helper can write anything into the stack */
3804− *stype = STACK_MISC;
4178+ if (clobber) {
4179+ /* helper can write anything into the stack */
4180+ *stype = STACK_MISC;
4181+ }
38054182 goto mark;
38064183 }
38074184
@@ -3812,22 +4189,24 @@ static int check_stack_boundary(struct bpf_verifier_env *env, int regno, · 4 unmodified lines
38124189 if (state->stack[spi].slot_type[0] == STACK_SPILL &&
38134190 (state->stack[spi].spilled_ptr.type == SCALAR_VALUE ||
38144191 env->allow_ptr_leaks)) {
3815− __mark_reg_unknown(env, &state->stack[spi].spilled_ptr);
3816− for (j = 0; j < BPF_REG_SIZE; j++)
3817− state->stack[spi].slot_type[j] = STACK_MISC;
4192+ if (clobber) {
4193+ __mark_reg_unknown(env, &state->stack[spi].spilled_ptr);
4194+ for (j = 0; j < BPF_REG_SIZE; j++)
4195+ state->stack[spi].slot_type[j] = STACK_MISC;
4196+ }
38184197 goto mark;
38194198 }
38204199
38214200 err:
38224201 if (tnum_is_const(reg->var_off)) {
3823− verbose(env, "invalid indirect read from stack off %d+%d size %d\n",
3824− min_off, i - min_off, access_size);
4202+ verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n",
4203+ err_extra, regno, min_off, i - min_off, access_size);
38254204 } else {
38264205 char tn_buf[48];
38274206
38284207 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3829− verbose(env, "invalid indirect read from stack var_off %s+%d size %d\n",
3830− tn_buf, i - min_off, access_size);
4208+ verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n",
4209+ err_extra, regno, tn_buf, i - min_off, access_size);
38314210 }
38324211 return -EACCES;
38334212 mark:
@@ -3876,8 +4255,10 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, · 42 unmodified lines
38764255 "rdwr",
38774256 &env->prog->aux->max_rdwr_access);
38784257 case PTR_TO_STACK:
3879− return check_stack_boundary(env, regno, access_size,
3880− zero_size_allowed, meta);
4258+ return check_stack_range_initialized(
4259+ env,
4260+ regno, reg->off, access_size,
4261+ zero_size_allowed, ACCESS_HELPER, meta);
38814262 default: /* scalar_value or invalid ptr */
38824263 /* Allow zero-byte read from NULL, regardless of pointer type */
38834264 if (zero_size_allowed && access_size == 0 &&
@@ -3891,6 +4272,29 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, · 7 unmodified lines
38914272 }
38924273 }
38934274
4275+int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
4276+ u32 regno, u32 mem_size)
4277+{
4278+ if (register_is_null(reg))
4279+ return 0;
4280+
4281+ if (reg_type_may_be_null(reg->type)) {
4282+ /* Assuming that the register contains a value check if the memory
4283+ * access is safe. Temporarily save and restore the register's state as
4284+ * the conversion shouldn't be visible to a caller.
4285+ */
4286+ const struct bpf_reg_state saved_reg = *reg;
4287+ int rv;
4288+
4289+ mark_ptr_not_null_reg(reg);
4290+ rv = check_helper_mem_access(env, regno, mem_size, true, NULL);
4291+ *reg = saved_reg;
4292+ return rv;
4293+ }
4294+
4295+ return check_helper_mem_access(env, regno, mem_size, true, NULL);
4296+}
4297+
38944298 /* Implementation details:
38954299 * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL
38964300 * Two bpf_map_lookups (even with the same key) will have different reg->id.
@@ -4875,8 +5279,9 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 978 unmodified lines
48755279 subprog);
48765280 clear_caller_saved_regs(env, caller->regs);
48775281
4878− /* All global functions return SCALAR_VALUE */
5282+ /* All global functions return a 64-bit SCALAR_VALUE */
48795283 mark_reg_unknown(env, caller->regs, BPF_REG_0);
5284+ caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
48805285
48815286 /* continue with next insn after call */
48825287 return 0;
@@ -5541,6 +5946,41 @@ do_sim: · 658 unmodified lines
55415946 return !ret ? -EFAULT : 0;
55425947 }
55435948
5949+/* check that stack access falls within stack limits and that 'reg' doesn't
5950+ * have a variable offset.
5951+ *
5952+ * Variable offset is prohibited for unprivileged mode for simplicity since it
5953+ * requires corresponding support in Spectre masking for stack ALU. See also
5954+ * retrieve_ptr_limit().
5955+ *
5956+ *
5957+ * 'off' includes 'reg->off'.
5958+ */
5959+static int check_stack_access_for_ptr_arithmetic(
5960+ struct bpf_verifier_env *env,
5961+ int regno,
5962+ const struct bpf_reg_state *reg,
5963+ int off)
5964+{
5965+ if (!tnum_is_const(reg->var_off)) {
5966+ char tn_buf[48];
5967+
5968+ tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
5969+ verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n",
5970+ regno, tn_buf, off);
5971+ return -EACCES;
5972+ }
5973+
5974+ if (off >= 0 || off < -MAX_BPF_STACK) {
5975+ verbose(env, "R%d stack pointer arithmetic goes out of range, "
5976+ "prohibited for !root; off=%d\n", regno, off);
5977+ return -EACCES;
5978+ }
5979+
5980+ return 0;
5981+}
5982+
5983+
55445984 /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off.
55455985 * Caller should also handle BPF_MOV case separately.
55465986 * If we return -EACCES, caller may want to try again treating pointer as a
@@ -5784,10 +6224,9 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, · 237 unmodified lines
57846224 "prohibited for !root\n", dst);
57856225 return -EACCES;
57866226 } else if (dst_reg->type == PTR_TO_STACK &&
5787− check_stack_access(env, dst_reg, dst_reg->off +
5788− dst_reg->var_off.value, 1)) {
5789− verbose(env, "R%d stack pointer arithmetic goes out of range, "
5790− "prohibited for !root\n", dst);
6227+ check_stack_access_for_ptr_arithmetic(
6228+ env, dst, dst_reg, dst_reg->off +
6229+ dst_reg->var_off.value)) {
57916230 return -EACCES;
57926231 }
57936232 }
@@ -6266,7 +6705,7 @@ static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, · 472 unmodified lines
62666705 * 3) the signed bounds cross zero, so they tell us nothing
62676706 * about the result
62686707 * If the value in dst_reg is known nonnegative, then again the
6269− * unsigned bounts capture the signed bounds.
6708+ * unsigned bounds capture the signed bounds.
62706709 * Thus, in all cases it suffices to blow away our signed bounds
62716710 * and rely on inferring new ones from the unsigned bounds and
62726711 * var_off of the result.
@@ -6297,7 +6736,7 @@ static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, · 24 unmodified lines
62976736 * 3) the signed bounds cross zero, so they tell us nothing
62986737 * about the result
62996738 * If the value in dst_reg is known nonnegative, then again the
6300− * unsigned bounts capture the signed bounds.
6739+ * unsigned bounds capture the signed bounds.
63016740 * Thus, in all cases it suffices to blow away our signed bounds
63026741 * and rely on inferring new ones from the unsigned bounds and
63036742 * var_off of the result.
@@ -7367,43 +7806,19 @@ static void mark_ptr_or_null_reg(struct bpf_func_state *state, · 1063 unmodified lines
73677806 }
73687807 if (is_null) {
73697808 reg->type = SCALAR_VALUE;
7370− } else if (reg->type == PTR_TO_MAP_VALUE_OR_NULL) {
7371− const struct bpf_map *map = reg->map_ptr;
7372−
7373− if (map->inner_map_meta) {
7374− reg->type = CONST_PTR_TO_MAP;
7375− reg->map_ptr = map->inner_map_meta;
7376− } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) {
7377− reg->type = PTR_TO_XDP_SOCK;
7378− } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP ||
7379− map->map_type == BPF_MAP_TYPE_SOCKHASH) {
7380− reg->type = PTR_TO_SOCKET;
7381− } else {
7382− reg->type = PTR_TO_MAP_VALUE;
7383− }
7384− } else if (reg->type == PTR_TO_SOCKET_OR_NULL) {
7385− reg->type = PTR_TO_SOCKET;
7386− } else if (reg->type == PTR_TO_SOCK_COMMON_OR_NULL) {
7387− reg->type = PTR_TO_SOCK_COMMON;
7388− } else if (reg->type == PTR_TO_TCP_SOCK_OR_NULL) {
7389− reg->type = PTR_TO_TCP_SOCK;
7390− } else if (reg->type == PTR_TO_BTF_ID_OR_NULL) {
7391− reg->type = PTR_TO_BTF_ID;
7392− } else if (reg->type == PTR_TO_MEM_OR_NULL) {
7393− reg->type = PTR_TO_MEM;
7394− } else if (reg->type == PTR_TO_RDONLY_BUF_OR_NULL) {
7395− reg->type = PTR_TO_RDONLY_BUF;
7396− } else if (reg->type == PTR_TO_RDWR_BUF_OR_NULL) {
7397− reg->type = PTR_TO_RDWR_BUF;
7398− }
7399− if (is_null) {
74007809 /* We don't need id and ref_obj_id from this point
74017810 * onwards anymore, thus we should better reset it,
74027811 * so that state pruning has chances to take effect.
74037812 */
74047813 reg->id = 0;
74057814 reg->ref_obj_id = 0;
7406− } else if (!reg_may_point_to_spin_lock(reg)) {
7815+
7816+ return;
7817+ }
7818+
7819+ mark_ptr_not_null_reg(reg);
7820+
7821+ if (!reg_may_point_to_spin_lock(reg)) {
74077822 /* For not-NULL ptr, reg->ref_obj_id will be reset
74087823 * in release_reg_references().
74097824 *
@@ -7986,6 +8401,9 @@ static int check_return_code(struct bpf_verifier_env *env) · 576 unmodified lines
79868401 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME ||
79878402 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME)
79888403 range = tnum_range(1, 1);
8404+ if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND ||
8405+ env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND)
8406+ range = tnum_range(0, 3);
79898407 break;
79908408 case BPF_PROG_TYPE_CGROUP_SKB:
79918409 if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) {
@@ -10015,15 +10433,22 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, · 2023 unmodified lines
1001510433 case BPF_MAP_TYPE_HASH:
1001610434 case BPF_MAP_TYPE_LRU_HASH:
1001710435 case BPF_MAP_TYPE_ARRAY:
10436+ case BPF_MAP_TYPE_PERCPU_HASH:
10437+ case BPF_MAP_TYPE_PERCPU_ARRAY:
10438+ case BPF_MAP_TYPE_LRU_PERCPU_HASH:
10439+ case BPF_MAP_TYPE_ARRAY_OF_MAPS:
10440+ case BPF_MAP_TYPE_HASH_OF_MAPS:
1001810441 if (!is_preallocated_map(map)) {
1001910442 verbose(env,
10020− "Sleepable programs can only use preallocated hash maps\n");
10443+ "Sleepable programs can only use preallocated maps\n");
1002110444 return -EINVAL;
1002210445 }
1002310446 break;
10447+ case BPF_MAP_TYPE_RINGBUF:
10448+ break;
1002410449 default:
1002510450 verbose(env,
10026− "Sleepable programs can only use array and hash maps\n");
10451+ "Sleepable programs can only use array, hash, and ringbuf maps\n");
1002710452 return -EINVAL;
1002810453 }
1002910454
@@ -10581,6 +11006,7 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, · 551 unmodified lines
1058111006 for (i = 0; i < len; i++) {
1058211007 int adj_idx = i + delta;
1058311008 struct bpf_insn insn;
11009+ u8 load_reg;
1058411010
1058511011 insn = insns[adj_idx];
1058611012 if (!aux[adj_idx].zext_dst) {
@@ -10623,9 +11049,27 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, · 36 unmodified lines
1062311049 if (!bpf_jit_needs_zext())
1062411050 continue;
1062511051
11052+ /* zext_dst means that we want to zero-extend whatever register
11053+ * the insn defines, which is dst_reg most of the time, with
11054+ * the notable exception of BPF_STX + BPF_ATOMIC + BPF_FETCH.
11055+ */
11056+ if (BPF_CLASS(insn.code) == BPF_STX &&
11057+ BPF_MODE(insn.code) == BPF_ATOMIC) {
11058+ /* BPF_STX + BPF_ATOMIC insns without BPF_FETCH do not
11059+ * define any registers, therefore zext_dst cannot be
11060+ * set.
11061+ */
11062+ if (WARN_ON(!(insn.imm & BPF_FETCH)))
11063+ return -EINVAL;
11064+ load_reg = insn.imm == BPF_CMPXCHG ? BPF_REG_0
11065+ : insn.src_reg;
11066+ } else {
11067+ load_reg = insn.dst_reg;
11068+ }
11069+
1062611070 zext_patch[0] = insn;
10627− zext_patch[1].dst_reg = insn.dst_reg;
10628− zext_patch[1].src_reg = insn.dst_reg;
11071+ zext_patch[1].dst_reg = load_reg;
11072+ zext_patch[1].src_reg = load_reg;
1062911073 patch = zext_patch;
1063011074 patch_len = 2;
1063111075 apply_patch_buffer:
@@ -10841,8 +11285,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 209 unmodified lines
1084111285 return 0;
1084211286
1084311287 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
10844− if (insn->code != (BPF_JMP | BPF_CALL) ||
10845− insn->src_reg != BPF_PSEUDO_CALL)
11288+ if (!bpf_pseudo_call(insn))
1084611289 continue;
1084711290 /* Upon error here we cannot fall back to interpreter but
1084811291 * need a hard reject of the program. Thus -EFAULT is
@@ -10883,7 +11326,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 34 unmodified lines
1088311326 /* BPF_PROG_RUN doesn't call subprogs directly,
1088411327 * hence main prog stats include the runtime of subprogs.
1088511328 * subprogs don't have IDs and not reachable via prog_get_next_id
10886− * func[i]->aux->stats will never be accessed and stays NULL
11329+ * func[i]->stats will never be accessed and stays NULL
1088711330 */
1088811331 func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER);
1088911332 if (!func[i])
@@ -10971,8 +11414,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 81 unmodified lines
1097111414 for (i = 0; i < env->subprog_cnt; i++) {
1097211415 insn = func[i]->insnsi;
1097311416 for (j = 0; j < func[i]->len; j++, insn++) {
10974− if (insn->code != (BPF_JMP | BPF_CALL) ||
10975− insn->src_reg != BPF_PSEUDO_CALL)
11417+ if (!bpf_pseudo_call(insn))
1097611418 continue;
1097711419 subprog = insn->off;
1097811420 insn->imm = BPF_CAST_CALL(func[subprog]->bpf_func) -
@@ -11017,8 +11459,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 38 unmodified lines
1101711459 * later look the same as if they were interpreted only.
1101811460 */
1101911461 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
11020− if (insn->code != (BPF_JMP | BPF_CALL) ||
11021− insn->src_reg != BPF_PSEUDO_CALL)
11462+ if (!bpf_pseudo_call(insn))
1102211463 continue;
1102311464 insn->off = env->insn_aux_data[i].call_imm;
1102411465 subprog = find_subprog(env, i + insn->off + 1);
@@ -11047,8 +11488,7 @@ out_undo_insn: · 22 unmodified lines
1104711488 /* cleanup main prog to be interpreted */
1104811489 prog->jit_requested = 0;
1104911490 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
11050− if (insn->code != (BPF_JMP | BPF_CALL) ||
11051− insn->src_reg != BPF_PSEUDO_CALL)
11491+ if (!bpf_pseudo_call(insn))
1105211492 continue;
1105311493 insn->off = 0;
1105411494 insn->imm = env->insn_aux_data[i].call_imm;
@@ -11083,8 +11523,7 @@ static int fixup_call_args(struct bpf_verifier_env *env) · 28 unmodified lines
1108311523 return -EINVAL;
1108411524 }
1108511525 for (i = 0; i < prog->len; i++, insn++) {
11086− if (insn->code != (BPF_JMP | BPF_CALL) ||
11087− insn->src_reg != BPF_PSEUDO_CALL)
11526+ if (!bpf_pseudo_call(insn))
1108811527 continue;
1108911528 depth = get_callee_stack_depth(env, insn, i);
1109011529 if (depth < 0)
@@ -11547,6 +11986,13 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) · 456 unmodified lines
1154711986 mark_reg_known_zero(env, regs, i);
1154811987 else if (regs[i].type == SCALAR_VALUE)
1154911988 mark_reg_unknown(env, regs, i);
11989+ else if (regs[i].type == PTR_TO_MEM_OR_NULL) {
11990+ const u32 mem_size = regs[i].mem_size;
11991+
11992+ mark_reg_known_zero(env, regs, i);
11993+ regs[i].mem_size = mem_size;
11994+ regs[i].id = ++env->id_gen;
11995+ }
1155011996 }
1155111997 } else {
1155211998 /* 1st arg to a function */
@@ -12125,6 +12571,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, · 572 unmodified lines
1212512571 env->strict_alignment = false;
1212612572
1212712573 env->allow_ptr_leaks = bpf_allow_ptr_leaks();
12574+ env->allow_uninit_stack = bpf_allow_uninit_stack();
1212812575 env->allow_ptr_to_map_access = bpf_allow_ptr_to_map_access();
1212912576 env->bypass_spec_v1 = bpf_bypass_spec_v1();
1213012577 env->bypass_spec_v4 = bpf_bypass_spec_v4();
kernel/trace/bpf_trace.c+6−0
@@ -1188,6 +1188,10 @@ BTF_SET_END(btf_allowlist_d_path) · 1187 unmodified lines
11881188
11891189 static bool bpf_d_path_allowed(const struct bpf_prog *prog)
11901190 {
1191+ if (prog->type == BPF_PROG_TYPE_TRACING &&
1192+ prog->expected_attach_type == BPF_TRACE_ITER)
1193+ return true;
1194+
11911195 if (prog->type == BPF_PROG_TYPE_LSM)
11921196 return bpf_lsm_is_sleepable_hook(prog->aux->attach_btf_id);
11931197
@@ -1757,6 +1761,8 @@ tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 563 unmodified lines
17571761 return &bpf_sk_storage_delete_tracing_proto;
17581762 case BPF_FUNC_sock_from_file:
17591763 return &bpf_sock_from_file_proto;
1764+ case BPF_FUNC_get_socket_cookie:
1765+ return &bpf_get_socket_ptr_cookie_proto;
17601766 #endif
17611767 case BPF_FUNC_seq_printf:
17621768 return prog->expected_attach_type == BPF_TRACE_ITER ?

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t