History

Merge tag 'bpf-next-6.16' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Pull bpf updates from Alexei Starovoitov: - Fix and improve BTF deduplication of identical BTF types (Alan Maguire and Andrii Nakryiko) - Support up to 12 arguments in BPF trampoline on arm64 (Xu Kuohai and Alexis Lothoré) - Support load-acquire and store-release instructions in BPF JIT on riscv64 (Andrea Parri) - Fix uninitialized values in BPF_{CORE,PROBE}_READ macros (Anton Protopopov) - Streamline allowed helpers across program types (Feng Yang) - Support atomic update for hashtab of BPF maps (Hou Tao) - Implement json output for BPF helpers (Ihor Solodrai) - Several s390 JIT fixes (Ilya Leoshkevich) - Various sockmap fixes (Jiayuan Chen) - Support mmap of vmlinux BTF data (Lorenz Bauer) - Support BPF rbtree traversal and list peeking (Martin KaFai Lau) - Tests for sockmap/sockhash redirection (Michal Luczaj) - Introduce kfuncs for memory reads into dynptrs (Mykyta Yatsenko) - Add support for dma-buf iterators in BPF (T.J. Mercier) - The verifier support for __bpf_trap() (Yonghong Song) * tag 'bpf-next-6.16' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next: (135 commits) bpf, arm64: Remove unused-but-set function and variable. selftests/bpf: Add tests with stack ptr register in conditional jmp bpf: Do not include stack ptr register in precision backtracking bookkeeping selftests/bpf: enable many-args tests for arm64 bpf, arm64: Support up to 12 function arguments bpf: Check rcu_read_lock_trace_held() in bpf_map_lookup_percpu_elem() bpf: Avoid __bpf_prog_ret0_warn when jit fails bpftool: Add support for custom BTF path in prog load/loadall selftests/bpf: Add unit tests with __bpf_trap() kfunc bpf: Warn with __bpf_trap() kfunc maybe due to uninitialized variable bpf: Remove special_kfunc_set from verifier selftests/bpf: Add test for open coded dmabuf_iter selftests/bpf: Add test for dmabuf_iter bpf: Add open coded dmabuf iterator bpf: Add dmabuf iterator dma-buf: Rename debugfs symbols bpf: Fix error return value in bpf_copy_from_user_dynptr libbpf: Use mmap to parse vmlinux BTF from sysfs selftests: bpf: Add a test for mmapable vmlinux BTF btf: Allow mmap of vmlinux btf ...

Linus Torvalds committed 1 year ago

Commit 90b83ef2 parents 1b98f35 c5cebb2108 files · +5801 −1713Browse files

Showing the changes to kernel/trace (2 of 108 files). Show all

Unified · Split

2 files
  1. kernel/trace/bpf_trace.c +211 −110
  2. kernel/trace/trace_uprobe.c +1 −1
kernel/trace/bpf_trace.c+211−110
@@ -572,7 +572,7 @@ BPF_CALL_2(bpf_perf_event_read, struct bpf_map *, map, u64, flags) · 571 unmodified lines
572572 return value;
573573 }
574574
575−static const struct bpf_func_proto bpf_perf_event_read_proto = {
575+const struct bpf_func_proto bpf_perf_event_read_proto = {
576576 .func = bpf_perf_event_read,
577577 .gpl_only = true,
578578 .ret_type = RET_INTEGER,
@@ -882,7 +882,7 @@ BPF_CALL_1(bpf_send_signal, u32, sig) · 303 unmodified lines
882882 return bpf_send_signal_common(sig, PIDTYPE_TGID, NULL, 0);
883883 }
884884
885−static const struct bpf_func_proto bpf_send_signal_proto = {
885+const struct bpf_func_proto bpf_send_signal_proto = {
886886 .func = bpf_send_signal,
887887 .gpl_only = false,
888888 .ret_type = RET_INTEGER,
@@ -894,7 +894,7 @@ BPF_CALL_1(bpf_send_signal_thread, u32, sig) · 5 unmodified lines
894894 return bpf_send_signal_common(sig, PIDTYPE_PID, NULL, 0);
895895 }
896896
897−static const struct bpf_func_proto bpf_send_signal_thread_proto = {
897+const struct bpf_func_proto bpf_send_signal_thread_proto = {
898898 .func = bpf_send_signal_thread,
899899 .gpl_only = false,
900900 .ret_type = RET_INTEGER,
@@ -1185,7 +1185,7 @@ BPF_CALL_3(bpf_get_branch_snapshot, void *, buf, u32, size, u64, flags) · 284 unmodified lines
11851185 return entry_cnt * br_entry_size;
11861186 }
11871187
1188−static const struct bpf_func_proto bpf_get_branch_snapshot_proto = {
1188+const struct bpf_func_proto bpf_get_branch_snapshot_proto = {
11891189 .func = bpf_get_branch_snapshot,
11901190 .gpl_only = true,
11911191 .ret_type = RET_INTEGER,
@@ -1430,56 +1430,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 238 unmodified lines
14301430 const struct bpf_func_proto *func_proto;
14311431
14321432 switch (func_id) {
1433− case BPF_FUNC_map_lookup_elem:
1434− return &bpf_map_lookup_elem_proto;
1435− case BPF_FUNC_map_update_elem:
1436− return &bpf_map_update_elem_proto;
1437− case BPF_FUNC_map_delete_elem:
1438− return &bpf_map_delete_elem_proto;
1439− case BPF_FUNC_map_push_elem:
1440− return &bpf_map_push_elem_proto;
1441− case BPF_FUNC_map_pop_elem:
1442− return &bpf_map_pop_elem_proto;
1443− case BPF_FUNC_map_peek_elem:
1444− return &bpf_map_peek_elem_proto;
1445− case BPF_FUNC_map_lookup_percpu_elem:
1446− return &bpf_map_lookup_percpu_elem_proto;
1447− case BPF_FUNC_ktime_get_ns:
1448− return &bpf_ktime_get_ns_proto;
1449− case BPF_FUNC_ktime_get_boot_ns:
1450− return &bpf_ktime_get_boot_ns_proto;
1451− case BPF_FUNC_tail_call:
1452− return &bpf_tail_call_proto;
1453− case BPF_FUNC_get_current_task:
1454− return &bpf_get_current_task_proto;
1455− case BPF_FUNC_get_current_task_btf:
1456− return &bpf_get_current_task_btf_proto;
1457− case BPF_FUNC_task_pt_regs:
1458− return &bpf_task_pt_regs_proto;
1459− case BPF_FUNC_get_current_uid_gid:
1460− return &bpf_get_current_uid_gid_proto;
1461− case BPF_FUNC_get_current_comm:
1462− return &bpf_get_current_comm_proto;
1463− case BPF_FUNC_trace_printk:
1464− return bpf_get_trace_printk_proto();
14651433 case BPF_FUNC_get_smp_processor_id:
14661434 return &bpf_get_smp_processor_id_proto;
1467− case BPF_FUNC_get_numa_node_id:
1468− return &bpf_get_numa_node_id_proto;
1469− case BPF_FUNC_perf_event_read:
1470− return &bpf_perf_event_read_proto;
1471− case BPF_FUNC_get_prandom_u32:
1472− return &bpf_get_prandom_u32_proto;
1473− case BPF_FUNC_probe_read_user:
1474− return &bpf_probe_read_user_proto;
1475− case BPF_FUNC_probe_read_kernel:
1476− return security_locked_down(LOCKDOWN_BPF_READ_KERNEL) < 0 ?
1477− NULL : &bpf_probe_read_kernel_proto;
1478− case BPF_FUNC_probe_read_user_str:
1479− return &bpf_probe_read_user_str_proto;
1480− case BPF_FUNC_probe_read_kernel_str:
1481− return security_locked_down(LOCKDOWN_BPF_READ_KERNEL) < 0 ?
1482− NULL : &bpf_probe_read_kernel_str_proto;
14831435 #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
14841436 case BPF_FUNC_probe_read:
14851437 return security_locked_down(LOCKDOWN_BPF_READ_KERNEL) < 0 ?
@@ -1488,65 +1440,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 2 unmodified lines
14881440 return security_locked_down(LOCKDOWN_BPF_READ_KERNEL) < 0 ?
14891441 NULL : &bpf_probe_read_compat_str_proto;
14901442 #endif
1491−#ifdef CONFIG_CGROUPS
1492− case BPF_FUNC_cgrp_storage_get:
1493− return &bpf_cgrp_storage_get_proto;
1494− case BPF_FUNC_cgrp_storage_delete:
1495− return &bpf_cgrp_storage_delete_proto;
1496− case BPF_FUNC_current_task_under_cgroup:
1497− return &bpf_current_task_under_cgroup_proto;
1498−#endif
1499− case BPF_FUNC_send_signal:
1500− return &bpf_send_signal_proto;
1501− case BPF_FUNC_send_signal_thread:
1502− return &bpf_send_signal_thread_proto;
1503− case BPF_FUNC_perf_event_read_value:
1504− return &bpf_perf_event_read_value_proto;
1505− case BPF_FUNC_ringbuf_output:
1506− return &bpf_ringbuf_output_proto;
1507− case BPF_FUNC_ringbuf_reserve:
1508− return &bpf_ringbuf_reserve_proto;
1509− case BPF_FUNC_ringbuf_submit:
1510− return &bpf_ringbuf_submit_proto;
1511− case BPF_FUNC_ringbuf_discard:
1512− return &bpf_ringbuf_discard_proto;
1513− case BPF_FUNC_ringbuf_query:
1514− return &bpf_ringbuf_query_proto;
1515− case BPF_FUNC_jiffies64:
1516− return &bpf_jiffies64_proto;
1517− case BPF_FUNC_get_task_stack:
1518− return prog->sleepable ? &bpf_get_task_stack_sleepable_proto
1519− : &bpf_get_task_stack_proto;
1520− case BPF_FUNC_copy_from_user:
1521− return &bpf_copy_from_user_proto;
1522− case BPF_FUNC_copy_from_user_task:
1523− return &bpf_copy_from_user_task_proto;
1524− case BPF_FUNC_snprintf_btf:
1525− return &bpf_snprintf_btf_proto;
1526− case BPF_FUNC_per_cpu_ptr:
1527− return &bpf_per_cpu_ptr_proto;
1528− case BPF_FUNC_this_cpu_ptr:
1529− return &bpf_this_cpu_ptr_proto;
1530− case BPF_FUNC_task_storage_get:
1531− if (bpf_prog_check_recur(prog))
1532− return &bpf_task_storage_get_recur_proto;
1533− return &bpf_task_storage_get_proto;
1534− case BPF_FUNC_task_storage_delete:
1535− if (bpf_prog_check_recur(prog))
1536− return &bpf_task_storage_delete_recur_proto;
1537− return &bpf_task_storage_delete_proto;
1538− case BPF_FUNC_for_each_map_elem:
1539− return &bpf_for_each_map_elem_proto;
1540− case BPF_FUNC_snprintf:
1541− return &bpf_snprintf_proto;
15421443 case BPF_FUNC_get_func_ip:
15431444 return &bpf_get_func_ip_proto_tracing;
1544− case BPF_FUNC_get_branch_snapshot:
1545− return &bpf_get_branch_snapshot_proto;
1546− case BPF_FUNC_find_vma:
1547− return &bpf_find_vma_proto;
1548− case BPF_FUNC_trace_vprintk:
1549− return bpf_get_trace_vprintk_proto();
15501445 default:
15511446 break;
15521447 }
@@ -1858,7 +1753,7 @@ static struct pt_regs *get_bpf_raw_tp_regs(void) · 305 unmodified lines
18581753 struct bpf_raw_tp_regs *tp_regs = this_cpu_ptr(&bpf_raw_tp_regs);
18591754 int nest_level = this_cpu_inc_return(bpf_raw_tp_nest_level);
18601755
1861− if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(tp_regs->regs))) {
1756+ if (nest_level > ARRAY_SIZE(tp_regs->regs)) {
18621757 this_cpu_dec(bpf_raw_tp_nest_level);
18631758 return ERR_PTR(-EBUSY);
18641759 }
@@ -2987,6 +2882,9 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr · 1122 unmodified lines
29872882 if (sizeof(u64) != sizeof(void *))
29882883 return -EOPNOTSUPP;
29892884
2885+ if (attr->link_create.flags)
2886+ return -EINVAL;
2887+
29902888 if (!is_kprobe_multi(prog))
29912889 return -EINVAL;
29922890
@@ -3376,6 +3274,9 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr · 383 unmodified lines
33763274 if (sizeof(u64) != sizeof(void *))
33773275 return -EOPNOTSUPP;
33783276
3277+ if (attr->link_create.flags)
3278+ return -EINVAL;
3279+
33793280 if (!is_uprobe_multi(prog))
33803281 return -EINVAL;
33813282
@@ -3417,7 +3318,9 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr · 35 unmodified lines
34173318 }
34183319
34193320 if (pid) {
3321+ rcu_read_lock();
34203322 task = get_pid_task(find_vpid(pid), PIDTYPE_TGID);
3323+ rcu_read_unlock();
34213324 if (!task) {
34223325 err = -ESRCH;
34233326 goto error_path_put;
@@ -3565,6 +3468,146 @@ static int __init bpf_kprobe_multi_kfuncs_init(void) · 141 unmodified lines
35653468
35663469 late_initcall(bpf_kprobe_multi_kfuncs_init);
35673470
3471+typedef int (*copy_fn_t)(void *dst, const void *src, u32 size, struct task_struct *tsk);
3472+
3473+/*
3474+ * The __always_inline is to make sure the compiler doesn't
3475+ * generate indirect calls into callbacks, which is expensive,
3476+ * on some kernel configurations. This allows compiler to put
3477+ * direct calls into all the specific callback implementations
3478+ * (copy_user_data_sleepable, copy_user_data_nofault, and so on)
3479+ */
3480+static __always_inline int __bpf_dynptr_copy_str(struct bpf_dynptr *dptr, u32 doff, u32 size,
3481+ const void *unsafe_src,
3482+ copy_fn_t str_copy_fn,
3483+ struct task_struct *tsk)
3484+{
3485+ struct bpf_dynptr_kern *dst;
3486+ u32 chunk_sz, off;
3487+ void *dst_slice;
3488+ int cnt, err;
3489+ char buf[256];
3490+
3491+ dst_slice = bpf_dynptr_slice_rdwr(dptr, doff, NULL, size);
3492+ if (likely(dst_slice))
3493+ return str_copy_fn(dst_slice, unsafe_src, size, tsk);
3494+
3495+ dst = (struct bpf_dynptr_kern *)dptr;
3496+ if (bpf_dynptr_check_off_len(dst, doff, size))
3497+ return -E2BIG;
3498+
3499+ for (off = 0; off < size; off += chunk_sz - 1) {
3500+ chunk_sz = min_t(u32, sizeof(buf), size - off);
3501+ /* Expect str_copy_fn to return count of copied bytes, including
3502+ * zero terminator. Next iteration increment off by chunk_sz - 1 to
3503+ * overwrite NUL.
3504+ */
3505+ cnt = str_copy_fn(buf, unsafe_src + off, chunk_sz, tsk);
3506+ if (cnt < 0)
3507+ return cnt;
3508+ err = __bpf_dynptr_write(dst, doff + off, buf, cnt, 0);
3509+ if (err)
3510+ return err;
3511+ if (cnt < chunk_sz || chunk_sz == 1) /* we are done */
3512+ return off + cnt;
3513+ }
3514+ return off;
3515+}
3516+
3517+static __always_inline int __bpf_dynptr_copy(const struct bpf_dynptr *dptr, u32 doff,
3518+ u32 size, const void *unsafe_src,
3519+ copy_fn_t copy_fn, struct task_struct *tsk)
3520+{
3521+ struct bpf_dynptr_kern *dst;
3522+ void *dst_slice;
3523+ char buf[256];
3524+ u32 off, chunk_sz;
3525+ int err;
3526+
3527+ dst_slice = bpf_dynptr_slice_rdwr(dptr, doff, NULL, size);
3528+ if (likely(dst_slice))
3529+ return copy_fn(dst_slice, unsafe_src, size, tsk);
3530+
3531+ dst = (struct bpf_dynptr_kern *)dptr;
3532+ if (bpf_dynptr_check_off_len(dst, doff, size))
3533+ return -E2BIG;
3534+
3535+ for (off = 0; off < size; off += chunk_sz) {
3536+ chunk_sz = min_t(u32, sizeof(buf), size - off);
3537+ err = copy_fn(buf, unsafe_src + off, chunk_sz, tsk);
3538+ if (err)
3539+ return err;
3540+ err = __bpf_dynptr_write(dst, doff + off, buf, chunk_sz, 0);
3541+ if (err)
3542+ return err;
3543+ }
3544+ return 0;
3545+}
3546+
3547+static __always_inline int copy_user_data_nofault(void *dst, const void *unsafe_src,
3548+ u32 size, struct task_struct *tsk)
3549+{
3550+ return copy_from_user_nofault(dst, (const void __user *)unsafe_src, size);
3551+}
3552+
3553+static __always_inline int copy_user_data_sleepable(void *dst, const void *unsafe_src,
3554+ u32 size, struct task_struct *tsk)
3555+{
3556+ int ret;
3557+
3558+ if (!tsk) { /* Read from the current task */
3559+ ret = copy_from_user(dst, (const void __user *)unsafe_src, size);
3560+ if (ret)
3561+ return -EFAULT;
3562+ return 0;
3563+ }
3564+
3565+ ret = access_process_vm(tsk, (unsigned long)unsafe_src, dst, size, 0);
3566+ if (ret != size)
3567+ return -EFAULT;
3568+ return 0;
3569+}
3570+
3571+static __always_inline int copy_kernel_data_nofault(void *dst, const void *unsafe_src,
3572+ u32 size, struct task_struct *tsk)
3573+{
3574+ return copy_from_kernel_nofault(dst, unsafe_src, size);
3575+}
3576+
3577+static __always_inline int copy_user_str_nofault(void *dst, const void *unsafe_src,
3578+ u32 size, struct task_struct *tsk)
3579+{
3580+ return strncpy_from_user_nofault(dst, (const void __user *)unsafe_src, size);
3581+}
3582+
3583+static __always_inline int copy_user_str_sleepable(void *dst, const void *unsafe_src,
3584+ u32 size, struct task_struct *tsk)
3585+{
3586+ int ret;
3587+
3588+ if (unlikely(size == 0))
3589+ return 0;
3590+
3591+ if (tsk) {
3592+ ret = copy_remote_vm_str(tsk, (unsigned long)unsafe_src, dst, size, 0);
3593+ } else {
3594+ ret = strncpy_from_user(dst, (const void __user *)unsafe_src, size - 1);
3595+ /* strncpy_from_user does not guarantee NUL termination */
3596+ if (ret >= 0)
3597+ ((char *)dst)[ret] = '\0';
3598+ }
3599+
3600+ if (ret < 0)
3601+ return ret;
3602+ return ret + 1;
3603+}
3604+
3605+static __always_inline int copy_kernel_str_nofault(void *dst, const void *unsafe_src,
3606+ u32 size, struct task_struct *tsk)
3607+{
3608+ return strncpy_from_kernel_nofault(dst, unsafe_src, size);
3609+}
3610+
35683611 __bpf_kfunc_start_defs();
35693612
35703613 __bpf_kfunc int bpf_send_signal_task(struct task_struct *task, int sig, enum pid_type type,
@@ -3576,4 +3619,62 @@ __bpf_kfunc int bpf_send_signal_task(struct task_struct *task, int sig, enum pid · 5 unmodified lines
35763619 return bpf_send_signal_common(sig, type, task, value);
35773620 }
35783621
3622+__bpf_kfunc int bpf_probe_read_user_dynptr(struct bpf_dynptr *dptr, u32 off,
3623+ u32 size, const void __user *unsafe_ptr__ign)
3624+{
3625+ return __bpf_dynptr_copy(dptr, off, size, (const void *)unsafe_ptr__ign,
3626+ copy_user_data_nofault, NULL);
3627+}
3628+
3629+__bpf_kfunc int bpf_probe_read_kernel_dynptr(struct bpf_dynptr *dptr, u32 off,
3630+ u32 size, const void *unsafe_ptr__ign)
3631+{
3632+ return __bpf_dynptr_copy(dptr, off, size, unsafe_ptr__ign,
3633+ copy_kernel_data_nofault, NULL);
3634+}
3635+
3636+__bpf_kfunc int bpf_probe_read_user_str_dynptr(struct bpf_dynptr *dptr, u32 off,
3637+ u32 size, const void __user *unsafe_ptr__ign)
3638+{
3639+ return __bpf_dynptr_copy_str(dptr, off, size, (const void *)unsafe_ptr__ign,
3640+ copy_user_str_nofault, NULL);
3641+}
3642+
3643+__bpf_kfunc int bpf_probe_read_kernel_str_dynptr(struct bpf_dynptr *dptr, u32 off,
3644+ u32 size, const void *unsafe_ptr__ign)
3645+{
3646+ return __bpf_dynptr_copy_str(dptr, off, size, unsafe_ptr__ign,
3647+ copy_kernel_str_nofault, NULL);
3648+}
3649+
3650+__bpf_kfunc int bpf_copy_from_user_dynptr(struct bpf_dynptr *dptr, u32 off,
3651+ u32 size, const void __user *unsafe_ptr__ign)
3652+{
3653+ return __bpf_dynptr_copy(dptr, off, size, (const void *)unsafe_ptr__ign,
3654+ copy_user_data_sleepable, NULL);
3655+}
3656+
3657+__bpf_kfunc int bpf_copy_from_user_str_dynptr(struct bpf_dynptr *dptr, u32 off,
3658+ u32 size, const void __user *unsafe_ptr__ign)
3659+{
3660+ return __bpf_dynptr_copy_str(dptr, off, size, (const void *)unsafe_ptr__ign,
3661+ copy_user_str_sleepable, NULL);
3662+}
3663+
3664+__bpf_kfunc int bpf_copy_from_user_task_dynptr(struct bpf_dynptr *dptr, u32 off,
3665+ u32 size, const void __user *unsafe_ptr__ign,
3666+ struct task_struct *tsk)
3667+{
3668+ return __bpf_dynptr_copy(dptr, off, size, (const void *)unsafe_ptr__ign,
3669+ copy_user_data_sleepable, tsk);
3670+}
3671+
3672+__bpf_kfunc int bpf_copy_from_user_task_str_dynptr(struct bpf_dynptr *dptr, u32 off,
3673+ u32 size, const void __user *unsafe_ptr__ign,
3674+ struct task_struct *tsk)
3675+{
3676+ return __bpf_dynptr_copy_str(dptr, off, size, (const void *)unsafe_ptr__ign,
3677+ copy_user_str_sleepable, tsk);
3678+}
3679+
35793680 __bpf_kfunc_end_defs();
kernel/trace/trace_uprobe.c+1−1
@@ -1489,7 +1489,7 @@ int bpf_get_uprobe_info(const struct perf_event *event, u32 *fd_type, · 1488 unmodified lines
14891489 : BPF_FD_TYPE_UPROBE;
14901490 *filename = tu->filename;
14911491 *probe_offset = tu->offset;
1492− *probe_addr = 0;
1492+ *probe_addr = tu->ref_ctr_offset;
14931493 return 0;
14941494 }
14951495 #endif /* CONFIG_PERF_EVENTS */