Add trace_call_bpf_faultable(), a variant of trace_call_bpf() for faultable tracepoints that supports sleepable BPF programs. It uses rcu_tasks_trace for lifetime protection and bpf_prog_run_array_sleepable() for per-program RCU flavor selection, following the uprobe_prog_run() pattern. Restructure perf_syscall_enter() and perf_syscall_exit() to run BPF programs before perf event processing. Previously, BPF ran after the per-cpu perf trace buffer was allocated under preempt_disable, requiring cleanup via perf_swevent_put_recursion_context() on filter. Now BPF runs in faultable context before preempt_disable, reading syscall arguments from local variables instead of the per-cpu trace record, removing the dependency on buffer allocation. This allows sleepable BPF programs to execute and avoids unnecessary buffer allocation when BPF filters the event. The perf event submission path (buffer allocation, fill, submit) remains under preempt_disable as before. Since BPF no longer runs within the buffer allocation context, the fake_regs output parameter to perf_trace_buf_alloc() is no longer needed and is replaced with NULL. Add an attach-time check in __perf_event_set_bpf_prog() to reject sleepable BPF_PROG_TYPE_TRACEPOINT programs on non-syscall tracepoints, since only syscall tracepoints run in faultable context. This prepares the classic tracepoint runtime and attach paths for sleepable programs. The verifier changes to allow loading sleepable BPF_PROG_TYPE_TRACEPOINT programs are in a subsequent patch. To: Peter Zijlstra <peterz@infradead.org> To: Steven Rostedt <rostedt@goodmis.org>
Showing the changes to kernel (3 of 4 files). Show all
Unified · Split
| @@ -11643,6 +11643,15 @@ static int __perf_event_set_bpf_prog(struct perf_event *event, · 11642 unmodified lines | |||
| 11643 | /* only uprobe programs are allowed to be sleepable */ | 11643 | /* only uprobe programs are allowed to be sleepable */ |
| 11644 | return -EINVAL; | 11644 | return -EINVAL; |
| 11645 | 11645 | ||
| 11646 | if (prog->type == BPF_PROG_TYPE_TRACEPOINT && prog->sleepable) { | ||
| 11647 | /* | ||
| 11648 | * Sleepable tracepoint programs can only attach to faultable | ||
| 11649 | * tracepoints. Currently only syscall tracepoints are faultable. | ||
| 11650 | */ | ||
| 11651 | if (!is_syscall_tp) | ||
| 11652 | return -EINVAL; | ||
| 11653 | } | ||
| 11654 | |||
| 11646 | /* Kprobe override only works for kprobes, not uprobes. */ | 11655 | /* Kprobe override only works for kprobes, not uprobes. */ |
| 11647 | if (prog->kprobe_override && !is_kprobe) | 11656 | if (prog->kprobe_override && !is_kprobe) |
| 11648 | return -EINVAL; | 11657 | return -EINVAL; |
| @@ -152,6 +152,34 @@ unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx) · 151 unmodified lines | |||
| 152 | return ret; | 152 | return ret; |
| 153 | } | 153 | } |
| 154 | 154 | ||
| 155 | /** | ||
| 156 | * trace_call_bpf_faultable - invoke BPF program in faultable context | ||
| 157 | * @call: tracepoint event | ||
| 158 | * @ctx: opaque context pointer | ||
| 159 | * | ||
| 160 | * Variant of trace_call_bpf() for faultable tracepoints (syscall | ||
| 161 | * tracepoints). Supports sleepable BPF programs by using rcu_tasks_trace | ||
| 162 | * for lifetime protection and bpf_prog_run_array_sleepable() for per-program | ||
| 163 | * RCU flavor selection, following the uprobe pattern. | ||
| 164 | * | ||
| 165 | * Per-program recursion protection is provided by | ||
| 166 | * bpf_prog_run_array_sleepable(). Global bpf_prog_active is not | ||
| 167 | * needed because syscall tracepoints cannot self-recurse. | ||
| 168 | * | ||
| 169 | * Must be called from a faultable/preemptible context. | ||
| 170 | */ | ||
| 171 | unsigned int trace_call_bpf_faultable(struct trace_event_call *call, void *ctx) | ||
| 172 | { | ||
| 173 | struct bpf_prog_array *prog_array; | ||
| 174 | |||
| 175 | might_fault(); | ||
| 176 | guard(rcu_tasks_trace)(); | ||
| 177 | |||
| 178 | prog_array = rcu_dereference_check(call->prog_array, | ||
| 179 | rcu_read_lock_trace_held()); | ||
| 180 | return bpf_prog_run_array_sleepable(prog_array, ctx, bpf_prog_run); | ||
| 181 | } | ||
| 182 | |||
| 155 | #ifdef CONFIG_BPF_KPROBE_OVERRIDE | 183 | #ifdef CONFIG_BPF_KPROBE_OVERRIDE |
| 156 | BPF_CALL_2(bpf_override_return, struct pt_regs *, regs, unsigned long, rc) | 184 | BPF_CALL_2(bpf_override_return, struct pt_regs *, regs, unsigned long, rc) |
| 157 | { | 185 | { |
| @@ -1371,33 +1371,33 @@ static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls); · 1370 unmodified lines | |||
| 1371 | static int sys_perf_refcount_enter; | 1371 | static int sys_perf_refcount_enter; |
| 1372 | static int sys_perf_refcount_exit; | 1372 | static int sys_perf_refcount_exit; |
| 1373 | 1373 | ||
| 1374 | static int perf_call_bpf_enter(struct trace_event_call *call, struct pt_regs *regs, | 1374 | static int perf_call_bpf_enter(struct trace_event_call *call, |
| 1375 | struct syscall_metadata *sys_data, | 1375 | struct syscall_metadata *sys_data, |
| 1376 | struct syscall_trace_enter *rec) | 1376 | int syscall_nr, unsigned long *args) |
| 1377 | { | 1377 | { |
| 1378 | struct syscall_tp_t { | 1378 | struct syscall_tp_t { |
| 1379 | struct trace_entry ent; | 1379 | struct trace_entry ent; |
| 1380 | int syscall_nr; | 1380 | int syscall_nr; |
| 1381 | unsigned long args[SYSCALL_DEFINE_MAXARGS]; | 1381 | unsigned long args[SYSCALL_DEFINE_MAXARGS]; |
| 1382 | } __aligned(8) param; | 1382 | } __aligned(8) param; |
| 1383 | struct pt_regs regs = {}; | ||
| 1383 | int i; | 1384 | int i; |
| 1384 | 1385 | ||
| 1385 | BUILD_BUG_ON(sizeof(param.ent) < sizeof(void *)); | 1386 | BUILD_BUG_ON(sizeof(param.ent) < sizeof(void *)); |
| 1386 | 1387 | ||
| 1387 | /* bpf prog requires 'regs' to be the first member in the ctx (a.k.a. ¶m) */ | 1388 | /* bpf prog requires 'regs' to be the first member in the ctx */ |
| 1388 | perf_fetch_caller_regs(regs); | 1389 | perf_fetch_caller_regs(®s); |
| 1389 | *(struct pt_regs **)¶m = regs; | 1390 | *(struct pt_regs **)¶m = ®s; |
| 1390 | param.syscall_nr = rec->nr; | 1391 | param.syscall_nr = syscall_nr; |
| 1391 | for (i = 0; i < sys_data->nb_args; i++) | 1392 | for (i = 0; i < sys_data->nb_args; i++) |
| 1392 | param.args[i] = rec->args[i]; | 1393 | param.args[i] = args[i]; |
| 1393 | return trace_call_bpf(call, ¶m); | 1394 | return trace_call_bpf_faultable(call, ¶m); |
| 1394 | } | 1395 | } |
| 1395 | 1396 | ||
| 1396 | static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) | 1397 | static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) |
| 1397 | { | 1398 | { |
| 1398 | struct syscall_metadata *sys_data; | 1399 | struct syscall_metadata *sys_data; |
| 1399 | struct syscall_trace_enter *rec; | 1400 | struct syscall_trace_enter *rec; |
| 1400 | struct pt_regs *fake_regs; | ||
| 1401 | struct hlist_head *head; | 1401 | struct hlist_head *head; |
| 1402 | unsigned long args[6]; | 1402 | unsigned long args[6]; |
| 1403 | bool valid_prog_array; | 1403 | bool valid_prog_array; |
| @@ -1410,12 +1410,7 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) · 6 unmodified lines | |||
| 1410 | int size = 0; | 1410 | int size = 0; |
| 1411 | int uargs = 0; | 1411 | int uargs = 0; |
| 1412 | 1412 | ||
| 1413 | /* | ||
| 1414 | * Syscall probe called with preemption enabled, but the ring | ||
| 1415 | * buffer and per-cpu data require preemption to be disabled. | ||
| 1416 | */ | ||
| 1417 | might_fault(); | 1413 | might_fault(); |
| 1418 | guard(preempt_notrace)(); | ||
| 1419 | 1414 | ||
| 1420 | syscall_nr = trace_get_syscall_nr(current, regs); | 1415 | syscall_nr = trace_get_syscall_nr(current, regs); |
| 1421 | if (syscall_nr < 0 || syscall_nr >= NR_syscalls) | 1416 | if (syscall_nr < 0 || syscall_nr >= NR_syscalls) |
| @@ -1429,6 +1424,26 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) · 7 unmodified lines | |||
| 1429 | 1424 | ||
| 1430 | syscall_get_arguments(current, regs, args); | 1425 | syscall_get_arguments(current, regs, args); |
| 1431 | 1426 | ||
| 1427 | /* | ||
| 1428 | * Run BPF program in faultable context before per-cpu buffer | ||
| 1429 | * allocation, allowing sleepable BPF programs to execute. | ||
| 1430 | */ | ||
| 1431 | valid_prog_array = bpf_prog_array_valid(sys_data->enter_event); | ||
| 1432 | if (valid_prog_array && | ||
| 1433 | !perf_call_bpf_enter(sys_data->enter_event, sys_data, | ||
| 1434 | syscall_nr, args)) | ||
| 1435 | return; | ||
| 1436 | |||
| 1437 | /* | ||
| 1438 | * Per-cpu ring buffer and perf event list operations require | ||
| 1439 | * preemption to be disabled. | ||
| 1440 | */ | ||
| 1441 | guard(preempt_notrace)(); | ||
| 1442 | |||
| 1443 | head = this_cpu_ptr(sys_data->enter_event->perf_events); | ||
| 1444 | if (hlist_empty(head)) | ||
| 1445 | return; | ||
| 1446 | |||
| 1432 | /* Check if this syscall event faults in user space memory */ | 1447 | /* Check if this syscall event faults in user space memory */ |
| 1433 | mayfault = sys_data->user_mask != 0; | 1448 | mayfault = sys_data->user_mask != 0; |
| 1434 | 1449 | ||
| @@ -1438,17 +1453,12 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) · 3 unmodified lines | |||
| 1438 | return; | 1453 | return; |
| 1439 | } | 1454 | } |
| 1440 | 1455 | ||
| 1441 | head = this_cpu_ptr(sys_data->enter_event->perf_events); | ||
| 1442 | valid_prog_array = bpf_prog_array_valid(sys_data->enter_event); | ||
| 1443 | if (!valid_prog_array && hlist_empty(head)) | ||
| 1444 | return; | ||
| 1445 | |||
| 1446 | /* get the size after alignment with the u32 buffer size field */ | 1456 | /* get the size after alignment with the u32 buffer size field */ |
| 1447 | size += sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec); | 1457 | size += sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec); |
| 1448 | size = ALIGN(size + sizeof(u32), sizeof(u64)); | 1458 | size = ALIGN(size + sizeof(u32), sizeof(u64)); |
| 1449 | size -= sizeof(u32); | 1459 | size -= sizeof(u32); |
| 1450 | 1460 | ||
| 1451 | rec = perf_trace_buf_alloc(size, &fake_regs, &rctx); | 1461 | rec = perf_trace_buf_alloc(size, NULL, &rctx); |
| 1452 | if (!rec) | 1462 | if (!rec) |
| 1453 | return; | 1463 | return; |
| 1454 | 1464 | ||
| @@ -1458,13 +1468,6 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) · 3 unmodified lines | |||
| 1458 | if (mayfault) | 1468 | if (mayfault) |
| 1459 | syscall_put_data(sys_data, rec, user_ptr, size, user_sizes, uargs); | 1469 | syscall_put_data(sys_data, rec, user_ptr, size, user_sizes, uargs); |
| 1460 | 1470 | ||
| 1461 | if ((valid_prog_array && | ||
| 1462 | !perf_call_bpf_enter(sys_data->enter_event, fake_regs, sys_data, rec)) || | ||
| 1463 | hlist_empty(head)) { | ||
| 1464 | perf_swevent_put_recursion_context(rctx); | ||
| 1465 | return; | ||
| 1466 | } | ||
| 1467 | |||
| 1468 | perf_trace_buf_submit(rec, size, rctx, | 1471 | perf_trace_buf_submit(rec, size, rctx, |
| 1469 | sys_data->enter_event->event.type, 1, regs, | 1472 | sys_data->enter_event->event.type, 1, regs, |
| 1470 | head, NULL); | 1473 | head, NULL); |
| @@ -1514,40 +1517,35 @@ static void perf_sysenter_disable(struct trace_event_call *call) · 43 unmodified lines | |||
| 1514 | syscall_fault_buffer_disable(); | 1517 | syscall_fault_buffer_disable(); |
| 1515 | } | 1518 | } |
| 1516 | 1519 | ||
| 1517 | static int perf_call_bpf_exit(struct trace_event_call *call, struct pt_regs *regs, | 1520 | static int perf_call_bpf_exit(struct trace_event_call *call, |
| 1518 | struct syscall_trace_exit *rec) | 1521 | int syscall_nr, long ret_val) |
| 1519 | { | 1522 | { |
| 1520 | struct syscall_tp_t { | 1523 | struct syscall_tp_t { |
| 1521 | struct trace_entry ent; | 1524 | struct trace_entry ent; |
| 1522 | int syscall_nr; | 1525 | int syscall_nr; |
| 1523 | unsigned long ret; | 1526 | unsigned long ret; |
| 1524 | } __aligned(8) param; | 1527 | } __aligned(8) param; |
| 1525 | 1528 | struct pt_regs regs = {}; | |
| 1526 | /* bpf prog requires 'regs' to be the first member in the ctx (a.k.a. ¶m) */ | 1529 | |
| 1527 | perf_fetch_caller_regs(regs); | 1530 | /* bpf prog requires 'regs' to be the first member in the ctx */ |
| 1528 | *(struct pt_regs **)¶m = regs; | 1531 | perf_fetch_caller_regs(®s); |
| 1529 | param.syscall_nr = rec->nr; | 1532 | *(struct pt_regs **)¶m = ®s; |
| 1530 | param.ret = rec->ret; | 1533 | param.syscall_nr = syscall_nr; |
| 1531 | return trace_call_bpf(call, ¶m); | 1534 | param.ret = ret_val; |
| 1535 | return trace_call_bpf_faultable(call, ¶m); | ||
| 1532 | } | 1536 | } |
| 1533 | 1537 | ||
| 1534 | static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret) | 1538 | static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret) |
| 1535 | { | 1539 | { |
| 1536 | struct syscall_metadata *sys_data; | 1540 | struct syscall_metadata *sys_data; |
| 1537 | struct syscall_trace_exit *rec; | 1541 | struct syscall_trace_exit *rec; |
| 1538 | struct pt_regs *fake_regs; | ||
| 1539 | struct hlist_head *head; | 1542 | struct hlist_head *head; |
| 1540 | bool valid_prog_array; | 1543 | bool valid_prog_array; |
| 1541 | int syscall_nr; | 1544 | int syscall_nr; |
| 1542 | int rctx; | 1545 | int rctx; |
| 1543 | int size; | 1546 | int size; |
| 1544 | 1547 | ||
| 1545 | /* | ||
| 1546 | * Syscall probe called with preemption enabled, but the ring | ||
| 1547 | * buffer and per-cpu data require preemption to be disabled. | ||
| 1548 | */ | ||
| 1549 | might_fault(); | 1548 | might_fault(); |
| 1550 | guard(preempt_notrace)(); | ||
| 1551 | 1549 | ||
| 1552 | syscall_nr = trace_get_syscall_nr(current, regs); | 1550 | syscall_nr = trace_get_syscall_nr(current, regs); |
| 1553 | if (syscall_nr < 0 || syscall_nr >= NR_syscalls) | 1551 | if (syscall_nr < 0 || syscall_nr >= NR_syscalls) |
| @@ -1559,29 +1557,37 @@ static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret) · 5 unmodified lines | |||
| 1559 | if (!sys_data) | 1557 | if (!sys_data) |
| 1560 | return; | 1558 | return; |
| 1561 | 1559 | ||
| 1562 | head = this_cpu_ptr(sys_data->exit_event->perf_events); | 1560 | /* |
| 1561 | * Run BPF program in faultable context before per-cpu buffer | ||
| 1562 | * allocation, allowing sleepable BPF programs to execute. | ||
| 1563 | */ | ||
| 1563 | valid_prog_array = bpf_prog_array_valid(sys_data->exit_event); | 1564 | valid_prog_array = bpf_prog_array_valid(sys_data->exit_event); |
| 1564 | if (!valid_prog_array && hlist_empty(head)) | 1565 | if (valid_prog_array && |
| 1566 | !perf_call_bpf_exit(sys_data->exit_event, syscall_nr, | ||
| 1567 | syscall_get_return_value(current, regs))) | ||
| 1568 | return; | ||
| 1569 | |||
| 1570 | /* | ||
| 1571 | * Per-cpu ring buffer and perf event list operations require | ||
| 1572 | * preemption to be disabled. | ||
| 1573 | */ | ||
| 1574 | guard(preempt_notrace)(); | ||
| 1575 | |||
| 1576 | head = this_cpu_ptr(sys_data->exit_event->perf_events); | ||
| 1577 | if (hlist_empty(head)) | ||
| 1565 | return; | 1578 | return; |
| 1566 | 1579 | ||
| 1567 | /* We can probably do that at build time */ | 1580 | /* We can probably do that at build time */ |
| 1568 | size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64)); | 1581 | size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64)); |
| 1569 | size -= sizeof(u32); | 1582 | size -= sizeof(u32); |
| 1570 | 1583 | ||
| 1571 | rec = perf_trace_buf_alloc(size, &fake_regs, &rctx); | 1584 | rec = perf_trace_buf_alloc(size, NULL, &rctx); |
| 1572 | if (!rec) | 1585 | if (!rec) |
| 1573 | return; | 1586 | return; |
| 1574 | 1587 | ||
| 1575 | rec->nr = syscall_nr; | 1588 | rec->nr = syscall_nr; |
| 1576 | rec->ret = syscall_get_return_value(current, regs); | 1589 | rec->ret = syscall_get_return_value(current, regs); |
| 1577 | 1590 | ||
| 1578 | if ((valid_prog_array && | ||
| 1579 | !perf_call_bpf_exit(sys_data->exit_event, fake_regs, rec)) || | ||
| 1580 | hlist_empty(head)) { | ||
| 1581 | perf_swevent_put_recursion_context(rctx); | ||
| 1582 | return; | ||
| 1583 | } | ||
| 1584 | |||
| 1585 | perf_trace_buf_submit(rec, size, rctx, sys_data->exit_event->event.type, | 1591 | perf_trace_buf_submit(rec, size, rctx, sys_data->exit_event->event.type, |
| 1586 | 1, regs, head, NULL); | 1592 | 1, regs, head, NULL); |
| 1587 | } | 1593 | } |