History

bpf: Add sleepable support for classic tracepoint programs

Add trace_call_bpf_faultable(), a variant of trace_call_bpf() for faultable tracepoints that supports sleepable BPF programs. It uses rcu_tasks_trace for lifetime protection and bpf_prog_run_array_sleepable() for per-program RCU flavor selection, following the uprobe_prog_run() pattern. Restructure perf_syscall_enter() and perf_syscall_exit() to run BPF programs before perf event processing. Previously, BPF ran after the per-cpu perf trace buffer was allocated under preempt_disable, requiring cleanup via perf_swevent_put_recursion_context() on filter. Now BPF runs in faultable context before preempt_disable, reading syscall arguments from local variables instead of the per-cpu trace record, removing the dependency on buffer allocation. This allows sleepable BPF programs to execute and avoids unnecessary buffer allocation when BPF filters the event. The perf event submission path (buffer allocation, fill, submit) remains under preempt_disable as before. Since BPF no longer runs within the buffer allocation context, the fake_regs output parameter to perf_trace_buf_alloc() is no longer needed and is replaced with NULL. Add an attach-time check in __perf_event_set_bpf_prog() to reject sleepable BPF_PROG_TYPE_TRACEPOINT programs on non-syscall tracepoints, since only syscall tracepoints run in faultable context. This prepares the classic tracepoint runtime and attach paths for sleepable programs. The verifier changes to allow loading sleepable BPF_PROG_TYPE_TRACEPOINT programs are in a subsequent patch. To: Peter Zijlstra <peterz@infradead.org> To: Steven Rostedt <rostedt@goodmis.org>

Mykyta Yatsenko committed 6 months ago

Commit 57918341 parent 12628ff4 files · +101 −52Browse files

Unified · Split

4 files
  1. include/linux/trace_events.h +6 −0
  2. kernel/events/core.c +9 −0
  3. kernel/trace/bpf_trace.c +28 −0
  4. kernel/trace/trace_syscalls.c +58 −52
include/linux/trace_events.h+6−0
@@ -770,6 +770,7 @@ trace_trigger_soft_disabled(struct trace_event_file *file) · 769 unmodified lines
770 770
771#ifdef CONFIG_BPF_EVENTS771#ifdef CONFIG_BPF_EVENTS
772unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx);772unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx);
773unsigned int trace_call_bpf_faultable(struct trace_event_call *call, void *ctx);
773int perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie);774int perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie);
774void perf_event_detach_bpf_prog(struct perf_event *event);775void perf_event_detach_bpf_prog(struct perf_event *event);
775int perf_event_query_prog_array(struct perf_event *event, void __user *info);776int perf_event_query_prog_array(struct perf_event *event, void __user *info);
@@ -792,6 +793,11 @@ static inline unsigned int trace_call_bpf(struct trace_event_call *call, void *c · 16 unmodified lines
792 return 1;793 return 1;
793}794}
794 795
796static inline unsigned int trace_call_bpf_faultable(struct trace_event_call *call, void *ctx)
797{
798 return 1;
799}
800
795static inline int801static inline int
796perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie)802perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie)
797{803{
kernel/events/core.c+9−0
@@ -11643,6 +11643,15 @@ static int __perf_event_set_bpf_prog(struct perf_event *event, · 11642 unmodified lines
11643 /* only uprobe programs are allowed to be sleepable */11643 /* only uprobe programs are allowed to be sleepable */
11644 return -EINVAL;11644 return -EINVAL;
11645 11645
11646 if (prog->type == BPF_PROG_TYPE_TRACEPOINT && prog->sleepable) {
11647 /*
11648 * Sleepable tracepoint programs can only attach to faultable
11649 * tracepoints. Currently only syscall tracepoints are faultable.
11650 */
11651 if (!is_syscall_tp)
11652 return -EINVAL;
11653 }
11654
11646 /* Kprobe override only works for kprobes, not uprobes. */11655 /* Kprobe override only works for kprobes, not uprobes. */
11647 if (prog->kprobe_override && !is_kprobe)11656 if (prog->kprobe_override && !is_kprobe)
11648 return -EINVAL;11657 return -EINVAL;
kernel/trace/bpf_trace.c+28−0
@@ -152,6 +152,34 @@ unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx) · 151 unmodified lines
152 return ret;152 return ret;
153}153}
154 154
155/**
156 * trace_call_bpf_faultable - invoke BPF program in faultable context
157 * @call: tracepoint event
158 * @ctx: opaque context pointer
159 *
160 * Variant of trace_call_bpf() for faultable tracepoints (syscall
161 * tracepoints). Supports sleepable BPF programs by using rcu_tasks_trace
162 * for lifetime protection and bpf_prog_run_array_sleepable() for per-program
163 * RCU flavor selection, following the uprobe pattern.
164 *
165 * Per-program recursion protection is provided by
166 * bpf_prog_run_array_sleepable(). Global bpf_prog_active is not
167 * needed because syscall tracepoints cannot self-recurse.
168 *
169 * Must be called from a faultable/preemptible context.
170 */
171unsigned int trace_call_bpf_faultable(struct trace_event_call *call, void *ctx)
172{
173 struct bpf_prog_array *prog_array;
174
175 might_fault();
176 guard(rcu_tasks_trace)();
177
178 prog_array = rcu_dereference_check(call->prog_array,
179 rcu_read_lock_trace_held());
180 return bpf_prog_run_array_sleepable(prog_array, ctx, bpf_prog_run);
181}
182
155#ifdef CONFIG_BPF_KPROBE_OVERRIDE183#ifdef CONFIG_BPF_KPROBE_OVERRIDE
156BPF_CALL_2(bpf_override_return, struct pt_regs *, regs, unsigned long, rc)184BPF_CALL_2(bpf_override_return, struct pt_regs *, regs, unsigned long, rc)
157{185{
kernel/trace/trace_syscalls.c+58−52
@@ -1371,33 +1371,33 @@ static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls); · 1370 unmodified lines
1371static int sys_perf_refcount_enter;1371static int sys_perf_refcount_enter;
1372static int sys_perf_refcount_exit;1372static int sys_perf_refcount_exit;
1373 1373
1374static int perf_call_bpf_enter(struct trace_event_call *call, struct pt_regs *regs,1374static int perf_call_bpf_enter(struct trace_event_call *call,
1375 struct syscall_metadata *sys_data,1375 struct syscall_metadata *sys_data,
1376 struct syscall_trace_enter *rec)1376 int syscall_nr, unsigned long *args)
1377{1377{
1378 struct syscall_tp_t {1378 struct syscall_tp_t {
1379 struct trace_entry ent;1379 struct trace_entry ent;
1380 int syscall_nr;1380 int syscall_nr;
1381 unsigned long args[SYSCALL_DEFINE_MAXARGS];1381 unsigned long args[SYSCALL_DEFINE_MAXARGS];
1382 } __aligned(8) param;1382 } __aligned(8) param;
1383 struct pt_regs regs = {};
1383 int i;1384 int i;
1384 1385
1385 BUILD_BUG_ON(sizeof(param.ent) < sizeof(void *));1386 BUILD_BUG_ON(sizeof(param.ent) < sizeof(void *));
1386 1387
1387 /* bpf prog requires 'regs' to be the first member in the ctx (a.k.a. &param) */1388 /* bpf prog requires 'regs' to be the first member in the ctx */
1388 perf_fetch_caller_regs(regs);1389 perf_fetch_caller_regs(&regs);
1389 *(struct pt_regs **)&param = regs;1390 *(struct pt_regs **)&param = &regs;
1390 param.syscall_nr = rec->nr;1391 param.syscall_nr = syscall_nr;
1391 for (i = 0; i < sys_data->nb_args; i++)1392 for (i = 0; i < sys_data->nb_args; i++)
1392 param.args[i] = rec->args[i];1393 param.args[i] = args[i];
1393 return trace_call_bpf(call, &param);1394 return trace_call_bpf_faultable(call, &param);
1394}1395}
1395 1396
1396static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)1397static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
1397{1398{
1398 struct syscall_metadata *sys_data;1399 struct syscall_metadata *sys_data;
1399 struct syscall_trace_enter *rec;1400 struct syscall_trace_enter *rec;
1400 struct pt_regs *fake_regs;
1401 struct hlist_head *head;1401 struct hlist_head *head;
1402 unsigned long args[6];1402 unsigned long args[6];
1403 bool valid_prog_array;1403 bool valid_prog_array;
@@ -1410,12 +1410,7 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) · 6 unmodified lines
1410 int size = 0;1410 int size = 0;
1411 int uargs = 0;1411 int uargs = 0;
1412 1412
1413 /*
1414 * Syscall probe called with preemption enabled, but the ring
1415 * buffer and per-cpu data require preemption to be disabled.
1416 */
1417 might_fault();1413 might_fault();
1418 guard(preempt_notrace)();
1419 1414
1420 syscall_nr = trace_get_syscall_nr(current, regs);1415 syscall_nr = trace_get_syscall_nr(current, regs);
1421 if (syscall_nr < 0 || syscall_nr >= NR_syscalls)1416 if (syscall_nr < 0 || syscall_nr >= NR_syscalls)
@@ -1429,6 +1424,26 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) · 7 unmodified lines
1429 1424
1430 syscall_get_arguments(current, regs, args);1425 syscall_get_arguments(current, regs, args);
1431 1426
1427 /*
1428 * Run BPF program in faultable context before per-cpu buffer
1429 * allocation, allowing sleepable BPF programs to execute.
1430 */
1431 valid_prog_array = bpf_prog_array_valid(sys_data->enter_event);
1432 if (valid_prog_array &&
1433 !perf_call_bpf_enter(sys_data->enter_event, sys_data,
1434 syscall_nr, args))
1435 return;
1436
1437 /*
1438 * Per-cpu ring buffer and perf event list operations require
1439 * preemption to be disabled.
1440 */
1441 guard(preempt_notrace)();
1442
1443 head = this_cpu_ptr(sys_data->enter_event->perf_events);
1444 if (hlist_empty(head))
1445 return;
1446
1432 /* Check if this syscall event faults in user space memory */1447 /* Check if this syscall event faults in user space memory */
1433 mayfault = sys_data->user_mask != 0;1448 mayfault = sys_data->user_mask != 0;
1434 1449
@@ -1438,17 +1453,12 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) · 3 unmodified lines
1438 return;1453 return;
1439 }1454 }
1440 1455
1441 head = this_cpu_ptr(sys_data->enter_event->perf_events);
1442 valid_prog_array = bpf_prog_array_valid(sys_data->enter_event);
1443 if (!valid_prog_array && hlist_empty(head))
1444 return;
1445
1446 /* get the size after alignment with the u32 buffer size field */1456 /* get the size after alignment with the u32 buffer size field */
1447 size += sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);1457 size += sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
1448 size = ALIGN(size + sizeof(u32), sizeof(u64));1458 size = ALIGN(size + sizeof(u32), sizeof(u64));
1449 size -= sizeof(u32);1459 size -= sizeof(u32);
1450 1460
1451 rec = perf_trace_buf_alloc(size, &fake_regs, &rctx);1461 rec = perf_trace_buf_alloc(size, NULL, &rctx);
1452 if (!rec)1462 if (!rec)
1453 return;1463 return;
1454 1464
@@ -1458,13 +1468,6 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) · 3 unmodified lines
1458 if (mayfault)1468 if (mayfault)
1459 syscall_put_data(sys_data, rec, user_ptr, size, user_sizes, uargs);1469 syscall_put_data(sys_data, rec, user_ptr, size, user_sizes, uargs);
1460 1470
1461 if ((valid_prog_array &&
1462 !perf_call_bpf_enter(sys_data->enter_event, fake_regs, sys_data, rec)) ||
1463 hlist_empty(head)) {
1464 perf_swevent_put_recursion_context(rctx);
1465 return;
1466 }
1467
1468 perf_trace_buf_submit(rec, size, rctx,1471 perf_trace_buf_submit(rec, size, rctx,
1469 sys_data->enter_event->event.type, 1, regs,1472 sys_data->enter_event->event.type, 1, regs,
1470 head, NULL);1473 head, NULL);
@@ -1514,40 +1517,35 @@ static void perf_sysenter_disable(struct trace_event_call *call) · 43 unmodified lines
1514 syscall_fault_buffer_disable();1517 syscall_fault_buffer_disable();
1515}1518}
1516 1519
1517static int perf_call_bpf_exit(struct trace_event_call *call, struct pt_regs *regs,1520static int perf_call_bpf_exit(struct trace_event_call *call,
1518 struct syscall_trace_exit *rec)1521 int syscall_nr, long ret_val)
1519{1522{
1520 struct syscall_tp_t {1523 struct syscall_tp_t {
1521 struct trace_entry ent;1524 struct trace_entry ent;
1522 int syscall_nr;1525 int syscall_nr;
1523 unsigned long ret;1526 unsigned long ret;
1524 } __aligned(8) param;1527 } __aligned(8) param;
1525 1528 struct pt_regs regs = {};
1526 /* bpf prog requires 'regs' to be the first member in the ctx (a.k.a. &param) */1529
1527 perf_fetch_caller_regs(regs);1530 /* bpf prog requires 'regs' to be the first member in the ctx */
1528 *(struct pt_regs **)&param = regs;1531 perf_fetch_caller_regs(&regs);
1529 param.syscall_nr = rec->nr;1532 *(struct pt_regs **)&param = &regs;
1530 param.ret = rec->ret;1533 param.syscall_nr = syscall_nr;
1531 return trace_call_bpf(call, &param);1534 param.ret = ret_val;
1535 return trace_call_bpf_faultable(call, &param);
1532}1536}
1533 1537
1534static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)1538static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
1535{1539{
1536 struct syscall_metadata *sys_data;1540 struct syscall_metadata *sys_data;
1537 struct syscall_trace_exit *rec;1541 struct syscall_trace_exit *rec;
1538 struct pt_regs *fake_regs;
1539 struct hlist_head *head;1542 struct hlist_head *head;
1540 bool valid_prog_array;1543 bool valid_prog_array;
1541 int syscall_nr;1544 int syscall_nr;
1542 int rctx;1545 int rctx;
1543 int size;1546 int size;
1544 1547
1545 /*
1546 * Syscall probe called with preemption enabled, but the ring
1547 * buffer and per-cpu data require preemption to be disabled.
1548 */
1549 might_fault();1548 might_fault();
1550 guard(preempt_notrace)();
1551 1549
1552 syscall_nr = trace_get_syscall_nr(current, regs);1550 syscall_nr = trace_get_syscall_nr(current, regs);
1553 if (syscall_nr < 0 || syscall_nr >= NR_syscalls)1551 if (syscall_nr < 0 || syscall_nr >= NR_syscalls)
@@ -1559,29 +1557,37 @@ static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret) · 5 unmodified lines
1559 if (!sys_data)1557 if (!sys_data)
1560 return;1558 return;
1561 1559
1562 head = this_cpu_ptr(sys_data->exit_event->perf_events);1560 /*
1561 * Run BPF program in faultable context before per-cpu buffer
1562 * allocation, allowing sleepable BPF programs to execute.
1563 */
1563 valid_prog_array = bpf_prog_array_valid(sys_data->exit_event);1564 valid_prog_array = bpf_prog_array_valid(sys_data->exit_event);
1564 if (!valid_prog_array && hlist_empty(head))1565 if (valid_prog_array &&
1566 !perf_call_bpf_exit(sys_data->exit_event, syscall_nr,
1567 syscall_get_return_value(current, regs)))
1568 return;
1569
1570 /*
1571 * Per-cpu ring buffer and perf event list operations require
1572 * preemption to be disabled.
1573 */
1574 guard(preempt_notrace)();
1575
1576 head = this_cpu_ptr(sys_data->exit_event->perf_events);
1577 if (hlist_empty(head))
1565 return;1578 return;
1566 1579
1567 /* We can probably do that at build time */1580 /* We can probably do that at build time */
1568 size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));1581 size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
1569 size -= sizeof(u32);1582 size -= sizeof(u32);
1570 1583
1571 rec = perf_trace_buf_alloc(size, &fake_regs, &rctx);1584 rec = perf_trace_buf_alloc(size, NULL, &rctx);
1572 if (!rec)1585 if (!rec)
1573 return;1586 return;
1574 1587
1575 rec->nr = syscall_nr;1588 rec->nr = syscall_nr;
1576 rec->ret = syscall_get_return_value(current, regs);1589 rec->ret = syscall_get_return_value(current, regs);
1577 1590
1578 if ((valid_prog_array &&
1579 !perf_call_bpf_exit(sys_data->exit_event, fake_regs, rec)) ||
1580 hlist_empty(head)) {
1581 perf_swevent_put_recursion_context(rctx);
1582 return;
1583 }
1584
1585 perf_trace_buf_submit(rec, size, rctx, sys_data->exit_event->event.type,1591 perf_trace_buf_submit(rec, size, rctx, sys_data->exit_event->event.type,
1586 1, regs, head, NULL);1592 1, regs, head, NULL);
1587}1593}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t