History

bpf: Add sleepable support for classic tracepoint programs

Add trace_call_bpf_faultable(), a variant of trace_call_bpf() for faultable tracepoints that supports sleepable BPF programs. It uses rcu_tasks_trace for lifetime protection and bpf_prog_run_array_sleepable() for per-program RCU flavor selection, following the uprobe_prog_run() pattern. Restructure perf_syscall_enter() and perf_syscall_exit() to run BPF programs before perf event processing. Previously, BPF ran after the per-cpu perf trace buffer was allocated under preempt_disable, requiring cleanup via perf_swevent_put_recursion_context() on filter. Now BPF runs in faultable context before preempt_disable, reading syscall arguments from local variables instead of the per-cpu trace record, removing the dependency on buffer allocation. This allows sleepable BPF programs to execute and avoids unnecessary buffer allocation when BPF filters the event. The perf event submission path (buffer allocation, fill, submit) remains under preempt_disable as before. Since BPF no longer runs within the buffer allocation context, the fake_regs output parameter to perf_trace_buf_alloc() is no longer needed and is replaced with NULL. Add an attach-time check in __perf_event_set_bpf_prog() to reject sleepable BPF_PROG_TYPE_TRACEPOINT programs on non-syscall tracepoints, since only syscall tracepoints run in faultable context. This prepares the classic tracepoint runtime and attach paths for sleepable programs. The verifier changes to allow loading sleepable BPF_PROG_TYPE_TRACEPOINT programs are in a subsequent patch. To: Peter Zijlstra <peterz@infradead.org> To: Steven Rostedt <rostedt@goodmis.org>

Mykyta Yatsenko committed 6 months ago

Commit 57918341 parent 12628ff4 files · +101 −52Browse files

Unified · Split

4 files
  1. include/linux/trace_events.h +6 −0
  2. kernel/events/core.c +9 −0
  3. kernel/trace/bpf_trace.c +28 −0
  4. kernel/trace/trace_syscalls.c +58 −52
include/linux/trace_events.h+6−0
@@ -770,6 +770,7 @@ trace_trigger_soft_disabled(struct trace_event_file *file) · 769 unmodified lines
770770
771771 #ifdef CONFIG_BPF_EVENTS
772772 unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx);
773+unsigned int trace_call_bpf_faultable(struct trace_event_call *call, void *ctx);
773774 int perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie);
774775 void perf_event_detach_bpf_prog(struct perf_event *event);
775776 int perf_event_query_prog_array(struct perf_event *event, void __user *info);
@@ -792,6 +793,11 @@ static inline unsigned int trace_call_bpf(struct trace_event_call *call, void *c · 16 unmodified lines
792793 return 1;
793794 }
794795
796+static inline unsigned int trace_call_bpf_faultable(struct trace_event_call *call, void *ctx)
797+{
798+ return 1;
799+}
800+
795801 static inline int
796802 perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie)
797803 {
kernel/events/core.c+9−0
@@ -11643,6 +11643,15 @@ static int __perf_event_set_bpf_prog(struct perf_event *event, · 11642 unmodified lines
1164311643 /* only uprobe programs are allowed to be sleepable */
1164411644 return -EINVAL;
1164511645
11646+ if (prog->type == BPF_PROG_TYPE_TRACEPOINT && prog->sleepable) {
11647+ /*
11648+ * Sleepable tracepoint programs can only attach to faultable
11649+ * tracepoints. Currently only syscall tracepoints are faultable.
11650+ */
11651+ if (!is_syscall_tp)
11652+ return -EINVAL;
11653+ }
11654+
1164611655 /* Kprobe override only works for kprobes, not uprobes. */
1164711656 if (prog->kprobe_override && !is_kprobe)
1164811657 return -EINVAL;
kernel/trace/bpf_trace.c+28−0
@@ -152,6 +152,34 @@ unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx) · 151 unmodified lines
152152 return ret;
153153 }
154154
155+/**
156+ * trace_call_bpf_faultable - invoke BPF program in faultable context
157+ * @call: tracepoint event
158+ * @ctx: opaque context pointer
159+ *
160+ * Variant of trace_call_bpf() for faultable tracepoints (syscall
161+ * tracepoints). Supports sleepable BPF programs by using rcu_tasks_trace
162+ * for lifetime protection and bpf_prog_run_array_sleepable() for per-program
163+ * RCU flavor selection, following the uprobe pattern.
164+ *
165+ * Per-program recursion protection is provided by
166+ * bpf_prog_run_array_sleepable(). Global bpf_prog_active is not
167+ * needed because syscall tracepoints cannot self-recurse.
168+ *
169+ * Must be called from a faultable/preemptible context.
170+ */
171+unsigned int trace_call_bpf_faultable(struct trace_event_call *call, void *ctx)
172+{
173+ struct bpf_prog_array *prog_array;
174+
175+ might_fault();
176+ guard(rcu_tasks_trace)();
177+
178+ prog_array = rcu_dereference_check(call->prog_array,
179+ rcu_read_lock_trace_held());
180+ return bpf_prog_run_array_sleepable(prog_array, ctx, bpf_prog_run);
181+}
182+
155183 #ifdef CONFIG_BPF_KPROBE_OVERRIDE
156184 BPF_CALL_2(bpf_override_return, struct pt_regs *, regs, unsigned long, rc)
157185 {
kernel/trace/trace_syscalls.c+58−52
@@ -1371,33 +1371,33 @@ static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls); · 1370 unmodified lines
13711371 static int sys_perf_refcount_enter;
13721372 static int sys_perf_refcount_exit;
13731373
1374−static int perf_call_bpf_enter(struct trace_event_call *call, struct pt_regs *regs,
1374+static int perf_call_bpf_enter(struct trace_event_call *call,
13751375 struct syscall_metadata *sys_data,
1376− struct syscall_trace_enter *rec)
1376+ int syscall_nr, unsigned long *args)
13771377 {
13781378 struct syscall_tp_t {
13791379 struct trace_entry ent;
13801380 int syscall_nr;
13811381 unsigned long args[SYSCALL_DEFINE_MAXARGS];
13821382 } __aligned(8) param;
1383+ struct pt_regs regs = {};
13831384 int i;
13841385
13851386 BUILD_BUG_ON(sizeof(param.ent) < sizeof(void *));
13861387
1387− /* bpf prog requires 'regs' to be the first member in the ctx (a.k.a. &param) */
1388− perf_fetch_caller_regs(regs);
1389− *(struct pt_regs **)&param = regs;
1390− param.syscall_nr = rec->nr;
1388+ /* bpf prog requires 'regs' to be the first member in the ctx */
1389+ perf_fetch_caller_regs(&regs);
1390+ *(struct pt_regs **)&param = &regs;
1391+ param.syscall_nr = syscall_nr;
13911392 for (i = 0; i < sys_data->nb_args; i++)
1392− param.args[i] = rec->args[i];
1393− return trace_call_bpf(call, &param);
1393+ param.args[i] = args[i];
1394+ return trace_call_bpf_faultable(call, &param);
13941395 }
13951396
13961397 static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
13971398 {
13981399 struct syscall_metadata *sys_data;
13991400 struct syscall_trace_enter *rec;
1400− struct pt_regs *fake_regs;
14011401 struct hlist_head *head;
14021402 unsigned long args[6];
14031403 bool valid_prog_array;
@@ -1410,12 +1410,7 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) · 6 unmodified lines
14101410 int size = 0;
14111411 int uargs = 0;
14121412
1413− /*
1414− * Syscall probe called with preemption enabled, but the ring
1415− * buffer and per-cpu data require preemption to be disabled.
1416− */
14171413 might_fault();
1418− guard(preempt_notrace)();
14191414
14201415 syscall_nr = trace_get_syscall_nr(current, regs);
14211416 if (syscall_nr < 0 || syscall_nr >= NR_syscalls)
@@ -1429,6 +1424,26 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) · 7 unmodified lines
14291424
14301425 syscall_get_arguments(current, regs, args);
14311426
1427+ /*
1428+ * Run BPF program in faultable context before per-cpu buffer
1429+ * allocation, allowing sleepable BPF programs to execute.
1430+ */
1431+ valid_prog_array = bpf_prog_array_valid(sys_data->enter_event);
1432+ if (valid_prog_array &&
1433+ !perf_call_bpf_enter(sys_data->enter_event, sys_data,
1434+ syscall_nr, args))
1435+ return;
1436+
1437+ /*
1438+ * Per-cpu ring buffer and perf event list operations require
1439+ * preemption to be disabled.
1440+ */
1441+ guard(preempt_notrace)();
1442+
1443+ head = this_cpu_ptr(sys_data->enter_event->perf_events);
1444+ if (hlist_empty(head))
1445+ return;
1446+
14321447 /* Check if this syscall event faults in user space memory */
14331448 mayfault = sys_data->user_mask != 0;
14341449
@@ -1438,17 +1453,12 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) · 3 unmodified lines
14381453 return;
14391454 }
14401455
1441− head = this_cpu_ptr(sys_data->enter_event->perf_events);
1442− valid_prog_array = bpf_prog_array_valid(sys_data->enter_event);
1443− if (!valid_prog_array && hlist_empty(head))
1444− return;
1445−
14461456 /* get the size after alignment with the u32 buffer size field */
14471457 size += sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
14481458 size = ALIGN(size + sizeof(u32), sizeof(u64));
14491459 size -= sizeof(u32);
14501460
1451− rec = perf_trace_buf_alloc(size, &fake_regs, &rctx);
1461+ rec = perf_trace_buf_alloc(size, NULL, &rctx);
14521462 if (!rec)
14531463 return;
14541464
@@ -1458,13 +1468,6 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) · 3 unmodified lines
14581468 if (mayfault)
14591469 syscall_put_data(sys_data, rec, user_ptr, size, user_sizes, uargs);
14601470
1461− if ((valid_prog_array &&
1462− !perf_call_bpf_enter(sys_data->enter_event, fake_regs, sys_data, rec)) ||
1463− hlist_empty(head)) {
1464− perf_swevent_put_recursion_context(rctx);
1465− return;
1466− }
1467−
14681471 perf_trace_buf_submit(rec, size, rctx,
14691472 sys_data->enter_event->event.type, 1, regs,
14701473 head, NULL);
@@ -1514,40 +1517,35 @@ static void perf_sysenter_disable(struct trace_event_call *call) · 43 unmodified lines
15141517 syscall_fault_buffer_disable();
15151518 }
15161519
1517−static int perf_call_bpf_exit(struct trace_event_call *call, struct pt_regs *regs,
1518− struct syscall_trace_exit *rec)
1520+static int perf_call_bpf_exit(struct trace_event_call *call,
1521+ int syscall_nr, long ret_val)
15191522 {
15201523 struct syscall_tp_t {
15211524 struct trace_entry ent;
15221525 int syscall_nr;
15231526 unsigned long ret;
15241527 } __aligned(8) param;
1525−
1526− /* bpf prog requires 'regs' to be the first member in the ctx (a.k.a. &param) */
1527− perf_fetch_caller_regs(regs);
1528− *(struct pt_regs **)&param = regs;
1529− param.syscall_nr = rec->nr;
1530− param.ret = rec->ret;
1531− return trace_call_bpf(call, &param);
1528+ struct pt_regs regs = {};
1529+
1530+ /* bpf prog requires 'regs' to be the first member in the ctx */
1531+ perf_fetch_caller_regs(&regs);
1532+ *(struct pt_regs **)&param = &regs;
1533+ param.syscall_nr = syscall_nr;
1534+ param.ret = ret_val;
1535+ return trace_call_bpf_faultable(call, &param);
15321536 }
15331537
15341538 static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
15351539 {
15361540 struct syscall_metadata *sys_data;
15371541 struct syscall_trace_exit *rec;
1538− struct pt_regs *fake_regs;
15391542 struct hlist_head *head;
15401543 bool valid_prog_array;
15411544 int syscall_nr;
15421545 int rctx;
15431546 int size;
15441547
1545− /*
1546− * Syscall probe called with preemption enabled, but the ring
1547− * buffer and per-cpu data require preemption to be disabled.
1548− */
15491548 might_fault();
1550− guard(preempt_notrace)();
15511549
15521550 syscall_nr = trace_get_syscall_nr(current, regs);
15531551 if (syscall_nr < 0 || syscall_nr >= NR_syscalls)
@@ -1559,29 +1557,37 @@ static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret) · 5 unmodified lines
15591557 if (!sys_data)
15601558 return;
15611559
1562− head = this_cpu_ptr(sys_data->exit_event->perf_events);
1560+ /*
1561+ * Run BPF program in faultable context before per-cpu buffer
1562+ * allocation, allowing sleepable BPF programs to execute.
1563+ */
15631564 valid_prog_array = bpf_prog_array_valid(sys_data->exit_event);
1564− if (!valid_prog_array && hlist_empty(head))
1565+ if (valid_prog_array &&
1566+ !perf_call_bpf_exit(sys_data->exit_event, syscall_nr,
1567+ syscall_get_return_value(current, regs)))
1568+ return;
1569+
1570+ /*
1571+ * Per-cpu ring buffer and perf event list operations require
1572+ * preemption to be disabled.
1573+ */
1574+ guard(preempt_notrace)();
1575+
1576+ head = this_cpu_ptr(sys_data->exit_event->perf_events);
1577+ if (hlist_empty(head))
15651578 return;
15661579
15671580 /* We can probably do that at build time */
15681581 size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
15691582 size -= sizeof(u32);
15701583
1571− rec = perf_trace_buf_alloc(size, &fake_regs, &rctx);
1584+ rec = perf_trace_buf_alloc(size, NULL, &rctx);
15721585 if (!rec)
15731586 return;
15741587
15751588 rec->nr = syscall_nr;
15761589 rec->ret = syscall_get_return_value(current, regs);
15771590
1578− if ((valid_prog_array &&
1579− !perf_call_bpf_exit(sys_data->exit_event, fake_regs, rec)) ||
1580− hlist_empty(head)) {
1581− perf_swevent_put_recursion_context(rctx);
1582− return;
1583− }
1584−
15851591 perf_trace_buf_submit(rec, size, rctx, sys_data->exit_event->event.type,
15861592 1, regs, head, NULL);
15871593 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t