History

Merge tag 'bpf-next-6.13' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Pull bpf updates from Alexei Starovoitov: - Add BPF uprobe session support (Jiri Olsa) - Optimize uprobe performance (Andrii Nakryiko) - Add bpf_fastcall support to helpers and kfuncs (Eduard Zingerman) - Avoid calling free_htab_elem() under hash map bucket lock (Hou Tao) - Prevent tailcall infinite loop caused by freplace (Leon Hwang) - Mark raw_tracepoint arguments as nullable (Kumar Kartikeya Dwivedi) - Introduce uptr support in the task local storage map (Martin KaFai Lau) - Stringify errno log messages in libbpf (Mykyta Yatsenko) - Add kmem_cache BPF iterator for perf's lock profiling (Namhyung Kim) - Support BPF objects of either endianness in libbpf (Tony Ambardar) - Add ksym to struct_ops trampoline to fix stack trace (Xu Kuohai) - Introduce private stack for eligible BPF programs (Yonghong Song) - Migrate samples/bpf tests to selftests/bpf test_progs (Daniel T. Lee) - Migrate test_sock to selftests/bpf test_progs (Jordan Rife) * tag 'bpf-next-6.13' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next: (152 commits) libbpf: Change hash_combine parameters from long to unsigned long selftests/bpf: Fix build error with llvm 19 libbpf: Fix memory leak in bpf_program__attach_uprobe_multi bpf: use common instruction history across all states bpf: Add necessary migrate_disable to range_tree. bpf: Do not alloc arena on unsupported arches selftests/bpf: Set test path for token/obj_priv_implicit_token_envvar selftests/bpf: Add a test for arena range tree algorithm bpf: Introduce range_tree data structure and use it in bpf arena samples/bpf: Remove unused variable in xdp2skb_meta_kern.c samples/bpf: Remove unused variables in tc_l2_redirect_kern.c bpftool: Cast variable `var` to long long bpf, x86: Propagate tailcall info only for subprogs bpf: Add kernel symbol for struct_ops trampoline bpf: Use function pointers count as struct_ops links count bpf: Remove unused member rcu from bpf_struct_ops_map selftests/bpf: Add struct_ops prog private stack tests bpf: Support private stack for struct_ops progs selftests/bpf: Add tracing prog private stack tests bpf, x86: Support private stack in jit ...

Linus Torvalds committed 1 year ago

Commit 6e95ef02 parents 43fb83c 2c8b09a211 files · +6961 −3473Browse files

Showing the changes to kernel/trace (1 of 211 files). Show all

Unified · Split

kernel/trace/bpf_trace.c+86−24
@@ -802,6 +802,8 @@ struct send_signal_irq_work { · 801 unmodified lines
802 struct task_struct *task;802 struct task_struct *task;
803 u32 sig;803 u32 sig;
804 enum pid_type type;804 enum pid_type type;
805 bool has_siginfo;
806 struct kernel_siginfo info;
805};807};
806 808
807static DEFINE_PER_CPU(struct send_signal_irq_work, send_signal_work);809static DEFINE_PER_CPU(struct send_signal_irq_work, send_signal_work);
@@ -809,27 +811,46 @@ static DEFINE_PER_CPU(struct send_signal_irq_work, send_signal_work); · 1 unmodified line
809static void do_bpf_send_signal(struct irq_work *entry)811static void do_bpf_send_signal(struct irq_work *entry)
810{812{
811 struct send_signal_irq_work *work;813 struct send_signal_irq_work *work;
814 struct kernel_siginfo *siginfo;
812 815
813 work = container_of(entry, struct send_signal_irq_work, irq_work);816 work = container_of(entry, struct send_signal_irq_work, irq_work);
814 group_send_sig_info(work->sig, SEND_SIG_PRIV, work->task, work->type);817 siginfo = work->has_siginfo ? &work->info : SEND_SIG_PRIV;
818
819 group_send_sig_info(work->sig, siginfo, work->task, work->type);
815 put_task_struct(work->task);820 put_task_struct(work->task);
816}821}
817 822
818static int bpf_send_signal_common(u32 sig, enum pid_type type)823static int bpf_send_signal_common(u32 sig, enum pid_type type, struct task_struct *task, u64 value)
819{824{
820 struct send_signal_irq_work *work = NULL;825 struct send_signal_irq_work *work = NULL;
826 struct kernel_siginfo info;
827 struct kernel_siginfo *siginfo;
828
829 if (!task) {
830 task = current;
831 siginfo = SEND_SIG_PRIV;
832 } else {
833 clear_siginfo(&info);
834 info.si_signo = sig;
835 info.si_errno = 0;
836 info.si_code = SI_KERNEL;
837 info.si_pid = 0;
838 info.si_uid = 0;
839 info.si_value.sival_ptr = (void *)(unsigned long)value;
840 siginfo = &info;
841 }
821 842
822 /* Similar to bpf_probe_write_user, task needs to be843 /* Similar to bpf_probe_write_user, task needs to be
823 * in a sound condition and kernel memory access be844 * in a sound condition and kernel memory access be
824 * permitted in order to send signal to the current845 * permitted in order to send signal to the current
825 * task.846 * task.
826 */847 */
827 if (unlikely(current->flags & (PF_KTHREAD | PF_EXITING)))848 if (unlikely(task->flags & (PF_KTHREAD | PF_EXITING)))
828 return -EPERM;849 return -EPERM;
829 if (unlikely(!nmi_uaccess_okay()))850 if (unlikely(!nmi_uaccess_okay()))
830 return -EPERM;851 return -EPERM;
831 /* Task should not be pid=1 to avoid kernel panic. */852 /* Task should not be pid=1 to avoid kernel panic. */
832 if (unlikely(is_global_init(current)))853 if (unlikely(is_global_init(task)))
833 return -EPERM;854 return -EPERM;
834 855
835 if (irqs_disabled()) {856 if (irqs_disabled()) {
@@ -847,19 +868,22 @@ static int bpf_send_signal_common(u32 sig, enum pid_type type) · 11 unmodified lines
847 * to the irq_work. The current task may change when queued868 * to the irq_work. The current task may change when queued
848 * irq works get executed.869 * irq works get executed.
849 */870 */
850 work->task = get_task_struct(current);871 work->task = get_task_struct(task);
872 work->has_siginfo = siginfo == &info;
873 if (work->has_siginfo)
874 copy_siginfo(&work->info, &info);
851 work->sig = sig;875 work->sig = sig;
852 work->type = type;876 work->type = type;
853 irq_work_queue(&work->irq_work);877 irq_work_queue(&work->irq_work);
854 return 0;878 return 0;
855 }879 }
856 880
857 return group_send_sig_info(sig, SEND_SIG_PRIV, current, type);881 return group_send_sig_info(sig, siginfo, task, type);
858}882}
859 883
860BPF_CALL_1(bpf_send_signal, u32, sig)884BPF_CALL_1(bpf_send_signal, u32, sig)
861{885{
862 return bpf_send_signal_common(sig, PIDTYPE_TGID);886 return bpf_send_signal_common(sig, PIDTYPE_TGID, NULL, 0);
863}887}
864 888
865static const struct bpf_func_proto bpf_send_signal_proto = {889static const struct bpf_func_proto bpf_send_signal_proto = {
@@ -871,7 +895,7 @@ static const struct bpf_func_proto bpf_send_signal_proto = { · 5 unmodified lines
871 895
872BPF_CALL_1(bpf_send_signal_thread, u32, sig)896BPF_CALL_1(bpf_send_signal_thread, u32, sig)
873{897{
874 return bpf_send_signal_common(sig, PIDTYPE_PID);898 return bpf_send_signal_common(sig, PIDTYPE_PID, NULL, 0);
875}899}
876 900
877static const struct bpf_func_proto bpf_send_signal_thread_proto = {901static const struct bpf_func_proto bpf_send_signal_thread_proto = {
@@ -1557,6 +1581,17 @@ static inline bool is_kprobe_session(const struct bpf_prog *prog) · 679 unmodified lines
1557 return prog->expected_attach_type == BPF_TRACE_KPROBE_SESSION;1581 return prog->expected_attach_type == BPF_TRACE_KPROBE_SESSION;
1558}1582}
1559 1583
1584static inline bool is_uprobe_multi(const struct bpf_prog *prog)
1585{
1586 return prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI ||
1587 prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION;
1588}
1589
1590static inline bool is_uprobe_session(const struct bpf_prog *prog)
1591{
1592 return prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION;
1593}
1594
1560static const struct bpf_func_proto *1595static const struct bpf_func_proto *
1561kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)1596kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1562{1597{
@@ -1574,13 +1609,13 @@ kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 11 unmodified lines
1574 case BPF_FUNC_get_func_ip:1609 case BPF_FUNC_get_func_ip:
1575 if (is_kprobe_multi(prog))1610 if (is_kprobe_multi(prog))
1576 return &bpf_get_func_ip_proto_kprobe_multi;1611 return &bpf_get_func_ip_proto_kprobe_multi;
1577 if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI)1612 if (is_uprobe_multi(prog))
1578 return &bpf_get_func_ip_proto_uprobe_multi;1613 return &bpf_get_func_ip_proto_uprobe_multi;
1579 return &bpf_get_func_ip_proto_kprobe;1614 return &bpf_get_func_ip_proto_kprobe;
1580 case BPF_FUNC_get_attach_cookie:1615 case BPF_FUNC_get_attach_cookie:
1581 if (is_kprobe_multi(prog))1616 if (is_kprobe_multi(prog))
1582 return &bpf_get_attach_cookie_proto_kmulti;1617 return &bpf_get_attach_cookie_proto_kmulti;
1583 if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI)1618 if (is_uprobe_multi(prog))
1584 return &bpf_get_attach_cookie_proto_umulti;1619 return &bpf_get_attach_cookie_proto_umulti;
1585 return &bpf_get_attach_cookie_proto_trace;1620 return &bpf_get_attach_cookie_proto_trace;
1586 default:1621 default:
@@ -3072,6 +3107,7 @@ struct bpf_uprobe { · 1485 unmodified lines
3072 u64 cookie;3107 u64 cookie;
3073 struct uprobe *uprobe;3108 struct uprobe *uprobe;
3074 struct uprobe_consumer consumer;3109 struct uprobe_consumer consumer;
3110 bool session;
3075};3111};
3076 3112
3077struct bpf_uprobe_multi_link {3113struct bpf_uprobe_multi_link {
@@ -3084,7 +3120,7 @@ struct bpf_uprobe_multi_link { · 6 unmodified lines
3084};3120};
3085 3121
3086struct bpf_uprobe_multi_run_ctx {3122struct bpf_uprobe_multi_run_ctx {
3087 struct bpf_run_ctx run_ctx;3123 struct bpf_session_run_ctx session_ctx;
3088 unsigned long entry_ip;3124 unsigned long entry_ip;
3089 struct bpf_uprobe *uprobe;3125 struct bpf_uprobe *uprobe;
3090};3126};
@@ -3195,17 +3231,22 @@ static const struct bpf_link_ops bpf_uprobe_multi_link_lops = { · 104 unmodified lines
3195 3231
3196static int uprobe_prog_run(struct bpf_uprobe *uprobe,3232static int uprobe_prog_run(struct bpf_uprobe *uprobe,
3197 unsigned long entry_ip,3233 unsigned long entry_ip,
3198 struct pt_regs *regs)3234 struct pt_regs *regs,
3235 bool is_return, void *data)
3199{3236{
3200 struct bpf_uprobe_multi_link *link = uprobe->link;3237 struct bpf_uprobe_multi_link *link = uprobe->link;
3201 struct bpf_uprobe_multi_run_ctx run_ctx = {3238 struct bpf_uprobe_multi_run_ctx run_ctx = {
3239 .session_ctx = {
3240 .is_return = is_return,
3241 .data = data,
3242 },
3202 .entry_ip = entry_ip,3243 .entry_ip = entry_ip,
3203 .uprobe = uprobe,3244 .uprobe = uprobe,
3204 };3245 };
3205 struct bpf_prog *prog = link->link.prog;3246 struct bpf_prog *prog = link->link.prog;
3206 bool sleepable = prog->sleepable;3247 bool sleepable = prog->sleepable;
3207 struct bpf_run_ctx *old_run_ctx;3248 struct bpf_run_ctx *old_run_ctx;
3208 int err = 0;3249 int err;
3209 3250
3210 if (link->task && !same_thread_group(current, link->task))3251 if (link->task && !same_thread_group(current, link->task))
3211 return 0;3252 return 0;
@@ -3217,7 +3258,7 @@ static int uprobe_prog_run(struct bpf_uprobe *uprobe, · 5 unmodified lines
3217 3258
3218 migrate_disable();3259 migrate_disable();
3219 3260
3220 old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);3261 old_run_ctx = bpf_set_run_ctx(&run_ctx.session_ctx.run_ctx);
3221 err = bpf_prog_run(link->link.prog, regs);3262 err = bpf_prog_run(link->link.prog, regs);
3222 bpf_reset_run_ctx(old_run_ctx);3263 bpf_reset_run_ctx(old_run_ctx);
3223 3264
@@ -3244,9 +3285,13 @@ uprobe_multi_link_handler(struct uprobe_consumer *con, struct pt_regs *regs, · 20 unmodified lines
3244 __u64 *data)3285 __u64 *data)
3245{3286{
3246 struct bpf_uprobe *uprobe;3287 struct bpf_uprobe *uprobe;
3288 int ret;
3247 3289
3248 uprobe = container_of(con, struct bpf_uprobe, consumer);3290 uprobe = container_of(con, struct bpf_uprobe, consumer);
3249 return uprobe_prog_run(uprobe, instruction_pointer(regs), regs);3291 ret = uprobe_prog_run(uprobe, instruction_pointer(regs), regs, false, data);
3292 if (uprobe->session)
3293 return ret ? UPROBE_HANDLER_IGNORE : 0;
3294 return 0;
3250}3295}
3251 3296
3252static int3297static int
@@ -3256,14 +3301,16 @@ uprobe_multi_link_ret_handler(struct uprobe_consumer *con, unsigned long func, s · 3 unmodified lines
3256 struct bpf_uprobe *uprobe;3301 struct bpf_uprobe *uprobe;
3257 3302
3258 uprobe = container_of(con, struct bpf_uprobe, consumer);3303 uprobe = container_of(con, struct bpf_uprobe, consumer);
3259 return uprobe_prog_run(uprobe, func, regs);3304 uprobe_prog_run(uprobe, func, regs, true, data);
3305 return 0;
3260}3306}
3261 3307
3262static u64 bpf_uprobe_multi_entry_ip(struct bpf_run_ctx *ctx)3308static u64 bpf_uprobe_multi_entry_ip(struct bpf_run_ctx *ctx)
3263{3309{
3264 struct bpf_uprobe_multi_run_ctx *run_ctx;3310 struct bpf_uprobe_multi_run_ctx *run_ctx;
3265 3311
3266 run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx, run_ctx);3312 run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx,
3313 session_ctx.run_ctx);
3267 return run_ctx->entry_ip;3314 return run_ctx->entry_ip;
3268}3315}
3269 3316
@@ -3271,7 +3318,8 @@ static u64 bpf_uprobe_multi_cookie(struct bpf_run_ctx *ctx) · 1 unmodified line
3271{3318{
3272 struct bpf_uprobe_multi_run_ctx *run_ctx;3319 struct bpf_uprobe_multi_run_ctx *run_ctx;
3273 3320
3274 run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx, run_ctx);3321 run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx,
3322 session_ctx.run_ctx);
3275 return run_ctx->uprobe->cookie;3323 return run_ctx->uprobe->cookie;
3276}3324}
3277 3325
@@ -3295,7 +3343,7 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr · 17 unmodified lines
3295 if (sizeof(u64) != sizeof(void *))3343 if (sizeof(u64) != sizeof(void *))
3296 return -EOPNOTSUPP;3344 return -EOPNOTSUPP;
3297 3345
3298 if (prog->expected_attach_type != BPF_TRACE_UPROBE_MULTI)3346 if (!is_uprobe_multi(prog))
3299 return -EINVAL;3347 return -EINVAL;
3300 3348
3301 flags = attr->link_create.uprobe_multi.flags;3349 flags = attr->link_create.uprobe_multi.flags;
@@ -3371,11 +3419,12 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr · 69 unmodified lines
3371 3419
3372 uprobes[i].link = link;3420 uprobes[i].link = link;
3373 3421
3374 if (flags & BPF_F_UPROBE_MULTI_RETURN)3422 if (!(flags & BPF_F_UPROBE_MULTI_RETURN))
3375 uprobes[i].consumer.ret_handler = uprobe_multi_link_ret_handler;
3376 else
3377 uprobes[i].consumer.handler = uprobe_multi_link_handler;3423 uprobes[i].consumer.handler = uprobe_multi_link_handler;
3378 3424 if (flags & BPF_F_UPROBE_MULTI_RETURN || is_uprobe_session(prog))
3425 uprobes[i].consumer.ret_handler = uprobe_multi_link_ret_handler;
3426 if (is_uprobe_session(prog))
3427 uprobes[i].session = true;
3379 if (pid)3428 if (pid)
3380 uprobes[i].consumer.filter = uprobe_multi_link_filter;3429 uprobes[i].consumer.filter = uprobe_multi_link_filter;
3381 }3430 }
@@ -3464,7 +3513,7 @@ static int bpf_kprobe_multi_filter(const struct bpf_prog *prog, u32 kfunc_id) · 82 unmodified lines
3464 if (!btf_id_set8_contains(&kprobe_multi_kfunc_set_ids, kfunc_id))3513 if (!btf_id_set8_contains(&kprobe_multi_kfunc_set_ids, kfunc_id))
3465 return 0;3514 return 0;
3466 3515
3467 if (!is_kprobe_session(prog))3516 if (!is_kprobe_session(prog) && !is_uprobe_session(prog))
3468 return -EACCES;3517 return -EACCES;
3469 3518
3470 return 0;3519 return 0;
@@ -3482,3 +3531,16 @@ static int __init bpf_kprobe_multi_kfuncs_init(void) · 11 unmodified lines
3482}3531}
3483 3532
3484late_initcall(bpf_kprobe_multi_kfuncs_init);3533late_initcall(bpf_kprobe_multi_kfuncs_init);
3534
3535__bpf_kfunc_start_defs();
3536
3537__bpf_kfunc int bpf_send_signal_task(struct task_struct *task, int sig, enum pid_type type,
3538 u64 value)
3539{
3540 if (type != PIDTYPE_PID && type != PIDTYPE_TGID)
3541 return -EINVAL;
3542
3543 return bpf_send_signal_common(sig, type, task, value);
3544}
3545
3546__bpf_kfunc_end_defs();