Merge tag 'bpf-next-6.13' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Pull bpf updates from Alexei Starovoitov: - Add BPF uprobe session support (Jiri Olsa) - Optimize uprobe performance (Andrii Nakryiko) - Add bpf_fastcall support to helpers and kfuncs (Eduard Zingerman) - Avoid calling free_htab_elem() under hash map bucket lock (Hou Tao) - Prevent tailcall infinite loop caused by freplace (Leon Hwang) - Mark raw_tracepoint arguments as nullable (Kumar Kartikeya Dwivedi) - Introduce uptr support in the task local storage map (Martin KaFai Lau) - Stringify errno log messages in libbpf (Mykyta Yatsenko) - Add kmem_cache BPF iterator for perf's lock profiling (Namhyung Kim) - Support BPF objects of either endianness in libbpf (Tony Ambardar) - Add ksym to struct_ops trampoline to fix stack trace (Xu Kuohai) - Introduce private stack for eligible BPF programs (Yonghong Song) - Migrate samples/bpf tests to selftests/bpf test_progs (Daniel T. Lee) - Migrate test_sock to selftests/bpf test_progs (Jordan Rife) * tag 'bpf-next-6.13' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next: (152 commits) libbpf: Change hash_combine parameters from long to unsigned long selftests/bpf: Fix build error with llvm 19 libbpf: Fix memory leak in bpf_program__attach_uprobe_multi bpf: use common instruction history across all states bpf: Add necessary migrate_disable to range_tree. bpf: Do not alloc arena on unsupported arches selftests/bpf: Set test path for token/obj_priv_implicit_token_envvar selftests/bpf: Add a test for arena range tree algorithm bpf: Introduce range_tree data structure and use it in bpf arena samples/bpf: Remove unused variable in xdp2skb_meta_kern.c samples/bpf: Remove unused variables in tc_l2_redirect_kern.c bpftool: Cast variable `var` to long long bpf, x86: Propagate tailcall info only for subprogs bpf: Add kernel symbol for struct_ops trampoline bpf: Use function pointers count as struct_ops links count bpf: Remove unused member rcu from bpf_struct_ops_map selftests/bpf: Add struct_ops prog private stack tests bpf: Support private stack for struct_ops progs selftests/bpf: Add tracing prog private stack tests bpf, x86: Support private stack in jit ...

Linus Torvalds committed 1 year ago

Commit 6e95ef02 parents 43fb83c 2c8b09a211 files · +6961 −3473Browse files

Showing the changes to kernel/trace (1 of 211 files). Show all

Unified · Split

kernel/trace/bpf_trace.c+86−24

@@ -802,6 +802,8 @@ struct send_signal_irq_work { · 801 unmodified lines
802	802	struct task_struct *task;
803	803	u32 sig;
804	804	enum pid_type type;
	805	+ bool has_siginfo;
	806	+ struct kernel_siginfo info;
805	807	};
806	808
807	809	static DEFINE_PER_CPU(struct send_signal_irq_work, send_signal_work);
@@ -809,27 +811,46 @@ static DEFINE_PER_CPU(struct send_signal_irq_work, send_signal_work); · 1 unmodified line
809	811	static void do_bpf_send_signal(struct irq_work *entry)
810	812	{
811	813	struct send_signal_irq_work *work;
	814	+ struct kernel_siginfo *siginfo;
812	815
813	816	work = container_of(entry, struct send_signal_irq_work, irq_work);
814		− group_send_sig_info(work->sig, SEND_SIG_PRIV, work->task, work->type);
	817	+ siginfo = work->has_siginfo ? &work->info : SEND_SIG_PRIV;
	818	+
	819	+ group_send_sig_info(work->sig, siginfo, work->task, work->type);
815	820	put_task_struct(work->task);
816	821	}
817	822
818		−static int bpf_send_signal_common(u32 sig, enum pid_type type)
	823	+static int bpf_send_signal_common(u32 sig, enum pid_type type, struct task_struct *task, u64 value)
819	824	{
820	825	struct send_signal_irq_work *work = NULL;
	826	+ struct kernel_siginfo info;
	827	+ struct kernel_siginfo *siginfo;
	828	+
	829	+ if (!task) {
	830	+ task = current;
	831	+ siginfo = SEND_SIG_PRIV;
	832	+ } else {
	833	+ clear_siginfo(&info);
	834	+ info.si_signo = sig;
	835	+ info.si_errno = 0;
	836	+ info.si_code = SI_KERNEL;
	837	+ info.si_pid = 0;
	838	+ info.si_uid = 0;
	839	+ info.si_value.sival_ptr = (void *)(unsigned long)value;
	840	+ siginfo = &info;
	841	+ }
821	842
822	843	/* Similar to bpf_probe_write_user, task needs to be
823	844	* in a sound condition and kernel memory access be
824	845	* permitted in order to send signal to the current
825	846	* task.
826	847	*/
827		− if (unlikely(current->flags & (PF_KTHREAD \| PF_EXITING)))
	848	+ if (unlikely(task->flags & (PF_KTHREAD \| PF_EXITING)))
828	849	return -EPERM;
829	850	if (unlikely(!nmi_uaccess_okay()))
830	851	return -EPERM;
831	852	/* Task should not be pid=1 to avoid kernel panic. */
832		− if (unlikely(is_global_init(current)))
	853	+ if (unlikely(is_global_init(task)))
833	854	return -EPERM;
834	855
835	856	if (irqs_disabled()) {
@@ -847,19 +868,22 @@ static int bpf_send_signal_common(u32 sig, enum pid_type type) · 11 unmodified lines
847	868	* to the irq_work. The current task may change when queued
848	869	* irq works get executed.
849	870	*/
850		− work->task = get_task_struct(current);
	871	+ work->task = get_task_struct(task);
	872	+ work->has_siginfo = siginfo == &info;
	873	+ if (work->has_siginfo)
	874	+ copy_siginfo(&work->info, &info);
851	875	work->sig = sig;
852	876	work->type = type;
853	877	irq_work_queue(&work->irq_work);
854	878	return 0;
855	879	}
856	880
857		− return group_send_sig_info(sig, SEND_SIG_PRIV, current, type);
	881	+ return group_send_sig_info(sig, siginfo, task, type);
858	882	}
859	883
860	884	BPF_CALL_1(bpf_send_signal, u32, sig)
861	885	{
862		− return bpf_send_signal_common(sig, PIDTYPE_TGID);
	886	+ return bpf_send_signal_common(sig, PIDTYPE_TGID, NULL, 0);
863	887	}
864	888
865	889	static const struct bpf_func_proto bpf_send_signal_proto = {
@@ -871,7 +895,7 @@ static const struct bpf_func_proto bpf_send_signal_proto = { · 5 unmodified lines
871	895
872	896	BPF_CALL_1(bpf_send_signal_thread, u32, sig)
873	897	{
874		− return bpf_send_signal_common(sig, PIDTYPE_PID);
	898	+ return bpf_send_signal_common(sig, PIDTYPE_PID, NULL, 0);
875	899	}
876	900
877	901	static const struct bpf_func_proto bpf_send_signal_thread_proto = {
@@ -1557,6 +1581,17 @@ static inline bool is_kprobe_session(const struct bpf_prog *prog) · 679 unmodified lines
1557	1581	return prog->expected_attach_type == BPF_TRACE_KPROBE_SESSION;
1558	1582	}
1559	1583
	1584	+static inline bool is_uprobe_multi(const struct bpf_prog *prog)
	1585	+{
	1586	+ return prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI \|\|
	1587	+ prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION;
	1588	+}
	1589	+
	1590	+static inline bool is_uprobe_session(const struct bpf_prog *prog)
	1591	+{
	1592	+ return prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION;
	1593	+}
	1594	+
1560	1595	static const struct bpf_func_proto *
1561	1596	kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1562	1597	{
@@ -1574,13 +1609,13 @@ kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 11 unmodified lines
1574	1609	case BPF_FUNC_get_func_ip:
1575	1610	if (is_kprobe_multi(prog))
1576	1611	return &bpf_get_func_ip_proto_kprobe_multi;
1577		− if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI)
	1612	+ if (is_uprobe_multi(prog))
1578	1613	return &bpf_get_func_ip_proto_uprobe_multi;
1579	1614	return &bpf_get_func_ip_proto_kprobe;
1580	1615	case BPF_FUNC_get_attach_cookie:
1581	1616	if (is_kprobe_multi(prog))
1582	1617	return &bpf_get_attach_cookie_proto_kmulti;
1583		− if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI)
	1618	+ if (is_uprobe_multi(prog))
1584	1619	return &bpf_get_attach_cookie_proto_umulti;
1585	1620	return &bpf_get_attach_cookie_proto_trace;
1586	1621	default:
@@ -3072,6 +3107,7 @@ struct bpf_uprobe { · 1485 unmodified lines
3072	3107	u64 cookie;
3073	3108	struct uprobe *uprobe;
3074	3109	struct uprobe_consumer consumer;
	3110	+ bool session;
3075	3111	};
3076	3112
3077	3113	struct bpf_uprobe_multi_link {
@@ -3084,7 +3120,7 @@ struct bpf_uprobe_multi_link { · 6 unmodified lines
3084	3120	};
3085	3121
3086	3122	struct bpf_uprobe_multi_run_ctx {
3087		− struct bpf_run_ctx run_ctx;
	3123	+ struct bpf_session_run_ctx session_ctx;
3088	3124	unsigned long entry_ip;
3089	3125	struct bpf_uprobe *uprobe;
3090	3126	};
@@ -3195,17 +3231,22 @@ static const struct bpf_link_ops bpf_uprobe_multi_link_lops = { · 104 unmodified lines
3195	3231
3196	3232	static int uprobe_prog_run(struct bpf_uprobe *uprobe,
3197	3233	unsigned long entry_ip,
3198		− struct pt_regs *regs)
	3234	+ struct pt_regs *regs,
	3235	+ bool is_return, void *data)
3199	3236	{
3200	3237	struct bpf_uprobe_multi_link *link = uprobe->link;
3201	3238	struct bpf_uprobe_multi_run_ctx run_ctx = {
	3239	+ .session_ctx = {
	3240	+ .is_return = is_return,
	3241	+ .data = data,
	3242	+ },
3202	3243	.entry_ip = entry_ip,
3203	3244	.uprobe = uprobe,
3204	3245	};
3205	3246	struct bpf_prog *prog = link->link.prog;
3206	3247	bool sleepable = prog->sleepable;
3207	3248	struct bpf_run_ctx *old_run_ctx;
3208		− int err = 0;
	3249	+ int err;
3209	3250
3210	3251	if (link->task && !same_thread_group(current, link->task))
3211	3252	return 0;
@@ -3217,7 +3258,7 @@ static int uprobe_prog_run(struct bpf_uprobe *uprobe, · 5 unmodified lines
3217	3258
3218	3259	migrate_disable();
3219	3260
3220		− old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
	3261	+ old_run_ctx = bpf_set_run_ctx(&run_ctx.session_ctx.run_ctx);
3221	3262	err = bpf_prog_run(link->link.prog, regs);
3222	3263	bpf_reset_run_ctx(old_run_ctx);
3223	3264
@@ -3244,9 +3285,13 @@ uprobe_multi_link_handler(struct uprobe_consumer con, struct pt_regs regs, · 20 unmodified lines
3244	3285	__u64 *data)
3245	3286	{
3246	3287	struct bpf_uprobe *uprobe;
	3288	+ int ret;
3247	3289
3248	3290	uprobe = container_of(con, struct bpf_uprobe, consumer);
3249		− return uprobe_prog_run(uprobe, instruction_pointer(regs), regs);
	3291	+ ret = uprobe_prog_run(uprobe, instruction_pointer(regs), regs, false, data);
	3292	+ if (uprobe->session)
	3293	+ return ret ? UPROBE_HANDLER_IGNORE : 0;
	3294	+ return 0;
3250	3295	}
3251	3296
3252	3297	static int
@@ -3256,14 +3301,16 @@ uprobe_multi_link_ret_handler(struct uprobe_consumer *con, unsigned long func, s · 3 unmodified lines
3256	3301	struct bpf_uprobe *uprobe;
3257	3302
3258	3303	uprobe = container_of(con, struct bpf_uprobe, consumer);
3259		− return uprobe_prog_run(uprobe, func, regs);
	3304	+ uprobe_prog_run(uprobe, func, regs, true, data);
	3305	+ return 0;
3260	3306	}
3261	3307
3262	3308	static u64 bpf_uprobe_multi_entry_ip(struct bpf_run_ctx *ctx)
3263	3309	{
3264	3310	struct bpf_uprobe_multi_run_ctx *run_ctx;
3265	3311
3266		− run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx, run_ctx);
	3312	+ run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx,
	3313	+ session_ctx.run_ctx);
3267	3314	return run_ctx->entry_ip;
3268	3315	}
3269	3316
@@ -3271,7 +3318,8 @@ static u64 bpf_uprobe_multi_cookie(struct bpf_run_ctx *ctx) · 1 unmodified line
3271	3318	{
3272	3319	struct bpf_uprobe_multi_run_ctx *run_ctx;
3273	3320
3274		− run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx, run_ctx);
	3321	+ run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx,
	3322	+ session_ctx.run_ctx);
3275	3323	return run_ctx->uprobe->cookie;
3276	3324	}
3277	3325
@@ -3295,7 +3343,7 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr attr, struct bpf_prog pr · 17 unmodified lines
3295	3343	if (sizeof(u64) != sizeof(void *))
3296	3344	return -EOPNOTSUPP;
3297	3345
3298		− if (prog->expected_attach_type != BPF_TRACE_UPROBE_MULTI)
	3346	+ if (!is_uprobe_multi(prog))
3299	3347	return -EINVAL;
3300	3348
3301	3349	flags = attr->link_create.uprobe_multi.flags;
@@ -3371,11 +3419,12 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr attr, struct bpf_prog pr · 69 unmodified lines
3371	3419
3372	3420	uprobes[i].link = link;
3373	3421
3374		− if (flags & BPF_F_UPROBE_MULTI_RETURN)
3375		− uprobes[i].consumer.ret_handler = uprobe_multi_link_ret_handler;
3376		− else
	3422	+ if (!(flags & BPF_F_UPROBE_MULTI_RETURN))
3377	3423	uprobes[i].consumer.handler = uprobe_multi_link_handler;
3378		−
	3424	+ if (flags & BPF_F_UPROBE_MULTI_RETURN \|\| is_uprobe_session(prog))
	3425	+ uprobes[i].consumer.ret_handler = uprobe_multi_link_ret_handler;
	3426	+ if (is_uprobe_session(prog))
	3427	+ uprobes[i].session = true;
3379	3428	if (pid)
3380	3429	uprobes[i].consumer.filter = uprobe_multi_link_filter;
3381	3430	}
@@ -3464,7 +3513,7 @@ static int bpf_kprobe_multi_filter(const struct bpf_prog *prog, u32 kfunc_id) · 82 unmodified lines
3464	3513	if (!btf_id_set8_contains(&kprobe_multi_kfunc_set_ids, kfunc_id))
3465	3514	return 0;
3466	3515
3467		− if (!is_kprobe_session(prog))
	3516	+ if (!is_kprobe_session(prog) && !is_uprobe_session(prog))
3468	3517	return -EACCES;
3469	3518
3470	3519	return 0;
@@ -3482,3 +3531,16 @@ static int __init bpf_kprobe_multi_kfuncs_init(void) · 11 unmodified lines
3482	3531	}
3483	3532
3484	3533	late_initcall(bpf_kprobe_multi_kfuncs_init);
	3534	+
	3535	+__bpf_kfunc_start_defs();
	3536	+
	3537	+__bpf_kfunc int bpf_send_signal_task(struct task_struct *task, int sig, enum pid_type type,
	3538	+ u64 value)
	3539	+{
	3540	+ if (type != PIDTYPE_PID && type != PIDTYPE_TGID)
	3541	+ return -EINVAL;
	3542	+
	3543	+ return bpf_send_signal_common(sig, type, task, value);
	3544	+}
	3545	+
	3546	+__bpf_kfunc_end_defs();