bpf: Use mutex lock pool for bpf trampolines

Adding mutex lock pool that replaces bpf trampolines mutex. For tracing_multi link coming in following changes we need to lock all the involved trampolines during the attachment. This could mean thousands of mutex locks, which is not convenient. As suggested by Andrii we can replace bpf trampolines mutex with mutex pool, where each trampoline is hash-ed to one of the locks from the pool. It's better to lock all the pool mutexes (32 at the moment) than thousands of them. There is 48 (MAX_LOCK_DEPTH) lock limit allowed to be simultaneously held by task, so we need to keep 32 mutexes (5 bits) in the pool, so when we lock them all in following changes the lockdep won't scream. Removing the mutex_is_locked in bpf_trampoline_put, because we removed the mutex from bpf_trampoline.

Jiri Olsa committed 5 months ago

Commit e6abd4c1 parent 2cd298c2 files · +53 −26Browse files

Showing the changes to kernel (1 of 2 files). Show all

Unified · Split

kernel/bpf/trampoline.c+53−24

@@ -30,6 +30,35 @@ static struct hlist_head trampoline_ip_table[TRAMPOLINE_TABLE_SIZE]; · 29 unmodified lines
30	30	/* serializes access to trampoline tables */
31	31	static DEFINE_MUTEX(trampoline_mutex);
32	32
	33	+/*
	34	+ * Keep 32 trampoline locks (5 bits) in the pool so trampoline_lock_all()
	35	+ * stays below MAX_LOCK_DEPTH. Each pool slot has a distinct lockdep
	36	+ * class because trampoline_lock_all() takes all pool mutexes at once;
	37	+ * otherwise lockdep would report recursive locking on same-class mutexes.
	38	+ */
	39	+#define TRAMPOLINE_LOCKS_BITS 5
	40	+#define TRAMPOLINE_LOCKS_TABLE_SIZE (1 << TRAMPOLINE_LOCKS_BITS)
	41	+
	42	+static struct {
	43	+ struct mutex mutex;
	44	+ struct lock_class_key key;
	45	+} trampoline_locks[TRAMPOLINE_LOCKS_TABLE_SIZE];
	46	+
	47	+static struct mutex select_trampoline_lock(struct bpf_trampoline tr)
	48	+{
	49	+ return &trampoline_locks[hash_ptr(tr, TRAMPOLINE_LOCKS_BITS)].mutex;
	50	+}
	51	+
	52	+static void trampoline_lock(struct bpf_trampoline *tr)
	53	+{
	54	+ mutex_lock(select_trampoline_lock(tr));
	55	+}
	56	+
	57	+static void trampoline_unlock(struct bpf_trampoline *tr)
	58	+{
	59	+ mutex_unlock(select_trampoline_lock(tr));
	60	+}
	61	+
33	62	#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
34	63	static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mutex);
35	64
@@ -69,9 +98,9 @@ static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, unsigned long ip, · 33 unmodified lines
69	98
70	99	if (cmd == FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF) {
71	100	/* This is called inside register_ftrace_direct_multi(), so
72		− * tr->mutex is already locked.
	101	+ * trampoline's mutex is already locked.
73	102	*/
74		− lockdep_assert_held_once(&tr->mutex);
	103	+ lockdep_assert_held_once(select_trampoline_lock(tr));
75	104
76	105	/* Instead of updating the trampoline here, we propagate
77	106	* -EAGAIN to register_ftrace_direct(). Then we can
@@ -91,7 +120,7 @@ static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, unsigned long ip, · 13 unmodified lines
91	120	}
92	121
93	122	/* The normal locking order is
94		− * tr->mutex => direct_mutex (ftrace.c) => ftrace_lock (ftrace.c)
	123	+ * select_trampoline_lock(tr) => direct_mutex (ftrace.c) => ftrace_lock (ftrace.c)
95	124	*
96	125	* The following two commands are called from
97	126	*
@@ -99,12 +128,12 @@ static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, unsigned long ip, · 1 unmodified line
99	128	* cleanup_direct_functions_after_ipmodify
100	129	*
101	130	* In both cases, direct_mutex is already locked. Use
102		− * mutex_trylock(&tr->mutex) to avoid deadlock in race condition
103		− * (something else is making changes to this same trampoline).
	131	+ * mutex_trylock(select_trampoline_lock(tr)) to avoid deadlock in race condition
	132	+ * (something else holds the same pool lock).
104	133	*/
105		− if (!mutex_trylock(&tr->mutex)) {
106		− /* sleep 1 ms to make sure whatever holding tr->mutex makes
107		− * some progress.
	134	+ if (!mutex_trylock(select_trampoline_lock(tr))) {
	135	+ /* sleep 1 ms to make sure whatever holding select_trampoline_lock(tr)
	136	+ * makes some progress.
108	137	*/
109	138	msleep(1);
110	139	return -EAGAIN;
@@ -129,7 +158,7 @@ static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, unsigned long ip, · 18 unmodified lines
129	158	break;
130	159	}
131	160
132		− mutex_unlock(&tr->mutex);
	161	+ trampoline_unlock(tr);
133	162	return ret;
134	163	}
135	164	#endif
@@ -359,7 +388,6 @@ static struct bpf_trampoline *bpf_trampoline_lookup(u64 key, unsigned long ip) · 223 unmodified lines
359	388	head = &trampoline_ip_table[hash_64(tr->ip, TRAMPOLINE_HASH_BITS)];
360	389	hlist_add_head(&tr->hlist_ip, head);
361	390	refcount_set(&tr->refcnt, 1);
362		− mutex_init(&tr->mutex);
363	391	for (i = 0; i < BPF_TRAMP_MAX; i++)
364	392	INIT_HLIST_HEAD(&tr->progs_hlist[i]);
365	393	out:
@@ -843,9 +871,9 @@ int bpf_trampoline_link_prog(struct bpf_tramp_link *link, · 477 unmodified lines
843	871	{
844	872	int err;
845	873
846		− mutex_lock(&tr->mutex);
	874	+ trampoline_lock(tr);
847	875	err = __bpf_trampoline_link_prog(link, tr, tgt_prog);
848		− mutex_unlock(&tr->mutex);
	876	+ trampoline_unlock(tr);
849	877	return err;
850	878	}
851	879
@@ -886,9 +914,9 @@ int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, · 34 unmodified lines
886	914	{
887	915	int err;
888	916
889		− mutex_lock(&tr->mutex);
	917	+ trampoline_lock(tr);
890	918	err = __bpf_trampoline_unlink_prog(link, tr, tgt_prog);
891		− mutex_unlock(&tr->mutex);
	919	+ trampoline_unlock(tr);
892	920	return err;
893	921	}
894	922
@@ -998,12 +1026,12 @@ int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, · 103 unmodified lines
998	1026	if (!tr)
999	1027	return -ENOMEM;
1000	1028
1001		− mutex_lock(&tr->mutex);
	1029	+ trampoline_lock(tr);
1002	1030
1003	1031	shim_link = cgroup_shim_find(tr, bpf_func);
1004	1032	if (shim_link && !IS_ERR(bpf_link_inc_not_zero(&shim_link->link.link))) {
1005	1033	/* Reusing existing shim attached by the other program. */
1006		− mutex_unlock(&tr->mutex);
	1034	+ trampoline_unlock(tr);
1007	1035	bpf_trampoline_put(tr); /* bpf_trampoline_get above */
1008	1036	return 0;
1009	1037	}
@@ -1023,16 +1051,16 @@ int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, · 13 unmodified lines
1023	1051	shim_link->trampoline = tr;
1024	1052	/* note, we're still holding tr refcnt from above */
1025	1053
1026		− mutex_unlock(&tr->mutex);
	1054	+ trampoline_unlock(tr);
1027	1055
1028	1056	return 0;
1029	1057	err:
1030		− mutex_unlock(&tr->mutex);
	1058	+ trampoline_unlock(tr);
1031	1059
1032	1060	if (shim_link)
1033	1061	bpf_link_put(&shim_link->link.link);
1034	1062
1035		− /* have to release tr while _not_ holding its mutex */
	1063	+ /* have to release tr while _not_ holding pool mutex for trampoline */
1036	1064	bpf_trampoline_put(tr); /* bpf_trampoline_get above */
1037	1065
1038	1066	return err;
@@ -1053,9 +1081,9 @@ void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog) · 14 unmodified lines
1053	1081	if (WARN_ON_ONCE(!tr))
1054	1082	return;
1055	1083
1056		− mutex_lock(&tr->mutex);
	1084	+ trampoline_lock(tr);
1057	1085	shim_link = cgroup_shim_find(tr, bpf_func);
1058		− mutex_unlock(&tr->mutex);
	1086	+ trampoline_unlock(tr);
1059	1087
1060	1088	if (shim_link)
1061	1089	bpf_link_put(&shim_link->link.link);
@@ -1073,14 +1101,14 @@ struct bpf_trampoline *bpf_trampoline_get(u64 key, · 11 unmodified lines
1073	1101	if (!tr)
1074	1102	return NULL;
1075	1103
1076		− mutex_lock(&tr->mutex);
	1104	+ trampoline_lock(tr);
1077	1105	if (tr->func.addr)
1078	1106	goto out;
1079	1107
1080	1108	memcpy(&tr->func.model, &tgt_info->fmodel, sizeof(tgt_info->fmodel));
1081	1109	tr->func.addr = (void *)tgt_info->tgt_addr;
1082	1110	out:
1083		− mutex_unlock(&tr->mutex);
	1111	+ trampoline_unlock(tr);
1084	1112	return tr;
1085	1113	}
1086	1114
@@ -1093,7 +1121,6 @@ void bpf_trampoline_put(struct bpf_trampoline *tr) · 6 unmodified lines
1093	1121	mutex_lock(&trampoline_mutex);
1094	1122	if (!refcount_dec_and_test(&tr->refcnt))
1095	1123	goto out;
1096		− WARN_ON_ONCE(mutex_is_locked(&tr->mutex));
1097	1124
1098	1125	for (i = 0; i < BPF_TRAMP_MAX; i++)
1099	1126	if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[i])))
@@ -1379,6 +1406,8 @@ static int __init init_trampolines(void) · 279 unmodified lines
1379	1406	INIT_HLIST_HEAD(&trampoline_key_table[i]);
1380	1407	for (i = 0; i < TRAMPOLINE_TABLE_SIZE; i++)
1381	1408	INIT_HLIST_HEAD(&trampoline_ip_table[i]);
	1409	+ for (i = 0; i < TRAMPOLINE_LOCKS_TABLE_SIZE; i++)
	1410	+ __mutex_init(&trampoline_locks[i].mutex, "trampoline_lock", &trampoline_locks[i].key);
1382	1411	return 0;
1383	1412	}
1384	1413	late_initcall(init_trampolines);