History

bpf: Fix NULL pointer dereference in bpf_get_local_storage() helper

Jiri Olsa reported a bug ([1]) in kernel where cgroup local storage pointer may be NULL in bpf_get_local_storage() helper. There are two issues uncovered by this bug: (1). kprobe or tracepoint prog incorrectly sets cgroup local storage before prog run, (2). due to change from preempt_disable to migrate_disable, preemption is possible and percpu storage might be overwritten by other tasks. This issue (1) is fixed in [2]. This patch tried to address issue (2). The following shows how things can go wrong: task 1: bpf_cgroup_storage_set() for percpu local storage preemption happens task 2: bpf_cgroup_storage_set() for percpu local storage preemption happens task 1: run bpf program task 1 will effectively use the percpu local storage setting by task 2 which will be either NULL or incorrect ones. Instead of just one common local storage per cpu, this patch fixed the issue by permitting 8 local storages per cpu and each local storage is identified by a task_struct pointer. This way, we allow at most 8 nested preemption between bpf_cgroup_storage_set() and bpf_cgroup_storage_unset(). The percpu local storage slot is released (calling bpf_cgroup_storage_unset()) by the same task after bpf program finished running. bpf_test_run() is also fixed to use the new bpf_cgroup_storage_set() interface. The patch is tested on top of [2] with reproducer in [1]. Without this patch, kernel will emit error in 2-3 minutes. With this patch, after one hour, still no error. [1] https://lore.kernel.org/bpf/CAKH8qBuXCfUz=w8L+Fj74OaUpbosO29niYwTki7e3Ag044_aww@mail.gmail.com/T [2] https://lore.kernel.org/bpf/20210309185028.3763817-1-yhs@fb.com

Yonghong Song committed 5 years ago

Commit b910eaa1 parent a46410d5 files · +86 −19Browse files

Unified · Split

5 files
  1. include/linux/bpf-cgroup.h +49 −8
  2. include/linux/bpf.h +18 −4
  3. kernel/bpf/helpers.c +11 −4
  4. kernel/bpf/local_storage.c +3 −2
  5. net/bpf/test_run.c +5 −1
include/linux/bpf-cgroup.h+49−8
@@ -20,14 +20,25 @@ struct bpf_sock_ops_kern; · 19 unmodified lines
2020 struct bpf_cgroup_storage;
2121 struct ctl_table;
2222 struct ctl_table_header;
23+struct task_struct;
2324
2425 #ifdef CONFIG_CGROUP_BPF
2526
2627 extern struct static_key_false cgroup_bpf_enabled_key[MAX_BPF_ATTACH_TYPE];
2728 #define cgroup_bpf_enabled(type) static_branch_unlikely(&cgroup_bpf_enabled_key[type])
2829
29−DECLARE_PER_CPU(struct bpf_cgroup_storage*,
30− bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);
30+#define BPF_CGROUP_STORAGE_NEST_MAX 8
31+
32+struct bpf_cgroup_storage_info {
33+ struct task_struct *task;
34+ struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE];
35+};
36+
37+/* For each cpu, permit maximum BPF_CGROUP_STORAGE_NEST_MAX number of tasks
38+ * to use bpf cgroup storage simultaneously.
39+ */
40+DECLARE_PER_CPU(struct bpf_cgroup_storage_info,
41+ bpf_cgroup_storage_info[BPF_CGROUP_STORAGE_NEST_MAX]);
3142
3243 #define for_each_cgroup_storage_type(stype) \
3344 for (stype = 0; stype < MAX_BPF_CGROUP_STORAGE_TYPE; stype++)
@@ -161,13 +172,42 @@ static inline enum bpf_cgroup_storage_type cgroup_storage_type( · 127 unmodified lines
161172 return BPF_CGROUP_STORAGE_SHARED;
162173 }
163174
164−static inline void bpf_cgroup_storage_set(struct bpf_cgroup_storage
165− *storage[MAX_BPF_CGROUP_STORAGE_TYPE])
175+static inline int bpf_cgroup_storage_set(struct bpf_cgroup_storage
176+ *storage[MAX_BPF_CGROUP_STORAGE_TYPE])
166177 {
167178 enum bpf_cgroup_storage_type stype;
179+ int i, err = 0;
180+
181+ preempt_disable();
182+ for (i = 0; i < BPF_CGROUP_STORAGE_NEST_MAX; i++) {
183+ if (unlikely(this_cpu_read(bpf_cgroup_storage_info[i].task) != NULL))
184+ continue;
185+
186+ this_cpu_write(bpf_cgroup_storage_info[i].task, current);
187+ for_each_cgroup_storage_type(stype)
188+ this_cpu_write(bpf_cgroup_storage_info[i].storage[stype],
189+ storage[stype]);
190+ goto out;
191+ }
192+ err = -EBUSY;
193+ WARN_ON_ONCE(1);
194+
195+out:
196+ preempt_enable();
197+ return err;
198+}
199+
200+static inline void bpf_cgroup_storage_unset(void)
201+{
202+ int i;
203+
204+ for (i = 0; i < BPF_CGROUP_STORAGE_NEST_MAX; i++) {
205+ if (unlikely(this_cpu_read(bpf_cgroup_storage_info[i].task) != current))
206+ continue;
168207
169− for_each_cgroup_storage_type(stype)
170− this_cpu_write(bpf_cgroup_storage[stype], storage[stype]);
208+ this_cpu_write(bpf_cgroup_storage_info[i].task, NULL);
209+ return;
210+ }
171211 }
172212
173213 struct bpf_cgroup_storage *
@@ -448,8 +488,9 @@ static inline int cgroup_bpf_prog_query(const union bpf_attr *attr, · 274 unmodified lines
448488 return -EINVAL;
449489 }
450490
451−static inline void bpf_cgroup_storage_set(
452− struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE]) {}
491+static inline int bpf_cgroup_storage_set(
492+ struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE]) { return 0; }
493+static inline void bpf_cgroup_storage_unset(void) {}
453494 static inline int bpf_cgroup_storage_assign(struct bpf_prog_aux *aux,
454495 struct bpf_map *map) { return 0; }
455496 static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(
include/linux/bpf.h+18−4
@@ -1106,6 +1106,13 @@ int bpf_prog_array_copy(struct bpf_prog_array *old_array, · 1105 unmodified lines
11061106 /* BPF program asks to set CN on the packet. */
11071107 #define BPF_RET_SET_CN (1 << 0)
11081108
1109+/* For BPF_PROG_RUN_ARRAY_FLAGS and __BPF_PROG_RUN_ARRAY,
1110+ * if bpf_cgroup_storage_set() failed, the rest of programs
1111+ * will not execute. This should be a really rare scenario
1112+ * as it requires BPF_CGROUP_STORAGE_NEST_MAX number of
1113+ * preemptions all between bpf_cgroup_storage_set() and
1114+ * bpf_cgroup_storage_unset() on the same cpu.
1115+ */
11091116 #define BPF_PROG_RUN_ARRAY_FLAGS(array, ctx, func, ret_flags) \
11101117 ({ \
11111118 struct bpf_prog_array_item *_item; \
@@ -1118,10 +1125,12 @@ int bpf_prog_array_copy(struct bpf_prog_array *old_array, · 6 unmodified lines
11181125 _array = rcu_dereference(array); \
11191126 _item = &_array->items[0]; \
11201127 while ((_prog = READ_ONCE(_item->prog))) { \
1121− bpf_cgroup_storage_set(_item->cgroup_storage); \
1128+ if (unlikely(bpf_cgroup_storage_set(_item->cgroup_storage))) \
1129+ break; \
11221130 func_ret = func(_prog, ctx); \
11231131 _ret &= (func_ret & 1); \
11241132 *(ret_flags) |= (func_ret >> 1); \
1133+ bpf_cgroup_storage_unset(); \
11251134 _item++; \
11261135 } \
11271136 rcu_read_unlock(); \
@@ -1142,9 +1151,14 @@ int bpf_prog_array_copy(struct bpf_prog_array *old_array, · 14 unmodified lines
11421151 goto _out; \
11431152 _item = &_array->items[0]; \
11441153 while ((_prog = READ_ONCE(_item->prog))) { \
1145− if (set_cg_storage) \
1146− bpf_cgroup_storage_set(_item->cgroup_storage); \
1147− _ret &= func(_prog, ctx); \
1154+ if (!set_cg_storage) { \
1155+ _ret &= func(_prog, ctx); \
1156+ } else { \
1157+ if (unlikely(bpf_cgroup_storage_set(_item->cgroup_storage))) \
1158+ break; \
1159+ _ret &= func(_prog, ctx); \
1160+ bpf_cgroup_storage_unset(); \
1161+ } \
11481162 _item++; \
11491163 } \
11501164 _out: \
kernel/bpf/helpers.c+11−4
@@ -382,8 +382,8 @@ const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto = { · 381 unmodified lines
382382 };
383383
384384 #ifdef CONFIG_CGROUP_BPF
385−DECLARE_PER_CPU(struct bpf_cgroup_storage*,
386− bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);
385+DECLARE_PER_CPU(struct bpf_cgroup_storage_info,
386+ bpf_cgroup_storage_info[BPF_CGROUP_STORAGE_NEST_MAX]);
387387
388388 BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags)
389389 {
@@ -392,10 +392,17 @@ BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags) · 2 unmodified lines
392392 * verifier checks that its value is correct.
393393 */
394394 enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);
395− struct bpf_cgroup_storage *storage;
395+ struct bpf_cgroup_storage *storage = NULL;
396396 void *ptr;
397+ int i;
397398
398− storage = this_cpu_read(bpf_cgroup_storage[stype]);
399+ for (i = 0; i < BPF_CGROUP_STORAGE_NEST_MAX; i++) {
400+ if (unlikely(this_cpu_read(bpf_cgroup_storage_info[i].task) != current))
401+ continue;
402+
403+ storage = this_cpu_read(bpf_cgroup_storage_info[i].storage[stype]);
404+ break;
405+ }
399406
400407 if (stype == BPF_CGROUP_STORAGE_SHARED)
401408 ptr = &READ_ONCE(storage->buf)->data[0];
kernel/bpf/local_storage.c+3−2
@@ -9,10 +9,11 @@ · 8 unmodified lines
99 #include <linux/slab.h>
1010 #include <uapi/linux/btf.h>
1111
12−DEFINE_PER_CPU(struct bpf_cgroup_storage*, bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);
13−
1412 #ifdef CONFIG_CGROUP_BPF
1513
14+DEFINE_PER_CPU(struct bpf_cgroup_storage_info,
15+ bpf_cgroup_storage_info[BPF_CGROUP_STORAGE_NEST_MAX]);
16+
1617 #include "../cgroup/cgroup-internal.h"
1718
1819 #define LOCAL_STORAGE_CREATE_FLAG_MASK \
net/bpf/test_run.c+5−1
@@ -106,12 +106,16 @@ static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, · 105 unmodified lines
106106
107107 bpf_test_timer_enter(&t);
108108 do {
109− bpf_cgroup_storage_set(storage);
109+ ret = bpf_cgroup_storage_set(storage);
110+ if (ret)
111+ break;
110112
111113 if (xdp)
112114 *retval = bpf_prog_run_xdp(prog, ctx);
113115 else
114116 *retval = BPF_PROG_RUN(prog, ctx);
117+
118+ bpf_cgroup_storage_unset();
115119 } while (bpf_test_timer_continue(&t, repeat, &ret, time));
116120 bpf_test_timer_leave(&t);
117121

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t