History

bpf: Fix NULL pointer dereference in bpf_get_local_storage() helper

Jiri Olsa reported a bug ([1]) in kernel where cgroup local storage pointer may be NULL in bpf_get_local_storage() helper. There are two issues uncovered by this bug: (1). kprobe or tracepoint prog incorrectly sets cgroup local storage before prog run, (2). due to change from preempt_disable to migrate_disable, preemption is possible and percpu storage might be overwritten by other tasks. This issue (1) is fixed in [2]. This patch tried to address issue (2). The following shows how things can go wrong: task 1: bpf_cgroup_storage_set() for percpu local storage preemption happens task 2: bpf_cgroup_storage_set() for percpu local storage preemption happens task 1: run bpf program task 1 will effectively use the percpu local storage setting by task 2 which will be either NULL or incorrect ones. Instead of just one common local storage per cpu, this patch fixed the issue by permitting 8 local storages per cpu and each local storage is identified by a task_struct pointer. This way, we allow at most 8 nested preemption between bpf_cgroup_storage_set() and bpf_cgroup_storage_unset(). The percpu local storage slot is released (calling bpf_cgroup_storage_unset()) by the same task after bpf program finished running. bpf_test_run() is also fixed to use the new bpf_cgroup_storage_set() interface. The patch is tested on top of [2] with reproducer in [1]. Without this patch, kernel will emit error in 2-3 minutes. With this patch, after one hour, still no error. [1] https://lore.kernel.org/bpf/CAKH8qBuXCfUz=w8L+Fj74OaUpbosO29niYwTki7e3Ag044_aww@mail.gmail.com/T [2] https://lore.kernel.org/bpf/20210309185028.3763817-1-yhs@fb.com

Yonghong Song committed 5 years ago

Commit b910eaa1 parent a46410d5 files · +86 −19Browse files

Unified · Split

5 files
  1. include/linux/bpf-cgroup.h +49 −8
  2. include/linux/bpf.h +18 −4
  3. kernel/bpf/helpers.c +11 −4
  4. kernel/bpf/local_storage.c +3 −2
  5. net/bpf/test_run.c +5 −1
include/linux/bpf-cgroup.h+49−8
@@ -20,14 +20,25 @@ struct bpf_sock_ops_kern; · 19 unmodified lines
20struct bpf_cgroup_storage;20struct bpf_cgroup_storage;
21struct ctl_table;21struct ctl_table;
22struct ctl_table_header;22struct ctl_table_header;
23struct task_struct;
23 24
24#ifdef CONFIG_CGROUP_BPF25#ifdef CONFIG_CGROUP_BPF
25 26
26extern struct static_key_false cgroup_bpf_enabled_key[MAX_BPF_ATTACH_TYPE];27extern struct static_key_false cgroup_bpf_enabled_key[MAX_BPF_ATTACH_TYPE];
27#define cgroup_bpf_enabled(type) static_branch_unlikely(&cgroup_bpf_enabled_key[type])28#define cgroup_bpf_enabled(type) static_branch_unlikely(&cgroup_bpf_enabled_key[type])
28 29
29DECLARE_PER_CPU(struct bpf_cgroup_storage*,30#define BPF_CGROUP_STORAGE_NEST_MAX 8
30 bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);31
32struct bpf_cgroup_storage_info {
33 struct task_struct *task;
34 struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE];
35};
36
37/* For each cpu, permit maximum BPF_CGROUP_STORAGE_NEST_MAX number of tasks
38 * to use bpf cgroup storage simultaneously.
39 */
40DECLARE_PER_CPU(struct bpf_cgroup_storage_info,
41 bpf_cgroup_storage_info[BPF_CGROUP_STORAGE_NEST_MAX]);
31 42
32#define for_each_cgroup_storage_type(stype) \43#define for_each_cgroup_storage_type(stype) \
33 for (stype = 0; stype < MAX_BPF_CGROUP_STORAGE_TYPE; stype++)44 for (stype = 0; stype < MAX_BPF_CGROUP_STORAGE_TYPE; stype++)
@@ -161,13 +172,42 @@ static inline enum bpf_cgroup_storage_type cgroup_storage_type( · 127 unmodified lines
161 return BPF_CGROUP_STORAGE_SHARED;172 return BPF_CGROUP_STORAGE_SHARED;
162}173}
163 174
164static inline void bpf_cgroup_storage_set(struct bpf_cgroup_storage175static inline int bpf_cgroup_storage_set(struct bpf_cgroup_storage
165 *storage[MAX_BPF_CGROUP_STORAGE_TYPE])176 *storage[MAX_BPF_CGROUP_STORAGE_TYPE])
166{177{
167 enum bpf_cgroup_storage_type stype;178 enum bpf_cgroup_storage_type stype;
179 int i, err = 0;
180
181 preempt_disable();
182 for (i = 0; i < BPF_CGROUP_STORAGE_NEST_MAX; i++) {
183 if (unlikely(this_cpu_read(bpf_cgroup_storage_info[i].task) != NULL))
184 continue;
185
186 this_cpu_write(bpf_cgroup_storage_info[i].task, current);
187 for_each_cgroup_storage_type(stype)
188 this_cpu_write(bpf_cgroup_storage_info[i].storage[stype],
189 storage[stype]);
190 goto out;
191 }
192 err = -EBUSY;
193 WARN_ON_ONCE(1);
194
195out:
196 preempt_enable();
197 return err;
198}
199
200static inline void bpf_cgroup_storage_unset(void)
201{
202 int i;
203
204 for (i = 0; i < BPF_CGROUP_STORAGE_NEST_MAX; i++) {
205 if (unlikely(this_cpu_read(bpf_cgroup_storage_info[i].task) != current))
206 continue;
168 207
169 for_each_cgroup_storage_type(stype)208 this_cpu_write(bpf_cgroup_storage_info[i].task, NULL);
170 this_cpu_write(bpf_cgroup_storage[stype], storage[stype]);209 return;
210 }
171}211}
172 212
173struct bpf_cgroup_storage *213struct bpf_cgroup_storage *
@@ -448,8 +488,9 @@ static inline int cgroup_bpf_prog_query(const union bpf_attr *attr, · 274 unmodified lines
448 return -EINVAL;488 return -EINVAL;
449}489}
450 490
451static inline void bpf_cgroup_storage_set(491static inline int bpf_cgroup_storage_set(
452 struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE]) {}492 struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE]) { return 0; }
493static inline void bpf_cgroup_storage_unset(void) {}
453static inline int bpf_cgroup_storage_assign(struct bpf_prog_aux *aux,494static inline int bpf_cgroup_storage_assign(struct bpf_prog_aux *aux,
454 struct bpf_map *map) { return 0; }495 struct bpf_map *map) { return 0; }
455static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(496static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(
include/linux/bpf.h+18−4
@@ -1106,6 +1106,13 @@ int bpf_prog_array_copy(struct bpf_prog_array *old_array, · 1105 unmodified lines
1106/* BPF program asks to set CN on the packet. */1106/* BPF program asks to set CN on the packet. */
1107#define BPF_RET_SET_CN (1 << 0)1107#define BPF_RET_SET_CN (1 << 0)
1108 1108
1109/* For BPF_PROG_RUN_ARRAY_FLAGS and __BPF_PROG_RUN_ARRAY,
1110 * if bpf_cgroup_storage_set() failed, the rest of programs
1111 * will not execute. This should be a really rare scenario
1112 * as it requires BPF_CGROUP_STORAGE_NEST_MAX number of
1113 * preemptions all between bpf_cgroup_storage_set() and
1114 * bpf_cgroup_storage_unset() on the same cpu.
1115 */
1109#define BPF_PROG_RUN_ARRAY_FLAGS(array, ctx, func, ret_flags) \1116#define BPF_PROG_RUN_ARRAY_FLAGS(array, ctx, func, ret_flags) \
1110 ({ \1117 ({ \
1111 struct bpf_prog_array_item *_item; \1118 struct bpf_prog_array_item *_item; \
@@ -1118,10 +1125,12 @@ int bpf_prog_array_copy(struct bpf_prog_array *old_array, · 6 unmodified lines
1118 _array = rcu_dereference(array); \1125 _array = rcu_dereference(array); \
1119 _item = &_array->items[0]; \1126 _item = &_array->items[0]; \
1120 while ((_prog = READ_ONCE(_item->prog))) { \1127 while ((_prog = READ_ONCE(_item->prog))) { \
1121 bpf_cgroup_storage_set(_item->cgroup_storage); \1128 if (unlikely(bpf_cgroup_storage_set(_item->cgroup_storage))) \
1129 break; \
1122 func_ret = func(_prog, ctx); \1130 func_ret = func(_prog, ctx); \
1123 _ret &= (func_ret & 1); \1131 _ret &= (func_ret & 1); \
1124 *(ret_flags) |= (func_ret >> 1); \1132 *(ret_flags) |= (func_ret >> 1); \
1133 bpf_cgroup_storage_unset(); \
1125 _item++; \1134 _item++; \
1126 } \1135 } \
1127 rcu_read_unlock(); \1136 rcu_read_unlock(); \
@@ -1142,9 +1151,14 @@ int bpf_prog_array_copy(struct bpf_prog_array *old_array, · 14 unmodified lines
1142 goto _out; \1151 goto _out; \
1143 _item = &_array->items[0]; \1152 _item = &_array->items[0]; \
1144 while ((_prog = READ_ONCE(_item->prog))) { \1153 while ((_prog = READ_ONCE(_item->prog))) { \
1145 if (set_cg_storage) \1154 if (!set_cg_storage) { \
1146 bpf_cgroup_storage_set(_item->cgroup_storage); \1155 _ret &= func(_prog, ctx); \
1147 _ret &= func(_prog, ctx); \1156 } else { \
1157 if (unlikely(bpf_cgroup_storage_set(_item->cgroup_storage))) \
1158 break; \
1159 _ret &= func(_prog, ctx); \
1160 bpf_cgroup_storage_unset(); \
1161 } \
1148 _item++; \1162 _item++; \
1149 } \1163 } \
1150_out: \1164_out: \
kernel/bpf/helpers.c+11−4
@@ -382,8 +382,8 @@ const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto = { · 381 unmodified lines
382};382};
383 383
384#ifdef CONFIG_CGROUP_BPF384#ifdef CONFIG_CGROUP_BPF
385DECLARE_PER_CPU(struct bpf_cgroup_storage*,385DECLARE_PER_CPU(struct bpf_cgroup_storage_info,
386 bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);386 bpf_cgroup_storage_info[BPF_CGROUP_STORAGE_NEST_MAX]);
387 387
388BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags)388BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags)
389{389{
@@ -392,10 +392,17 @@ BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags) · 2 unmodified lines
392 * verifier checks that its value is correct.392 * verifier checks that its value is correct.
393 */393 */
394 enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);394 enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);
395 struct bpf_cgroup_storage *storage;395 struct bpf_cgroup_storage *storage = NULL;
396 void *ptr;396 void *ptr;
397 int i;
397 398
398 storage = this_cpu_read(bpf_cgroup_storage[stype]);399 for (i = 0; i < BPF_CGROUP_STORAGE_NEST_MAX; i++) {
400 if (unlikely(this_cpu_read(bpf_cgroup_storage_info[i].task) != current))
401 continue;
402
403 storage = this_cpu_read(bpf_cgroup_storage_info[i].storage[stype]);
404 break;
405 }
399 406
400 if (stype == BPF_CGROUP_STORAGE_SHARED)407 if (stype == BPF_CGROUP_STORAGE_SHARED)
401 ptr = &READ_ONCE(storage->buf)->data[0];408 ptr = &READ_ONCE(storage->buf)->data[0];
kernel/bpf/local_storage.c+3−2
@@ -9,10 +9,11 @@ · 8 unmodified lines
9#include <linux/slab.h>9#include <linux/slab.h>
10#include <uapi/linux/btf.h>10#include <uapi/linux/btf.h>
11 11
12DEFINE_PER_CPU(struct bpf_cgroup_storage*, bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);
13
14#ifdef CONFIG_CGROUP_BPF12#ifdef CONFIG_CGROUP_BPF
15 13
14DEFINE_PER_CPU(struct bpf_cgroup_storage_info,
15 bpf_cgroup_storage_info[BPF_CGROUP_STORAGE_NEST_MAX]);
16
16#include "../cgroup/cgroup-internal.h"17#include "../cgroup/cgroup-internal.h"
17 18
18#define LOCAL_STORAGE_CREATE_FLAG_MASK \19#define LOCAL_STORAGE_CREATE_FLAG_MASK \
net/bpf/test_run.c+5−1
@@ -106,12 +106,16 @@ static int bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, · 105 unmodified lines
106 106
107 bpf_test_timer_enter(&t);107 bpf_test_timer_enter(&t);
108 do {108 do {
109 bpf_cgroup_storage_set(storage);109 ret = bpf_cgroup_storage_set(storage);
110 if (ret)
111 break;
110 112
111 if (xdp)113 if (xdp)
112 *retval = bpf_prog_run_xdp(prog, ctx);114 *retval = bpf_prog_run_xdp(prog, ctx);
113 else115 else
114 *retval = BPF_PROG_RUN(prog, ctx);116 *retval = BPF_PROG_RUN(prog, ctx);
117
118 bpf_cgroup_storage_unset();
115 } while (bpf_test_timer_continue(&t, repeat, &ret, time));119 } while (bpf_test_timer_continue(&t, repeat, &ret, time));
116 bpf_test_timer_leave(&t);120 bpf_test_timer_leave(&t);
117 121

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t