bpf: Convert queue_stack map to rqspinlock

Replace all usage of raw_spinlock_t in queue_stack_maps.c with rqspinlock. This is a map type with a set of open syzbot reports reproducing possible deadlocks. Prior attempt to fix the issues was at [0], but was dropped in favor of this approach. Make sure we return the -EBUSY error in case of possible deadlocks or timeouts, just to make sure user space or BPF programs relying on the error code to detect problems do not break. With these changes, the map should be safe to access in any context, including NMIs. [0]: https://lore.kernel.org/all/20240429165658.1305969-1-sidchintamaneni@gmail.com

Kumar Kartikeya Dwivedi committed 1 year ago

Commit 2f415031 parent 92b90f71 file · +12 −23Browse files

Showing the changes to kernel/bpf/queue_stack_maps.c (1 of 1 file). Show all

Unified · Split

kernel/bpf/queue_stack_maps.c+12−23

@@ -9,13 +9,14 @@ · 8 unmodified lines
9	9	#include <linux/slab.h>
10	10	#include <linux/btf_ids.h>
11	11	#include "percpu_freelist.h"
	12	+#include <asm/rqspinlock.h>
12	13
13	14	#define QUEUE_STACK_CREATE_FLAG_MASK \
14	15	(BPF_F_NUMA_NODE \| BPF_F_ACCESS_MASK)
15	16
16	17	struct bpf_queue_stack {
17	18	struct bpf_map map;
18		− raw_spinlock_t lock;
	19	+ rqspinlock_t lock;
19	20	u32 head, tail;
20	21	u32 size; /* max_entries + 1 */
21	22
@@ -78,7 +79,7 @@ static struct bpf_map queue_stack_map_alloc(union bpf_attr attr) · 56 unmodified lines
78	79
79	80	qs->size = size;
80	81
81		− raw_spin_lock_init(&qs->lock);
	82	+ raw_res_spin_lock_init(&qs->lock);
82	83
83	84	return &qs->map;
84	85	}
@@ -98,12 +99,8 @@ static long __queue_map_get(struct bpf_map map, void value, bool delete) · 13 unmodified lines
98	99	int err = 0;
99	100	void *ptr;
100	101
101		− if (in_nmi()) {
102		− if (!raw_spin_trylock_irqsave(&qs->lock, flags))
103		− return -EBUSY;
104		− } else {
105		− raw_spin_lock_irqsave(&qs->lock, flags);
106		− }
	102	+ if (raw_res_spin_lock_irqsave(&qs->lock, flags))
	103	+ return -EBUSY;
107	104
108	105	if (queue_stack_map_is_empty(qs)) {
109	106	memset(value, 0, qs->map.value_size);
@@ -120,7 +117,7 @@ static long __queue_map_get(struct bpf_map map, void value, bool delete) · 10 unmodified lines
120	117	}
121	118
122	119	out:
123		− raw_spin_unlock_irqrestore(&qs->lock, flags);
	120	+ raw_res_spin_unlock_irqrestore(&qs->lock, flags);
124	121	return err;
125	122	}
126	123
@@ -133,12 +130,8 @@ static long __stack_map_get(struct bpf_map map, void value, bool delete) · 6 unmodified lines
133	130	void *ptr;
134	131	u32 index;
135	132
136		− if (in_nmi()) {
137		− if (!raw_spin_trylock_irqsave(&qs->lock, flags))
138		− return -EBUSY;
139		− } else {
140		− raw_spin_lock_irqsave(&qs->lock, flags);
141		− }
	133	+ if (raw_res_spin_lock_irqsave(&qs->lock, flags))
	134	+ return -EBUSY;
142	135
143	136	if (queue_stack_map_is_empty(qs)) {
144	137	memset(value, 0, qs->map.value_size);
@@ -157,7 +150,7 @@ static long __stack_map_get(struct bpf_map map, void value, bool delete) · 12 unmodified lines
157	150	qs->head = index;
158	151
159	152	out:
160		− raw_spin_unlock_irqrestore(&qs->lock, flags);
	153	+ raw_res_spin_unlock_irqrestore(&qs->lock, flags);
161	154	return err;
162	155	}
163	156
@@ -203,12 +196,8 @@ static long queue_stack_map_push_elem(struct bpf_map map, void value, · 39 unmodified lines
203	196	if (flags & BPF_NOEXIST \|\| flags > BPF_EXIST)
204	197	return -EINVAL;
205	198
206		− if (in_nmi()) {
207		− if (!raw_spin_trylock_irqsave(&qs->lock, irq_flags))
208		− return -EBUSY;
209		− } else {
210		− raw_spin_lock_irqsave(&qs->lock, irq_flags);
211		− }
	199	+ if (raw_res_spin_lock_irqsave(&qs->lock, irq_flags))
	200	+ return -EBUSY;
212	201
213	202	if (queue_stack_map_is_full(qs)) {
214	203	if (!replace) {
@@ -227,7 +216,7 @@ static long queue_stack_map_push_elem(struct bpf_map map, void value, · 12 unmodified lines
227	216	qs->head = 0;
228	217
229	218	out:
230		− raw_spin_unlock_irqrestore(&qs->lock, irq_flags);
	219	+ raw_res_spin_unlock_irqrestore(&qs->lock, irq_flags);
231	220	return err;
232	221	}
233	222