History

bpf: Convert queue_stack map to rqspinlock

Replace all usage of raw_spinlock_t in queue_stack_maps.c with rqspinlock. This is a map type with a set of open syzbot reports reproducing possible deadlocks. Prior attempt to fix the issues was at [0], but was dropped in favor of this approach. Make sure we return the -EBUSY error in case of possible deadlocks or timeouts, just to make sure user space or BPF programs relying on the error code to detect problems do not break. With these changes, the map should be safe to access in any context, including NMIs. [0]: https://lore.kernel.org/all/20240429165658.1305969-1-sidchintamaneni@gmail.com

Kumar Kartikeya Dwivedi committed 1 year ago

Commit 2f415031 parent 92b90f71 file · +12 −23Browse files

Showing the changes to kernel/bpf/queue_stack_maps.c (1 of 1 file). Show all

Unified · Split

kernel/bpf/queue_stack_maps.c+12−23
@@ -9,13 +9,14 @@ · 8 unmodified lines
9#include <linux/slab.h>9#include <linux/slab.h>
10#include <linux/btf_ids.h>10#include <linux/btf_ids.h>
11#include "percpu_freelist.h"11#include "percpu_freelist.h"
12#include <asm/rqspinlock.h>
12 13
13#define QUEUE_STACK_CREATE_FLAG_MASK \14#define QUEUE_STACK_CREATE_FLAG_MASK \
14 (BPF_F_NUMA_NODE | BPF_F_ACCESS_MASK)15 (BPF_F_NUMA_NODE | BPF_F_ACCESS_MASK)
15 16
16struct bpf_queue_stack {17struct bpf_queue_stack {
17 struct bpf_map map;18 struct bpf_map map;
18 raw_spinlock_t lock;19 rqspinlock_t lock;
19 u32 head, tail;20 u32 head, tail;
20 u32 size; /* max_entries + 1 */21 u32 size; /* max_entries + 1 */
21 22
@@ -78,7 +79,7 @@ static struct bpf_map *queue_stack_map_alloc(union bpf_attr *attr) · 56 unmodified lines
78 79
79 qs->size = size;80 qs->size = size;
80 81
81 raw_spin_lock_init(&qs->lock);82 raw_res_spin_lock_init(&qs->lock);
82 83
83 return &qs->map;84 return &qs->map;
84}85}
@@ -98,12 +99,8 @@ static long __queue_map_get(struct bpf_map *map, void *value, bool delete) · 13 unmodified lines
98 int err = 0;99 int err = 0;
99 void *ptr;100 void *ptr;
100 101
101 if (in_nmi()) {102 if (raw_res_spin_lock_irqsave(&qs->lock, flags))
102 if (!raw_spin_trylock_irqsave(&qs->lock, flags))103 return -EBUSY;
103 return -EBUSY;
104 } else {
105 raw_spin_lock_irqsave(&qs->lock, flags);
106 }
107 104
108 if (queue_stack_map_is_empty(qs)) {105 if (queue_stack_map_is_empty(qs)) {
109 memset(value, 0, qs->map.value_size);106 memset(value, 0, qs->map.value_size);
@@ -120,7 +117,7 @@ static long __queue_map_get(struct bpf_map *map, void *value, bool delete) · 10 unmodified lines
120 }117 }
121 118
122out:119out:
123 raw_spin_unlock_irqrestore(&qs->lock, flags);120 raw_res_spin_unlock_irqrestore(&qs->lock, flags);
124 return err;121 return err;
125}122}
126 123
@@ -133,12 +130,8 @@ static long __stack_map_get(struct bpf_map *map, void *value, bool delete) · 6 unmodified lines
133 void *ptr;130 void *ptr;
134 u32 index;131 u32 index;
135 132
136 if (in_nmi()) {133 if (raw_res_spin_lock_irqsave(&qs->lock, flags))
137 if (!raw_spin_trylock_irqsave(&qs->lock, flags))134 return -EBUSY;
138 return -EBUSY;
139 } else {
140 raw_spin_lock_irqsave(&qs->lock, flags);
141 }
142 135
143 if (queue_stack_map_is_empty(qs)) {136 if (queue_stack_map_is_empty(qs)) {
144 memset(value, 0, qs->map.value_size);137 memset(value, 0, qs->map.value_size);
@@ -157,7 +150,7 @@ static long __stack_map_get(struct bpf_map *map, void *value, bool delete) · 12 unmodified lines
157 qs->head = index;150 qs->head = index;
158 151
159out:152out:
160 raw_spin_unlock_irqrestore(&qs->lock, flags);153 raw_res_spin_unlock_irqrestore(&qs->lock, flags);
161 return err;154 return err;
162}155}
163 156
@@ -203,12 +196,8 @@ static long queue_stack_map_push_elem(struct bpf_map *map, void *value, · 39 unmodified lines
203 if (flags & BPF_NOEXIST || flags > BPF_EXIST)196 if (flags & BPF_NOEXIST || flags > BPF_EXIST)
204 return -EINVAL;197 return -EINVAL;
205 198
206 if (in_nmi()) {199 if (raw_res_spin_lock_irqsave(&qs->lock, irq_flags))
207 if (!raw_spin_trylock_irqsave(&qs->lock, irq_flags))200 return -EBUSY;
208 return -EBUSY;
209 } else {
210 raw_spin_lock_irqsave(&qs->lock, irq_flags);
211 }
212 201
213 if (queue_stack_map_is_full(qs)) {202 if (queue_stack_map_is_full(qs)) {
214 if (!replace) {203 if (!replace) {
@@ -227,7 +216,7 @@ static long queue_stack_map_push_elem(struct bpf_map *map, void *value, · 12 unmodified lines
227 qs->head = 0;216 qs->head = 0;
228 217
229out:218out:
230 raw_spin_unlock_irqrestore(&qs->lock, irq_flags);219 raw_res_spin_unlock_irqrestore(&qs->lock, irq_flags);
231 return err;220 return err;
232}221}
233 222