Sysbot discovered that the queue and stack maps can deadlock if they are being used from a BPF program that can be called from NMI context (such as one that is attached to a perf HW counter event). To fix this, add an in_nmi() check and use raw_spin_trylock() in NMI context, erroring out if grabbing the lock fails.
Showing the changes to kernel/bpf/queue_stack_maps.c (1 of 1 file). Show all
Unified · Split
| @@ -98,7 +98,12 @@ static long __queue_map_get(struct bpf_map *map, void *value, bool delete) · 97 unmodified lines | |||
| 98 | int err = 0; | 98 | int err = 0; |
| 99 | void *ptr; | 99 | void *ptr; |
| 100 | 100 | ||
| 101 | raw_spin_lock_irqsave(&qs->lock, flags); | 101 | if (in_nmi()) { |
| 102 | if (!raw_spin_trylock_irqsave(&qs->lock, flags)) | ||
| 103 | return -EBUSY; | ||
| 104 | } else { | ||
| 105 | raw_spin_lock_irqsave(&qs->lock, flags); | ||
| 106 | } | ||
| 102 | 107 | ||
| 103 | if (queue_stack_map_is_empty(qs)) { | 108 | if (queue_stack_map_is_empty(qs)) { |
| 104 | memset(value, 0, qs->map.value_size); | 109 | memset(value, 0, qs->map.value_size); |
| @@ -128,7 +133,12 @@ static long __stack_map_get(struct bpf_map *map, void *value, bool delete) · 23 unmodified lines | |||
| 128 | void *ptr; | 133 | void *ptr; |
| 129 | u32 index; | 134 | u32 index; |
| 130 | 135 | ||
| 131 | raw_spin_lock_irqsave(&qs->lock, flags); | 136 | if (in_nmi()) { |
| 137 | if (!raw_spin_trylock_irqsave(&qs->lock, flags)) | ||
| 138 | return -EBUSY; | ||
| 139 | } else { | ||
| 140 | raw_spin_lock_irqsave(&qs->lock, flags); | ||
| 141 | } | ||
| 132 | 142 | ||
| 133 | if (queue_stack_map_is_empty(qs)) { | 143 | if (queue_stack_map_is_empty(qs)) { |
| 134 | memset(value, 0, qs->map.value_size); | 144 | memset(value, 0, qs->map.value_size); |
| @@ -193,7 +203,12 @@ static long queue_stack_map_push_elem(struct bpf_map *map, void *value, · 58 unmodified lines | |||
| 193 | if (flags & BPF_NOEXIST || flags > BPF_EXIST) | 203 | if (flags & BPF_NOEXIST || flags > BPF_EXIST) |
| 194 | return -EINVAL; | 204 | return -EINVAL; |
| 195 | 205 | ||
| 196 | raw_spin_lock_irqsave(&qs->lock, irq_flags); | 206 | if (in_nmi()) { |
| 207 | if (!raw_spin_trylock_irqsave(&qs->lock, irq_flags)) | ||
| 208 | return -EBUSY; | ||
| 209 | } else { | ||
| 210 | raw_spin_lock_irqsave(&qs->lock, irq_flags); | ||
| 211 | } | ||
| 197 | 212 | ||
| 198 | if (queue_stack_map_is_full(qs)) { | 213 | if (queue_stack_map_is_full(qs)) { |
| 199 | if (!replace) { | 214 | if (!replace) { |