stack_map_get_build_id_offset() introduced a per-CPU irq_work to defer mmap_read_unlock() from NMI context, and bpf_find_vma() later reused the same mmap_unlock_work. Both callers only check whether the work is busy before taking mmap_lock, so a nested caller can reuse the slot before the first caller queues it. Two read locks may then be acquired while only one deferred unlock runs, leaking a read lock and blocking exit_mmap(). Reserve the per-CPU slot before mmap_read_trylock(). Use the same wrapper in stackmap and bpf_find_vma() so both callers release the reservation on trylock failure. Keep rejecting the slot while the irq_work remains busy. Release it after the irq_work callback unlocks the mm.
Showing the changes to kernel/bpf/stackmap.c (1 of 3 files). Show all
Unified · Split
| @@ -414,8 +414,7 @@ static void stack_map_get_build_id_offset_sleepable(struct bpf_stack_build_id *i · 413 unmodified lines | ||
| 414 | 414 | static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, |
| 415 | 415 | u32 trace_nr, bool user, bool may_fault) |
| 416 | 416 | { |
| 417 | − struct mmap_unlock_irq_work *work = NULL; | |
| 418 | − bool irq_work_busy = bpf_mmap_unlock_get_irq_work(&work); | |
| 417 | + struct mmap_unlock_irq_work *work; | |
| 419 | 418 | bool has_user_ctx = user && current && current->mm; |
| 420 | 419 | struct stack_map_build_id_cache cache = {}; |
| 421 | 420 | struct vm_area_struct *vma; |
| @@ -426,15 +425,16 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, · 4 unmodified lines | ||
| 426 | 425 | return; |
| 427 | 426 | } |
| 428 | 427 | |
| 429 | − /* If the irq_work is in use, fall back to report ips. Same | |
| 430 | − * fallback is used for kernel stack (!user) on a stackmap with | |
| 431 | − * build_id. | |
| 432 | − */ | |
| 433 | − if (!has_user_ctx || irq_work_busy || !mmap_read_trylock(current->mm)) { | |
| 434 | − /* cannot access current->mm, fall back to ips */ | |
| 435 | − for (i = 0; i < trace_nr; i++) | |
| 436 | − stack_map_build_id_set_ip(&id_offs[i]); | |
| 437 | − return; | |
| 428 | + if (!has_user_ctx) | |
| 429 | + goto fallback; | |
| 430 | + | |
| 431 | + work = bpf_mmap_unlock_guard_get(); | |
| 432 | + if (IS_ERR(work)) | |
| 433 | + goto fallback; | |
| 434 | + | |
| 435 | + if (!mmap_read_trylock(current->mm)) { | |
| 436 | + bpf_mmap_unlock_guard_put(work); | |
| 437 | + goto fallback; | |
| 438 | 438 | } |
| 439 | 439 | |
| 440 | 440 | for (i = 0; i < trace_nr; i++) { |
| @@ -465,6 +465,12 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, · 24 unmodified lines | ||
| 465 | 465 | vma->vm_pgoff); |
| 466 | 466 | } |
| 467 | 467 | bpf_mmap_unlock_mm(work, current->mm); |
| 468 | + return; | |
| 469 | + | |
| 470 | +fallback: | |
| 471 | + /* cannot access current->mm, fall back to ips */ | |
| 472 | + for (i = 0; i < trace_nr; i++) | |
| 473 | + stack_map_build_id_set_ip(&id_offs[i]); | |
| 468 | 474 | } |
| 469 | 475 | |
| 470 | 476 | static struct perf_callchain_entry * |