stack_map_get_build_id_offset() introduced a per-CPU irq_work to defer mmap_read_unlock() from NMI context, and bpf_find_vma() later reused the same mmap_unlock_work. Both callers only check whether the work is busy before taking mmap_lock, so a nested caller can reuse the slot before the first caller queues it. Two read locks may then be acquired while only one deferred unlock runs, leaking a read lock and blocking exit_mmap(). Reserve the per-CPU slot before mmap_read_trylock(). Use the same wrapper in stackmap and bpf_find_vma() so both callers release the reservation on trylock failure. Keep rejecting the slot while the irq_work remains busy. Release it after the irq_work callback unlocks the mm.
Showing the changes to kernel/bpf/stackmap.c (1 of 3 files). Show all
Unified · Split
| @@ -414,8 +414,7 @@ static void stack_map_get_build_id_offset_sleepable(struct bpf_stack_build_id *i · 413 unmodified lines | |||
| 414 | static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, | 414 | static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, |
| 415 | u32 trace_nr, bool user, bool may_fault) | 415 | u32 trace_nr, bool user, bool may_fault) |
| 416 | { | 416 | { |
| 417 | struct mmap_unlock_irq_work *work = NULL; | 417 | struct mmap_unlock_irq_work *work; |
| 418 | bool irq_work_busy = bpf_mmap_unlock_get_irq_work(&work); | ||
| 419 | bool has_user_ctx = user && current && current->mm; | 418 | bool has_user_ctx = user && current && current->mm; |
| 420 | struct stack_map_build_id_cache cache = {}; | 419 | struct stack_map_build_id_cache cache = {}; |
| 421 | struct vm_area_struct *vma; | 420 | struct vm_area_struct *vma; |
| @@ -426,15 +425,16 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, · 4 unmodified lines | |||
| 426 | return; | 425 | return; |
| 427 | } | 426 | } |
| 428 | 427 | ||
| 429 | /* If the irq_work is in use, fall back to report ips. Same | 428 | if (!has_user_ctx) |
| 430 | * fallback is used for kernel stack (!user) on a stackmap with | 429 | goto fallback; |
| 431 | * build_id. | 430 | |
| 432 | */ | 431 | work = bpf_mmap_unlock_guard_get(); |
| 433 | if (!has_user_ctx || irq_work_busy || !mmap_read_trylock(current->mm)) { | 432 | if (IS_ERR(work)) |
| 434 | /* cannot access current->mm, fall back to ips */ | 433 | goto fallback; |
| 435 | for (i = 0; i < trace_nr; i++) | 434 | |
| 436 | stack_map_build_id_set_ip(&id_offs[i]); | 435 | if (!mmap_read_trylock(current->mm)) { |
| 437 | return; | 436 | bpf_mmap_unlock_guard_put(work); |
| 437 | goto fallback; | ||
| 438 | } | 438 | } |
| 439 | 439 | ||
| 440 | for (i = 0; i < trace_nr; i++) { | 440 | for (i = 0; i < trace_nr; i++) { |
| @@ -465,6 +465,12 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, · 24 unmodified lines | |||
| 465 | vma->vm_pgoff); | 465 | vma->vm_pgoff); |
| 466 | } | 466 | } |
| 467 | bpf_mmap_unlock_mm(work, current->mm); | 467 | bpf_mmap_unlock_mm(work, current->mm); |
| 468 | return; | ||
| 469 | |||
| 470 | fallback: | ||
| 471 | /* cannot access current->mm, fall back to ips */ | ||
| 472 | for (i = 0; i < trace_nr; i++) | ||
| 473 | stack_map_build_id_set_ip(&id_offs[i]); | ||
| 468 | } | 474 | } |
| 469 | 475 | ||
| 470 | static struct perf_callchain_entry * | 476 | static struct perf_callchain_entry * |