History

bpf: Fix mmap_lock leak in irq_work path

stack_map_get_build_id_offset() introduced a per-CPU irq_work to defer mmap_read_unlock() from NMI context, and bpf_find_vma() later reused the same mmap_unlock_work. Both callers only check whether the work is busy before taking mmap_lock, so a nested caller can reuse the slot before the first caller queues it. Two read locks may then be acquired while only one deferred unlock runs, leaking a read lock and blocking exit_mmap(). Reserve the per-CPU slot before mmap_read_trylock(). Use the same wrapper in stackmap and bpf_find_vma() so both callers release the reservation on trylock failure. Keep rejecting the slot while the irq_work remains busy. Release it after the irq_work callback unlocks the mm.

Sanghyun Park committed 3 months ago

Commit fa9dcac1 parent 3f562c53 files · +56 −37Browse files

Unified · Split

3 files
  1. kernel/bpf/mmap_unlock_work.h +29 −22
  2. kernel/bpf/stackmap.c +17 −11
  3. kernel/bpf/task_iter.c +10 −4
kernel/bpf/mmap_unlock_work.h+29−22
@@ -4,12 +4,15 @@ · 3 unmodified lines
4 4
5#ifndef __MMAP_UNLOCK_WORK_H__5#ifndef __MMAP_UNLOCK_WORK_H__
6#define __MMAP_UNLOCK_WORK_H__6#define __MMAP_UNLOCK_WORK_H__
7#include <linux/atomic.h>
8#include <linux/err.h>
7#include <linux/irq_work.h>9#include <linux/irq_work.h>
8 10
9/* irq_work to run mmap_read_unlock() in irq_work */11/* irq_work to run mmap_read_unlock() in irq_work */
10struct mmap_unlock_irq_work {12struct mmap_unlock_irq_work {
11 struct irq_work irq_work;13 struct irq_work irq_work;
12 struct mm_struct *mm;14 struct mm_struct *mm;
15 atomic_t active;
13};16};
14 17
15DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work);18DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work);
@@ -18,32 +21,36 @@ DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work); · 2 unmodified lines
18 * We cannot do mmap_read_unlock() when the irq is disabled, because of21 * We cannot do mmap_read_unlock() when the irq is disabled, because of
19 * risk to deadlock with rq_lock. To look up vma when the irqs are22 * risk to deadlock with rq_lock. To look up vma when the irqs are
20 * disabled, we need to run mmap_read_unlock() in irq_work. We use a23 * disabled, we need to run mmap_read_unlock() in irq_work. We use a
21 * percpu variable to do the irq_work. If the irq_work is already used24 * percpu variable to do the irq_work. The active flag reserves the slot
22 * by another lookup, we fall over.25 * before mmap_read_trylock() and until the irq_work callback consumes mm.
23 */26 */
24static inline bool bpf_mmap_unlock_get_irq_work(struct mmap_unlock_irq_work **work_ptr)27static inline struct mmap_unlock_irq_work *bpf_mmap_unlock_guard_get(void)
25{28{
26 struct mmap_unlock_irq_work *work = NULL;29 struct mmap_unlock_irq_work *work;
27 bool irq_work_busy = false;
28 30
29 if (irqs_disabled()) {31 if (!irqs_disabled())
30 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {32 return NULL;
31 work = this_cpu_ptr(&mmap_unlock_work);33
32 if (irq_work_is_busy(&work->irq_work)) {34 /*
33 /* cannot queue more up_read, fallback */35 * PREEMPT_RT does not allow to trylock mmap sem in interrupt
34 irq_work_busy = true;36 * disabled context. Force the fallback code.
35 }37 */
36 } else {38 if (IS_ENABLED(CONFIG_PREEMPT_RT))
37 /*39 return ERR_PTR(-EBUSY);
38 * PREEMPT_RT does not allow to trylock mmap sem in40
39 * interrupt disabled context. Force the fallback code.41 work = this_cpu_ptr(&mmap_unlock_work);
40 */42 if (irq_work_is_busy(&work->irq_work) ||
41 irq_work_busy = true;43 atomic_cmpxchg_acquire(&work->active, 0, 1))
42 }44 return ERR_PTR(-EBUSY);
43 }
44 45
45 *work_ptr = work;46 return work;
46 return irq_work_busy;47}
48
49static inline void
50bpf_mmap_unlock_guard_put(struct mmap_unlock_irq_work *work)
51{
52 if (work)
53 atomic_set_release(&work->active, 0);
47}54}
48 55
49static inline void bpf_mmap_unlock_mm(struct mmap_unlock_irq_work *work, struct mm_struct *mm)56static inline void bpf_mmap_unlock_mm(struct mmap_unlock_irq_work *work, struct mm_struct *mm)
kernel/bpf/stackmap.c+17−11
@@ -414,8 +414,7 @@ static void stack_map_get_build_id_offset_sleepable(struct bpf_stack_build_id *i · 413 unmodified lines
414static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs,414static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs,
415 u32 trace_nr, bool user, bool may_fault)415 u32 trace_nr, bool user, bool may_fault)
416{416{
417 struct mmap_unlock_irq_work *work = NULL;417 struct mmap_unlock_irq_work *work;
418 bool irq_work_busy = bpf_mmap_unlock_get_irq_work(&work);
419 bool has_user_ctx = user && current && current->mm;418 bool has_user_ctx = user && current && current->mm;
420 struct stack_map_build_id_cache cache = {};419 struct stack_map_build_id_cache cache = {};
421 struct vm_area_struct *vma;420 struct vm_area_struct *vma;
@@ -426,15 +425,16 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, · 4 unmodified lines
426 return;425 return;
427 }426 }
428 427
429 /* If the irq_work is in use, fall back to report ips. Same428 if (!has_user_ctx)
430 * fallback is used for kernel stack (!user) on a stackmap with429 goto fallback;
431 * build_id.430
432 */431 work = bpf_mmap_unlock_guard_get();
433 if (!has_user_ctx || irq_work_busy || !mmap_read_trylock(current->mm)) {432 if (IS_ERR(work))
434 /* cannot access current->mm, fall back to ips */433 goto fallback;
435 for (i = 0; i < trace_nr; i++)434
436 stack_map_build_id_set_ip(&id_offs[i]);435 if (!mmap_read_trylock(current->mm)) {
437 return;436 bpf_mmap_unlock_guard_put(work);
437 goto fallback;
438 }438 }
439 439
440 for (i = 0; i < trace_nr; i++) {440 for (i = 0; i < trace_nr; i++) {
@@ -465,6 +465,12 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, · 24 unmodified lines
465 vma->vm_pgoff);465 vma->vm_pgoff);
466 }466 }
467 bpf_mmap_unlock_mm(work, current->mm);467 bpf_mmap_unlock_mm(work, current->mm);
468 return;
469
470fallback:
471 /* cannot access current->mm, fall back to ips */
472 for (i = 0; i < trace_nr; i++)
473 stack_map_build_id_set_ip(&id_offs[i]);
468}474}
469 475
470static struct perf_callchain_entry *476static struct perf_callchain_entry *
kernel/bpf/task_iter.c+10−4
@@ -753,9 +753,8 @@ static struct bpf_iter_reg task_vma_reg_info = { · 752 unmodified lines
753BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start,753BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start,
754 bpf_callback_t, callback_fn, void *, callback_ctx, u64, flags)754 bpf_callback_t, callback_fn, void *, callback_ctx, u64, flags)
755{755{
756 struct mmap_unlock_irq_work *work = NULL;756 struct mmap_unlock_irq_work *work;
757 struct vm_area_struct *vma;757 struct vm_area_struct *vma;
758 bool irq_work_busy = false;
759 bool __maybe_unused mmput_needed = false;758 bool __maybe_unused mmput_needed = false;
760 struct mm_struct *mm;759 struct mm_struct *mm;
761 int ret = -ENOENT;760 int ret = -ENOENT;
@@ -792,9 +791,14 @@ BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, · 30 unmodified lines
792 if (!mm)791 if (!mm)
793 return -ENOENT;792 return -ENOENT;
794 793
795 irq_work_busy = bpf_mmap_unlock_get_irq_work(&work);794 work = bpf_mmap_unlock_guard_get();
795 if (IS_ERR(work)) {
796 ret = PTR_ERR(work);
797 goto out;
798 }
796 799
797 if (irq_work_busy || !mmap_read_trylock(mm)) {800 if (!mmap_read_trylock(mm)) {
801 bpf_mmap_unlock_guard_put(work);
798 ret = -EBUSY;802 ret = -EBUSY;
799 goto out;803 goto out;
800 }804 }
@@ -1191,6 +1195,8 @@ static void do_mmap_read_unlock(struct irq_work *entry) · 390 unmodified lines
1191 1195
1192 work = container_of(entry, struct mmap_unlock_irq_work, irq_work);1196 work = container_of(entry, struct mmap_unlock_irq_work, irq_work);
1193 mmap_read_unlock_non_owner(work->mm);1197 mmap_read_unlock_non_owner(work->mm);
1198 work->mm = NULL;
1199 bpf_mmap_unlock_guard_put(work);
1194}1200}
1195 1201
1196static int __init task_iter_init(void)1202static int __init task_iter_init(void)