History

mmap locking API: add mmap_assert_locked() and mmap_assert_write_locked()

Add new APIs to assert that mmap_sem is held. Using this instead of rwsem_is_locked and lockdep_assert_held[_write] makes the assertions more tolerant of future changes to the lock type.

Michel Lespinasse committed 6 years ago

Commit 42fc5411 parent 14c36569 files · +28 −14Browse files

Unified · Split

9 files
  1. arch/x86/events/core.c +1 −1
  2. fs/userfaultfd.c +3 −3
  3. include/linux/mmap_lock.h +14 −0
  4. mm/gup.c +1 −1
  5. mm/hmm.c +1 −1
  6. mm/memory.c +1 −1
  7. mm/mmu_notifier.c +3 −3
  8. mm/pagewalk.c +3 −3
  9. mm/util.c +1 −1
arch/x86/events/core.c+1−1
@@ -2181,7 +2181,7 @@ static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm) · 2180 unmodified lines
21812181 * For now, this can't happen because all callers hold mmap_sem
21822182 * for write. If this changes, we'll need a different solution.
21832183 */
2184− lockdep_assert_held_write(&mm->mmap_sem);
2184+ mmap_assert_write_locked(mm);
21852185
21862186 if (atomic_inc_return(&mm->context.perf_rdpmc_allowed) == 1)
21872187 on_each_cpu_mask(mm_cpumask(mm), cr4_update_pce, NULL, 1);
fs/userfaultfd.c+3−3
@@ -234,7 +234,7 @@ static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx, · 233 unmodified lines
234234 pte_t *ptep, pte;
235235 bool ret = true;
236236
237− VM_BUG_ON(!rwsem_is_locked(&mm->mmap_sem));
237+ mmap_assert_locked(mm);
238238
239239 ptep = huge_pte_offset(mm, address, vma_mmu_pagesize(vma));
240240
@@ -286,7 +286,7 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, · 45 unmodified lines
286286 pte_t *pte;
287287 bool ret = true;
288288
289− VM_BUG_ON(!rwsem_is_locked(&mm->mmap_sem));
289+ mmap_assert_locked(mm);
290290
291291 pgd = pgd_offset(mm, address);
292292 if (!pgd_present(*pgd))
@@ -405,7 +405,7 @@ vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason) · 112 unmodified lines
405405 * Coredumping runs without mmap_sem so we can only check that
406406 * the mmap_sem is held, if PF_DUMPCORE was not set.
407407 */
408− WARN_ON_ONCE(!rwsem_is_locked(&mm->mmap_sem));
408+ mmap_assert_locked(mm);
409409
410410 ctx = vmf->vma->vm_userfaultfd_ctx.ctx;
411411 if (!ctx)
include/linux/mmap_lock.h+14−0
@@ -1,6 +1,8 @@
11 #ifndef _LINUX_MMAP_LOCK_H
22 #define _LINUX_MMAP_LOCK_H
33
4+#include <linux/mmdebug.h>
5+
46 #define MMAP_LOCK_INITIALIZER(name) \
57 .mmap_sem = __RWSEM_INITIALIZER((name).mmap_sem),
68
@@ -73,4 +75,16 @@ static inline void mmap_read_unlock_non_owner(struct mm_struct *mm) · 66 unmodified lines
7375 up_read_non_owner(&mm->mmap_sem);
7476 }
7577
78+static inline void mmap_assert_locked(struct mm_struct *mm)
79+{
80+ lockdep_assert_held(&mm->mmap_sem);
81+ VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_sem), mm);
82+}
83+
84+static inline void mmap_assert_write_locked(struct mm_struct *mm)
85+{
86+ lockdep_assert_held_write(&mm->mmap_sem);
87+ VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_sem), mm);
88+}
89+
7690 #endif /* _LINUX_MMAP_LOCK_H */
mm/gup.c+1−1
@@ -1425,7 +1425,7 @@ long populate_vma_page_range(struct vm_area_struct *vma, · 1424 unmodified lines
14251425 VM_BUG_ON(end & ~PAGE_MASK);
14261426 VM_BUG_ON_VMA(start < vma->vm_start, vma);
14271427 VM_BUG_ON_VMA(end > vma->vm_end, vma);
1428− VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_sem), mm);
1428+ mmap_assert_locked(mm);
14291429
14301430 gup_flags = FOLL_TOUCH | FOLL_POPULATE | FOLL_MLOCK;
14311431 if (vma->vm_flags & VM_LOCKONFAULT)
mm/hmm.c+1−1
@@ -563,7 +563,7 @@ int hmm_range_fault(struct hmm_range *range) · 562 unmodified lines
563563 struct mm_struct *mm = range->notifier->mm;
564564 int ret;
565565
566− lockdep_assert_held(&mm->mmap_sem);
566+ mmap_assert_locked(mm);
567567
568568 do {
569569 /* If range is no longer valid force retry. */
mm/memory.c+1−1
@@ -1211,7 +1211,7 @@ static inline unsigned long zap_pud_range(struct mmu_gather *tlb, · 1210 unmodified lines
12111211 next = pud_addr_end(addr, end);
12121212 if (pud_trans_huge(*pud) || pud_devmap(*pud)) {
12131213 if (next - addr != HPAGE_PUD_SIZE) {
1214− VM_BUG_ON_VMA(!rwsem_is_locked(&tlb->mm->mmap_sem), vma);
1214+ mmap_assert_locked(tlb->mm);
12151215 split_huge_pud(vma, pud, addr);
12161216 } else if (zap_huge_pud(tlb, vma, pud, addr))
12171217 goto next;
mm/mmu_notifier.c+3−3
@@ -609,7 +609,7 @@ int __mmu_notifier_register(struct mmu_notifier *subscription, · 608 unmodified lines
609609 struct mmu_notifier_subscriptions *subscriptions = NULL;
610610 int ret;
611611
612− lockdep_assert_held_write(&mm->mmap_sem);
612+ mmap_assert_write_locked(mm);
613613 BUG_ON(atomic_read(&mm->mm_users) <= 0);
614614
615615 if (IS_ENABLED(CONFIG_LOCKDEP)) {
@@ -761,7 +761,7 @@ struct mmu_notifier *mmu_notifier_get_locked(const struct mmu_notifier_ops *ops, · 145 unmodified lines
761761 struct mmu_notifier *subscription;
762762 int ret;
763763
764− lockdep_assert_held_write(&mm->mmap_sem);
764+ mmap_assert_write_locked(mm);
765765
766766 if (mm->notifier_subscriptions) {
767767 subscription = find_get_mmu_notifier(mm, ops);
@@ -1006,7 +1006,7 @@ int mmu_interval_notifier_insert_locked( · 238 unmodified lines
10061006 mm->notifier_subscriptions;
10071007 int ret;
10081008
1009− lockdep_assert_held_write(&mm->mmap_sem);
1009+ mmap_assert_write_locked(mm);
10101010
10111011 if (!subscriptions || !subscriptions->has_itree) {
10121012 ret = __mmu_notifier_register(NULL, mm);
mm/pagewalk.c+3−3
@@ -395,7 +395,7 @@ int walk_page_range(struct mm_struct *mm, unsigned long start, · 394 unmodified lines
395395 if (!walk.mm)
396396 return -EINVAL;
397397
398− lockdep_assert_held(&walk.mm->mmap_sem);
398+ mmap_assert_locked(walk.mm);
399399
400400 vma = find_vma(walk.mm, start);
401401 do {
@@ -453,7 +453,7 @@ int walk_page_range_novma(struct mm_struct *mm, unsigned long start, · 51 unmodified lines
453453 if (start >= end || !walk.mm)
454454 return -EINVAL;
455455
456− lockdep_assert_held(&walk.mm->mmap_sem);
456+ mmap_assert_locked(walk.mm);
457457
458458 return __walk_page_range(start, end, &walk);
459459 }
@@ -472,7 +472,7 @@ int walk_page_vma(struct vm_area_struct *vma, const struct mm_walk_ops *ops, · 12 unmodified lines
472472 if (!walk.mm)
473473 return -EINVAL;
474474
475− lockdep_assert_held(&walk.mm->mmap_sem);
475+ mmap_assert_locked(walk.mm);
476476
477477 err = walk_page_test(vma->vm_start, vma->vm_end, &walk);
478478 if (err > 0)
mm/util.c+1−1
@@ -437,7 +437,7 @@ int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc, · 436 unmodified lines
437437 unsigned long locked_vm, limit;
438438 int ret = 0;
439439
440− lockdep_assert_held_write(&mm->mmap_sem);
440+ mmap_assert_write_locked(mm);
441441
442442 locked_vm = mm->locked_vm;
443443 if (inc) {