mmap locking API: add mmap_assert_locked() and mmap_assert_write_locked()

Add new APIs to assert that mmap_sem is held. Using this instead of rwsem_is_locked and lockdep_assert_held[_write] makes the assertions more tolerant of future changes to the lock type.

Michel Lespinasse committed 6 years ago

Commit 42fc5411 parent 14c36569 files · +28 −14Browse files

Unified · Split

9 files

arch/x86/events/core.c +1 −1
fs/userfaultfd.c +3 −3
include/linux/mmap_lock.h +14 −0
mm/gup.c +1 −1
mm/hmm.c +1 −1
mm/memory.c +1 −1
mm/mmu_notifier.c +3 −3
mm/pagewalk.c +3 −3
mm/util.c +1 −1

arch/x86/events/core.c+1−1

@@ -2181,7 +2181,7 @@ static void x86_pmu_event_mapped(struct perf_event event, struct mm_struct mm) · 2180 unmodified lines
2181	* For now, this can't happen because all callers hold mmap_sem	2181	* For now, this can't happen because all callers hold mmap_sem
2182	* for write. If this changes, we'll need a different solution.	2182	* for write. If this changes, we'll need a different solution.
2183	*/	2183	*/
2184	lockdep_assert_held_write(&mm->mmap_sem);	2184	mmap_assert_write_locked(mm);
2185		2185
2186	if (atomic_inc_return(&mm->context.perf_rdpmc_allowed) == 1)	2186	if (atomic_inc_return(&mm->context.perf_rdpmc_allowed) == 1)
2187	on_each_cpu_mask(mm_cpumask(mm), cr4_update_pce, NULL, 1);	2187	on_each_cpu_mask(mm_cpumask(mm), cr4_update_pce, NULL, 1);

fs/userfaultfd.c+3−3

@@ -234,7 +234,7 @@ static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx, · 233 unmodified lines
234	pte_t *ptep, pte;	234	pte_t *ptep, pte;
235	bool ret = true;	235	bool ret = true;
236		236
237	VM_BUG_ON(!rwsem_is_locked(&mm->mmap_sem));	237	mmap_assert_locked(mm);
238		238
239	ptep = huge_pte_offset(mm, address, vma_mmu_pagesize(vma));	239	ptep = huge_pte_offset(mm, address, vma_mmu_pagesize(vma));
240		240
@@ -286,7 +286,7 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, · 45 unmodified lines
286	pte_t *pte;	286	pte_t *pte;
287	bool ret = true;	287	bool ret = true;
288		288
289	VM_BUG_ON(!rwsem_is_locked(&mm->mmap_sem));	289	mmap_assert_locked(mm);
290		290
291	pgd = pgd_offset(mm, address);	291	pgd = pgd_offset(mm, address);
292	if (!pgd_present(*pgd))	292	if (!pgd_present(*pgd))
@@ -405,7 +405,7 @@ vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason) · 112 unmodified lines
405	* Coredumping runs without mmap_sem so we can only check that	405	* Coredumping runs without mmap_sem so we can only check that
406	* the mmap_sem is held, if PF_DUMPCORE was not set.	406	* the mmap_sem is held, if PF_DUMPCORE was not set.
407	*/	407	*/
408	WARN_ON_ONCE(!rwsem_is_locked(&mm->mmap_sem));	408	mmap_assert_locked(mm);
409		409
410	ctx = vmf->vma->vm_userfaultfd_ctx.ctx;	410	ctx = vmf->vma->vm_userfaultfd_ctx.ctx;
411	if (!ctx)	411	if (!ctx)

include/linux/mmap_lock.h+14−0

@@ -1,6 +1,8 @@
1	#ifndef _LINUX_MMAP_LOCK_H	1	#ifndef _LINUX_MMAP_LOCK_H
2	#define _LINUX_MMAP_LOCK_H	2	#define _LINUX_MMAP_LOCK_H
3		3
		4	#include <linux/mmdebug.h>
		5
4	#define MMAP_LOCK_INITIALIZER(name) \	6	#define MMAP_LOCK_INITIALIZER(name) \
5	.mmap_sem = __RWSEM_INITIALIZER((name).mmap_sem),	7	.mmap_sem = __RWSEM_INITIALIZER((name).mmap_sem),
6		8
@@ -73,4 +75,16 @@ static inline void mmap_read_unlock_non_owner(struct mm_struct *mm) · 66 unmodified lines
73	up_read_non_owner(&mm->mmap_sem);	75	up_read_non_owner(&mm->mmap_sem);
74	}	76	}
75		77
		78	static inline void mmap_assert_locked(struct mm_struct *mm)
		79	{
		80	lockdep_assert_held(&mm->mmap_sem);
		81	VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_sem), mm);
		82	}
		83
		84	static inline void mmap_assert_write_locked(struct mm_struct *mm)
		85	{
		86	lockdep_assert_held_write(&mm->mmap_sem);
		87	VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_sem), mm);
		88	}
		89
76	#endif /* _LINUX_MMAP_LOCK_H */	90	#endif /* _LINUX_MMAP_LOCK_H */

mm/gup.c+1−1

@@ -1425,7 +1425,7 @@ long populate_vma_page_range(struct vm_area_struct *vma, · 1424 unmodified lines
1425	VM_BUG_ON(end & ~PAGE_MASK);	1425	VM_BUG_ON(end & ~PAGE_MASK);
1426	VM_BUG_ON_VMA(start < vma->vm_start, vma);	1426	VM_BUG_ON_VMA(start < vma->vm_start, vma);
1427	VM_BUG_ON_VMA(end > vma->vm_end, vma);	1427	VM_BUG_ON_VMA(end > vma->vm_end, vma);
1428	VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_sem), mm);	1428	mmap_assert_locked(mm);
1429		1429
1430	gup_flags = FOLL_TOUCH \| FOLL_POPULATE \| FOLL_MLOCK;	1430	gup_flags = FOLL_TOUCH \| FOLL_POPULATE \| FOLL_MLOCK;
1431	if (vma->vm_flags & VM_LOCKONFAULT)	1431	if (vma->vm_flags & VM_LOCKONFAULT)

mm/hmm.c+1−1

@@ -563,7 +563,7 @@ int hmm_range_fault(struct hmm_range *range) · 562 unmodified lines
563	struct mm_struct *mm = range->notifier->mm;	563	struct mm_struct *mm = range->notifier->mm;
564	int ret;	564	int ret;
565		565
566	lockdep_assert_held(&mm->mmap_sem);	566	mmap_assert_locked(mm);
567		567
568	do {	568	do {
569	/* If range is no longer valid force retry. */	569	/* If range is no longer valid force retry. */

mm/memory.c+1−1

@@ -1211,7 +1211,7 @@ static inline unsigned long zap_pud_range(struct mmu_gather *tlb, · 1210 unmodified lines
1211	next = pud_addr_end(addr, end);	1211	next = pud_addr_end(addr, end);
1212	if (pud_trans_huge(pud) \|\| pud_devmap(pud)) {	1212	if (pud_trans_huge(pud) \|\| pud_devmap(pud)) {
1213	if (next - addr != HPAGE_PUD_SIZE) {	1213	if (next - addr != HPAGE_PUD_SIZE) {
1214	VM_BUG_ON_VMA(!rwsem_is_locked(&tlb->mm->mmap_sem), vma);	1214	mmap_assert_locked(tlb->mm);
1215	split_huge_pud(vma, pud, addr);	1215	split_huge_pud(vma, pud, addr);
1216	} else if (zap_huge_pud(tlb, vma, pud, addr))	1216	} else if (zap_huge_pud(tlb, vma, pud, addr))
1217	goto next;	1217	goto next;

mm/mmu_notifier.c+3−3

@@ -609,7 +609,7 @@ int __mmu_notifier_register(struct mmu_notifier *subscription, · 608 unmodified lines
609	struct mmu_notifier_subscriptions *subscriptions = NULL;	609	struct mmu_notifier_subscriptions *subscriptions = NULL;
610	int ret;	610	int ret;
611		611
612	lockdep_assert_held_write(&mm->mmap_sem);	612	mmap_assert_write_locked(mm);
613	BUG_ON(atomic_read(&mm->mm_users) <= 0);	613	BUG_ON(atomic_read(&mm->mm_users) <= 0);
614		614
615	if (IS_ENABLED(CONFIG_LOCKDEP)) {	615	if (IS_ENABLED(CONFIG_LOCKDEP)) {
@@ -761,7 +761,7 @@ struct mmu_notifier mmu_notifier_get_locked(const struct mmu_notifier_ops ops, · 145 unmodified lines
761	struct mmu_notifier *subscription;	761	struct mmu_notifier *subscription;
762	int ret;	762	int ret;
763		763
764	lockdep_assert_held_write(&mm->mmap_sem);	764	mmap_assert_write_locked(mm);
765		765
766	if (mm->notifier_subscriptions) {	766	if (mm->notifier_subscriptions) {
767	subscription = find_get_mmu_notifier(mm, ops);	767	subscription = find_get_mmu_notifier(mm, ops);
@@ -1006,7 +1006,7 @@ int mmu_interval_notifier_insert_locked( · 238 unmodified lines
1006	mm->notifier_subscriptions;	1006	mm->notifier_subscriptions;
1007	int ret;	1007	int ret;
1008		1008
1009	lockdep_assert_held_write(&mm->mmap_sem);	1009	mmap_assert_write_locked(mm);
1010		1010
1011	if (!subscriptions \|\| !subscriptions->has_itree) {	1011	if (!subscriptions \|\| !subscriptions->has_itree) {
1012	ret = __mmu_notifier_register(NULL, mm);	1012	ret = __mmu_notifier_register(NULL, mm);

mm/pagewalk.c+3−3

@@ -395,7 +395,7 @@ int walk_page_range(struct mm_struct *mm, unsigned long start, · 394 unmodified lines
395	if (!walk.mm)	395	if (!walk.mm)
396	return -EINVAL;	396	return -EINVAL;
397		397
398	lockdep_assert_held(&walk.mm->mmap_sem);	398	mmap_assert_locked(walk.mm);
399		399
400	vma = find_vma(walk.mm, start);	400	vma = find_vma(walk.mm, start);
401	do {	401	do {
@@ -453,7 +453,7 @@ int walk_page_range_novma(struct mm_struct *mm, unsigned long start, · 51 unmodified lines
453	if (start >= end \|\| !walk.mm)	453	if (start >= end \|\| !walk.mm)
454	return -EINVAL;	454	return -EINVAL;
455		455
456	lockdep_assert_held(&walk.mm->mmap_sem);	456	mmap_assert_locked(walk.mm);
457		457
458	return __walk_page_range(start, end, &walk);	458	return __walk_page_range(start, end, &walk);
459	}	459	}
@@ -472,7 +472,7 @@ int walk_page_vma(struct vm_area_struct vma, const struct mm_walk_ops ops, · 12 unmodified lines
472	if (!walk.mm)	472	if (!walk.mm)
473	return -EINVAL;	473	return -EINVAL;
474		474
475	lockdep_assert_held(&walk.mm->mmap_sem);	475	mmap_assert_locked(walk.mm);
476		476
477	err = walk_page_test(vma->vm_start, vma->vm_end, &walk);	477	err = walk_page_test(vma->vm_start, vma->vm_end, &walk);
478	if (err > 0)	478	if (err > 0)

mm/util.c+1−1

@@ -437,7 +437,7 @@ int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc, · 436 unmodified lines
437	unsigned long locked_vm, limit;	437	unsigned long locked_vm, limit;
438	int ret = 0;	438	int ret = 0;
439		439
440	lockdep_assert_held_write(&mm->mmap_sem);	440	mmap_assert_write_locked(mm);
441		441
442	locked_vm = mm->locked_vm;	442	locked_vm = mm->locked_vm;
443	if (inc) {	443	if (inc) {