mm/mmu_notifier: add an interval tree notifier

Of the 13 users of mmu_notifiers, 8 of them use only invalidate_range_start/end() and immediately intersect the mmu_notifier_range with some kind of internal list of VAs. 4 use an interval tree (i915_gem, radeon_mn, umem_odp, hfi1). 4 use a linked list of some kind (scif_dma, vhost, gntdev, hmm) And the remaining 5 either don't use invalidate_range_start() or do some special thing with it. It turns out that building a correct scheme with an interval tree is pretty complicated, particularly if the use case is synchronizing against another thread doing get_user_pages(). Many of these implementations have various subtle and difficult to fix races. This approach puts the interval tree as common code at the top of the mmu notifier call tree and implements a shareable locking scheme. It includes: - An interval tree tracking VA ranges, with per-range callbacks - A read/write locking scheme for the interval tree that avoids sleeping in the notifier path (for OOM killer) - A sequence counter based collision-retry locking scheme to tell device page fault that a VA range is being concurrently invalidated. This is based on various ideas: - hmm accumulates invalidated VA ranges and releases them when all invalidates are done, via active_invalidate_ranges count. This approach avoids having to intersect the interval tree twice (as umem_odp does) at the potential cost of a longer device page fault. - kvm/umem_odp use a sequence counter to drive the collision retry, via invalidate_seq - a deferred work todo list on unlock scheme like RTNL, via deferred_list. This makes adding/removing interval tree members more deterministic - seqlock, except this version makes the seqlock idea multi-holder on the write side by protecting it with active_invalidate_ranges and a spinlock To minimize MM overhead when only the interval tree is being used, the entire SRCU and hlist overheads are dropped using some simple branches. Similarly the interval tree overhead is dropped when in hlist mode. The overhead from the mandatory spinlock is broadly the same as most of existing users which already had a lock (or two) of some sort on the invalidation path.

Jason Gunthorpe committed 7 years ago

Commit 99cb2521 parent 56f434f3 files · +620 −26Browse files

Unified · Split

3 files

include/linux/mmu_notifier.h +101 −0
mm/Kconfig +1 −0
mm/mmu_notifier.c +518 −26

include/linux/mmu_notifier.h+101−0

@@ -6,10 +6,12 @@ · 5 unmodified lines
6	#include <linux/spinlock.h>	6	#include <linux/spinlock.h>
7	#include <linux/mm_types.h>	7	#include <linux/mm_types.h>
8	#include <linux/srcu.h>	8	#include <linux/srcu.h>
		9	#include <linux/interval_tree.h>
9		10
10	struct mmu_notifier_mm;	11	struct mmu_notifier_mm;
11	struct mmu_notifier;	12	struct mmu_notifier;
12	struct mmu_notifier_range;	13	struct mmu_notifier_range;
		14	struct mmu_interval_notifier;
13		15
14	/**	16	/**
15	* enum mmu_notifier_event - reason for the mmu notifier callback	17	* enum mmu_notifier_event - reason for the mmu notifier callback
@@ -32,6 +34,9 @@ struct mmu_notifier_range; · 16 unmodified lines
32	* access flags). User should soft dirty the page in the end callback to make	34	* access flags). User should soft dirty the page in the end callback to make
33	* sure that anyone relying on soft dirtyness catch pages that might be written	35	* sure that anyone relying on soft dirtyness catch pages that might be written
34	* through non CPU mappings.	36	* through non CPU mappings.
		37	*
		38	* @MMU_NOTIFY_RELEASE: used during mmu_interval_notifier invalidate to signal
		39	* that the mm refcount is zero and the range is no longer accessible.
35	*/	40	*/
36	enum mmu_notifier_event {	41	enum mmu_notifier_event {
37	MMU_NOTIFY_UNMAP = 0,	42	MMU_NOTIFY_UNMAP = 0,
@@ -39,6 +44,7 @@ enum mmu_notifier_event { · 1 unmodified line
39	MMU_NOTIFY_PROTECTION_VMA,	44	MMU_NOTIFY_PROTECTION_VMA,
40	MMU_NOTIFY_PROTECTION_PAGE,	45	MMU_NOTIFY_PROTECTION_PAGE,
41	MMU_NOTIFY_SOFT_DIRTY,	46	MMU_NOTIFY_SOFT_DIRTY,
		47	MMU_NOTIFY_RELEASE,
42	};	48	};
43		49
44	#define MMU_NOTIFIER_RANGE_BLOCKABLE (1 << 0)	50	#define MMU_NOTIFIER_RANGE_BLOCKABLE (1 << 0)
@@ -222,6 +228,26 @@ struct mmu_notifier { · 177 unmodified lines
222	unsigned int users;	228	unsigned int users;
223	};	229	};
224		230
		231	/**
		232	* struct mmu_interval_notifier_ops
		233	* @invalidate: Upon return the caller must stop using any SPTEs within this
		234	* range. This function can sleep. Return false only if sleeping
		235	* was required but mmu_notifier_range_blockable(range) is false.
		236	*/
		237	struct mmu_interval_notifier_ops {
		238	bool (invalidate)(struct mmu_interval_notifier mni,
		239	const struct mmu_notifier_range *range,
		240	unsigned long cur_seq);
		241	};
		242
		243	struct mmu_interval_notifier {
		244	struct interval_tree_node interval_tree;
		245	const struct mmu_interval_notifier_ops *ops;
		246	struct mm_struct *mm;
		247	struct hlist_node deferred_item;
		248	unsigned long invalidate_seq;
		249	};
		250
225	#ifdef CONFIG_MMU_NOTIFIER	251	#ifdef CONFIG_MMU_NOTIFIER
226		252
227	#ifdef CONFIG_LOCKDEP	253	#ifdef CONFIG_LOCKDEP
@@ -263,6 +289,81 @@ extern int __mmu_notifier_register(struct mmu_notifier *mn, · 35 unmodified lines
263	struct mm_struct *mm);	289	struct mm_struct *mm);
264	extern void mmu_notifier_unregister(struct mmu_notifier *mn,	290	extern void mmu_notifier_unregister(struct mmu_notifier *mn,
265	struct mm_struct *mm);	291	struct mm_struct *mm);
		292
		293	unsigned long mmu_interval_read_begin(struct mmu_interval_notifier *mni);
		294	int mmu_interval_notifier_insert(struct mmu_interval_notifier *mni,
		295	struct mm_struct *mm, unsigned long start,
		296	unsigned long length,
		297	const struct mmu_interval_notifier_ops *ops);
		298	int mmu_interval_notifier_insert_locked(
		299	struct mmu_interval_notifier mni, struct mm_struct mm,
		300	unsigned long start, unsigned long length,
		301	const struct mmu_interval_notifier_ops *ops);
		302	void mmu_interval_notifier_remove(struct mmu_interval_notifier *mni);
		303
		304	/**
		305	* mmu_interval_set_seq - Save the invalidation sequence
		306	* @mni - The mni passed to invalidate
		307	* @cur_seq - The cur_seq passed to the invalidate() callback
		308	*
		309	* This must be called unconditionally from the invalidate callback of a
		310	* struct mmu_interval_notifier_ops under the same lock that is used to call
		311	* mmu_interval_read_retry(). It updates the sequence number for later use by
		312	* mmu_interval_read_retry(). The provided cur_seq will always be odd.
		313	*
		314	* If the caller does not call mmu_interval_read_begin() or
		315	* mmu_interval_read_retry() then this call is not required.
		316	*/
		317	static inline void mmu_interval_set_seq(struct mmu_interval_notifier *mni,
		318	unsigned long cur_seq)
		319	{
		320	WRITE_ONCE(mni->invalidate_seq, cur_seq);
		321	}
		322
		323	/**
		324	* mmu_interval_read_retry - End a read side critical section against a VA range
		325	* mni: The range
		326	* seq: The return of the paired mmu_interval_read_begin()
		327	*
		328	* This MUST be called under a user provided lock that is also held
		329	* unconditionally by op->invalidate() when it calls mmu_interval_set_seq().
		330	*
		331	* Each call should be paired with a single mmu_interval_read_begin() and
		332	* should be used to conclude the read side.
		333	*
		334	* Returns true if an invalidation collided with this critical section, and
		335	* the caller should retry.
		336	*/
		337	static inline bool mmu_interval_read_retry(struct mmu_interval_notifier *mni,
		338	unsigned long seq)
		339	{
		340	return mni->invalidate_seq != seq;
		341	}
		342
		343	/**
		344	* mmu_interval_check_retry - Test if a collision has occurred
		345	* mni: The range
		346	* seq: The return of the matching mmu_interval_read_begin()
		347	*
		348	* This can be used in the critical section between mmu_interval_read_begin()
		349	* and mmu_interval_read_retry(). A return of true indicates an invalidation
		350	* has collided with this critical region and a future
		351	* mmu_interval_read_retry() will return true.
		352	*
		353	* False is not reliable and only suggests a collision may not have
		354	* occured. It can be called many times and does not have to hold the user
		355	* provided lock.
		356	*
		357	* This call can be used as part of loops and other expensive operations to
		358	* expedite a retry.
		359	*/
		360	static inline bool mmu_interval_check_retry(struct mmu_interval_notifier *mni,
		361	unsigned long seq)
		362	{
		363	/* Pairs with the WRITE_ONCE in mmu_interval_set_seq() */
		364	return READ_ONCE(mni->invalidate_seq) != seq;
		365	}
		366
266	extern void __mmu_notifier_mm_destroy(struct mm_struct *mm);	367	extern void __mmu_notifier_mm_destroy(struct mm_struct *mm);
267	extern void __mmu_notifier_release(struct mm_struct *mm);	368	extern void __mmu_notifier_release(struct mm_struct *mm);
268	extern int __mmu_notifier_clear_flush_young(struct mm_struct *mm,	369	extern int __mmu_notifier_clear_flush_young(struct mm_struct *mm,

mm/Kconfig+1−0

@@ -284,6 +284,7 @@ config VIRT_TO_BUS · 283 unmodified lines
284	config MMU_NOTIFIER	284	config MMU_NOTIFIER
285	bool	285	bool
286	select SRCU	286	select SRCU
		287	select INTERVAL_TREE
287		288
288	config KSM	289	config KSM
289	bool "Enable KSM for page merging"	290	bool "Enable KSM for page merging"

mm/mmu_notifier.c+518−26

@@ -12,6 +12,7 @@ · 11 unmodified lines
12	#include <linux/export.h>	12	#include <linux/export.h>
13	#include <linux/mm.h>	13	#include <linux/mm.h>
14	#include <linux/err.h>	14	#include <linux/err.h>
		15	#include <linux/interval_tree.h>
15	#include <linux/srcu.h>	16	#include <linux/srcu.h>
16	#include <linux/rcupdate.h>	17	#include <linux/rcupdate.h>
17	#include <linux/sched.h>	18	#include <linux/sched.h>
@@ -36,10 +37,245 @@ struct lockdep_map __mmu_notifier_invalidate_range_start_map = { · 18 unmodified lines
36	struct mmu_notifier_mm {	37	struct mmu_notifier_mm {
37	/* all mmu notifiers registered in this mm are queued in this list */	38	/* all mmu notifiers registered in this mm are queued in this list */
38	struct hlist_head list;	39	struct hlist_head list;
		40	bool has_itree;
39	/* to serialize the list modifications and hlist_unhashed */	41	/* to serialize the list modifications and hlist_unhashed */
40	spinlock_t lock;	42	spinlock_t lock;
		43	unsigned long invalidate_seq;
		44	unsigned long active_invalidate_ranges;
		45	struct rb_root_cached itree;
		46	wait_queue_head_t wq;
		47	struct hlist_head deferred_list;
41	};	48	};
42		49
		50	/*
		51	* This is a collision-retry read-side/write-side 'lock', a lot like a
		52	* seqcount, however this allows multiple write-sides to hold it at
		53	* once. Conceptually the write side is protecting the values of the PTEs in
		54	* this mm, such that PTES cannot be read into SPTEs (shadow PTEs) while any
		55	* writer exists.
		56	*
		57	* Note that the core mm creates nested invalidate_range_start()/end() regions
		58	* within the same thread, and runs invalidate_range_start()/end() in parallel
		59	* on multiple CPUs. This is designed to not reduce concurrency or block
		60	* progress on the mm side.
		61	*
		62	* As a secondary function, holding the full write side also serves to prevent
		63	* writers for the itree, this is an optimization to avoid extra locking
		64	* during invalidate_range_start/end notifiers.
		65	*
		66	* The write side has two states, fully excluded:
		67	* - mm->active_invalidate_ranges != 0
		68	* - mnn->invalidate_seq & 1 == True (odd)
		69	* - some range on the mm_struct is being invalidated
		70	* - the itree is not allowed to change
		71	*
		72	* And partially excluded:
		73	* - mm->active_invalidate_ranges != 0
		74	* - mnn->invalidate_seq & 1 == False (even)
		75	* - some range on the mm_struct is being invalidated
		76	* - the itree is allowed to change
		77	*
		78	* Operations on mmu_notifier_mm->invalidate_seq (under spinlock):
		79	* seq \|= 1 # Begin writing
		80	* seq++ # Release the writing state
		81	* seq & 1 # True if a writer exists
		82	*
		83	* The later state avoids some expensive work on inv_end in the common case of
		84	* no mni monitoring the VA.
		85	*/
		86	static bool mn_itree_is_invalidating(struct mmu_notifier_mm *mmn_mm)
		87	{
		88	lockdep_assert_held(&mmn_mm->lock);
		89	return mmn_mm->invalidate_seq & 1;
		90	}
		91
		92	static struct mmu_interval_notifier *
		93	mn_itree_inv_start_range(struct mmu_notifier_mm *mmn_mm,
		94	const struct mmu_notifier_range *range,
		95	unsigned long *seq)
		96	{
		97	struct interval_tree_node *node;
		98	struct mmu_interval_notifier *res = NULL;
		99
		100	spin_lock(&mmn_mm->lock);
		101	mmn_mm->active_invalidate_ranges++;
		102	node = interval_tree_iter_first(&mmn_mm->itree, range->start,
		103	range->end - 1);
		104	if (node) {
		105	mmn_mm->invalidate_seq \|= 1;
		106	res = container_of(node, struct mmu_interval_notifier,
		107	interval_tree);
		108	}
		109
		110	*seq = mmn_mm->invalidate_seq;
		111	spin_unlock(&mmn_mm->lock);
		112	return res;
		113	}
		114
		115	static struct mmu_interval_notifier *
		116	mn_itree_inv_next(struct mmu_interval_notifier *mni,
		117	const struct mmu_notifier_range *range)
		118	{
		119	struct interval_tree_node *node;
		120
		121	node = interval_tree_iter_next(&mni->interval_tree, range->start,
		122	range->end - 1);
		123	if (!node)
		124	return NULL;
		125	return container_of(node, struct mmu_interval_notifier, interval_tree);
		126	}
		127
		128	static void mn_itree_inv_end(struct mmu_notifier_mm *mmn_mm)
		129	{
		130	struct mmu_interval_notifier *mni;
		131	struct hlist_node *next;
		132
		133	spin_lock(&mmn_mm->lock);
		134	if (--mmn_mm->active_invalidate_ranges \|\|
		135	!mn_itree_is_invalidating(mmn_mm)) {
		136	spin_unlock(&mmn_mm->lock);
		137	return;
		138	}
		139
		140	/* Make invalidate_seq even */
		141	mmn_mm->invalidate_seq++;
		142
		143	/*
		144	* The inv_end incorporates a deferred mechanism like rtnl_unlock().
		145	* Adds and removes are queued until the final inv_end happens then
		146	* they are progressed. This arrangement for tree updates is used to
		147	* avoid using a blocking lock during invalidate_range_start.
		148	*/
		149	hlist_for_each_entry_safe(mni, next, &mmn_mm->deferred_list,
		150	deferred_item) {
		151	if (RB_EMPTY_NODE(&mni->interval_tree.rb))
		152	interval_tree_insert(&mni->interval_tree,
		153	&mmn_mm->itree);
		154	else
		155	interval_tree_remove(&mni->interval_tree,
		156	&mmn_mm->itree);
		157	hlist_del(&mni->deferred_item);
		158	}
		159	spin_unlock(&mmn_mm->lock);
		160
		161	wake_up_all(&mmn_mm->wq);
		162	}
		163
		164	/**
		165	* mmu_interval_read_begin - Begin a read side critical section against a VA
		166	* range
		167	* mni: The range to use
		168	*
		169	* mmu_iterval_read_begin()/mmu_iterval_read_retry() implement a
		170	* collision-retry scheme similar to seqcount for the VA range under mni. If
		171	* the mm invokes invalidation during the critical section then
		172	* mmu_interval_read_retry() will return true.
		173	*
		174	* This is useful to obtain shadow PTEs where teardown or setup of the SPTEs
		175	* require a blocking context. The critical region formed by this can sleep,
		176	* and the required 'user_lock' can also be a sleeping lock.
		177	*
		178	* The caller is required to provide a 'user_lock' to serialize both teardown
		179	* and setup.
		180	*
		181	* The return value should be passed to mmu_interval_read_retry().
		182	*/
		183	unsigned long mmu_interval_read_begin(struct mmu_interval_notifier *mni)
		184	{
		185	struct mmu_notifier_mm *mmn_mm = mni->mm->mmu_notifier_mm;
		186	unsigned long seq;
		187	bool is_invalidating;
		188
		189	/*
		190	* If the mni has a different seq value under the user_lock than we
		191	* started with then it has collided.
		192	*
		193	* If the mni currently has the same seq value as the mmn_mm seq, then
		194	* it is currently between invalidate_start/end and is colliding.
		195	*
		196	* The locking looks broadly like this:
		197	* mn_tree_invalidate_start(): mmu_interval_read_begin():
		198	* spin_lock
		199	* seq = READ_ONCE(mni->invalidate_seq);
		200	* seq == mmn_mm->invalidate_seq
		201	* spin_unlock
		202	* spin_lock
		203	* seq = ++mmn_mm->invalidate_seq
		204	* spin_unlock
		205	* op->invalidate_range():
		206	* user_lock
		207	* mmu_interval_set_seq()
		208	* mni->invalidate_seq = seq
		209	* user_unlock
		210	*
		211	* [Required: mmu_interval_read_retry() == true]
		212	*
		213	* mn_itree_inv_end():
		214	* spin_lock
		215	* seq = ++mmn_mm->invalidate_seq
		216	* spin_unlock
		217	*
		218	* user_lock
		219	* mmu_interval_read_retry():
		220	* mni->invalidate_seq != seq
		221	* user_unlock
		222	*
		223	* Barriers are not needed here as any races here are closed by an
		224	* eventual mmu_interval_read_retry(), which provides a barrier via the
		225	* user_lock.
		226	*/
		227	spin_lock(&mmn_mm->lock);
		228	/* Pairs with the WRITE_ONCE in mmu_interval_set_seq() */
		229	seq = READ_ONCE(mni->invalidate_seq);
		230	is_invalidating = seq == mmn_mm->invalidate_seq;
		231	spin_unlock(&mmn_mm->lock);
		232
		233	/*
		234	* mni->invalidate_seq must always be set to an odd value via
		235	* mmu_interval_set_seq() using the provided cur_seq from
		236	* mn_itree_inv_start_range(). This ensures that if seq does wrap we
		237	* will always clear the below sleep in some reasonable time as
		238	* mmn_mm->invalidate_seq is even in the idle state.
		239	*/
		240	lock_map_acquire(&__mmu_notifier_invalidate_range_start_map);
		241	lock_map_release(&__mmu_notifier_invalidate_range_start_map);
		242	if (is_invalidating)
		243	wait_event(mmn_mm->wq,
		244	READ_ONCE(mmn_mm->invalidate_seq) != seq);
		245
		246	/*
		247	* Notice that mmu_interval_read_retry() can already be true at this
		248	* point, avoiding loops here allows the caller to provide a global
		249	* time bound.
		250	*/
		251
		252	return seq;
		253	}
		254	EXPORT_SYMBOL_GPL(mmu_interval_read_begin);
		255
		256	static void mn_itree_release(struct mmu_notifier_mm *mmn_mm,
		257	struct mm_struct *mm)
		258	{
		259	struct mmu_notifier_range range = {
		260	.flags = MMU_NOTIFIER_RANGE_BLOCKABLE,
		261	.event = MMU_NOTIFY_RELEASE,
		262	.mm = mm,
		263	.start = 0,
		264	.end = ULONG_MAX,
		265	};
		266	struct mmu_interval_notifier *mni;
		267	unsigned long cur_seq;
		268	bool ret;
		269
		270	for (mni = mn_itree_inv_start_range(mmn_mm, &range, &cur_seq); mni;
		271	mni = mn_itree_inv_next(mni, &range)) {
		272	ret = mni->ops->invalidate(mni, &range, cur_seq);
		273	WARN_ON(!ret);
		274	}
		275
		276	mn_itree_inv_end(mmn_mm);
		277	}
		278
43	/*	279	/*
44	* This function can't run concurrently against mmu_notifier_register	280	* This function can't run concurrently against mmu_notifier_register
45	* because mm->mm_users > 0 during mmu_notifier_register and exit_mmap	281	* because mm->mm_users > 0 during mmu_notifier_register and exit_mmap
@@ -52,7 +288,8 @@ struct mmu_notifier_mm { · 6 unmodified lines
52	* can't go away from under us as exit_mmap holds an mm_count pin	288	* can't go away from under us as exit_mmap holds an mm_count pin
53	* itself.	289	* itself.
54	*/	290	*/
55	void __mmu_notifier_release(struct mm_struct *mm)	291	static void mn_hlist_release(struct mmu_notifier_mm *mmn_mm,
		292	struct mm_struct *mm)
56	{	293	{
57	struct mmu_notifier *mn;	294	struct mmu_notifier *mn;
58	int id;	295	int id;
@@ -62,7 +299,7 @@ void __mmu_notifier_release(struct mm_struct *mm) · 3 unmodified lines
62	* ->release returns.	299	* ->release returns.
63	*/	300	*/
64	id = srcu_read_lock(&srcu);	301	id = srcu_read_lock(&srcu);
65	hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist)	302	hlist_for_each_entry_rcu(mn, &mmn_mm->list, hlist)
66	/*	303	/*
67	* If ->release runs before mmu_notifier_unregister it must be	304	* If ->release runs before mmu_notifier_unregister it must be
68	* handled, as it's the only way for the driver to flush all	305	* handled, as it's the only way for the driver to flush all
@@ -72,10 +309,9 @@ void __mmu_notifier_release(struct mm_struct *mm) · 3 unmodified lines
72	if (mn->ops->release)	309	if (mn->ops->release)
73	mn->ops->release(mn, mm);	310	mn->ops->release(mn, mm);
74		311
75	spin_lock(&mm->mmu_notifier_mm->lock);	312	spin_lock(&mmn_mm->lock);
76	while (unlikely(!hlist_empty(&mm->mmu_notifier_mm->list))) {	313	while (unlikely(!hlist_empty(&mmn_mm->list))) {
77	mn = hlist_entry(mm->mmu_notifier_mm->list.first,	314	mn = hlist_entry(mmn_mm->list.first, struct mmu_notifier,
78	struct mmu_notifier,
79	hlist);	315	hlist);
80	/*	316	/*
81	* We arrived before mmu_notifier_unregister so	317	* We arrived before mmu_notifier_unregister so
@@ -85,7 +321,7 @@ void __mmu_notifier_release(struct mm_struct *mm) · 3 unmodified lines
85	*/	321	*/
86	hlist_del_init_rcu(&mn->hlist);	322	hlist_del_init_rcu(&mn->hlist);
87	}	323	}
88	spin_unlock(&mm->mmu_notifier_mm->lock);	324	spin_unlock(&mmn_mm->lock);
89	srcu_read_unlock(&srcu, id);	325	srcu_read_unlock(&srcu, id);
90		326
91	/*	327	/*
@@ -100,6 +336,17 @@ void __mmu_notifier_release(struct mm_struct *mm) · 8 unmodified lines
100	synchronize_srcu(&srcu);	336	synchronize_srcu(&srcu);
101	}	337	}
102		338
		339	void __mmu_notifier_release(struct mm_struct *mm)
		340	{
		341	struct mmu_notifier_mm *mmn_mm = mm->mmu_notifier_mm;
		342
		343	if (mmn_mm->has_itree)
		344	mn_itree_release(mmn_mm, mm);
		345
		346	if (!hlist_empty(&mmn_mm->list))
		347	mn_hlist_release(mmn_mm, mm);
		348	}
		349
103	/*	350	/*
104	* If no young bitflag is supported by the hardware, ->clear_flush_young can	351	* If no young bitflag is supported by the hardware, ->clear_flush_young can
105	* unmap the address and return 1 or 0 depending if the mapping previously	352	* unmap the address and return 1 or 0 depending if the mapping previously
@@ -172,14 +419,43 @@ void __mmu_notifier_change_pte(struct mm_struct *mm, unsigned long address, · 66 unmodified lines
172	srcu_read_unlock(&srcu, id);	419	srcu_read_unlock(&srcu, id);
173	}	420	}
174		421
175	int __mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range)	422	static int mn_itree_invalidate(struct mmu_notifier_mm *mmn_mm,
		423	const struct mmu_notifier_range *range)
		424	{
		425	struct mmu_interval_notifier *mni;
		426	unsigned long cur_seq;
		427
		428	for (mni = mn_itree_inv_start_range(mmn_mm, range, &cur_seq); mni;
		429	mni = mn_itree_inv_next(mni, range)) {
		430	bool ret;
		431
		432	ret = mni->ops->invalidate(mni, range, cur_seq);
		433	if (!ret) {
		434	if (WARN_ON(mmu_notifier_range_blockable(range)))
		435	continue;
		436	goto out_would_block;
		437	}
		438	}
		439	return 0;
		440
		441	out_would_block:
		442	/*
		443	* On -EAGAIN the non-blocking caller is not allowed to call
		444	* invalidate_range_end()
		445	*/
		446	mn_itree_inv_end(mmn_mm);
		447	return -EAGAIN;
		448	}
		449
		450	static int mn_hlist_invalidate_range_start(struct mmu_notifier_mm *mmn_mm,
		451	struct mmu_notifier_range *range)
176	{	452	{
177	struct mmu_notifier *mn;	453	struct mmu_notifier *mn;
178	int ret = 0;	454	int ret = 0;
179	int id;	455	int id;
180		456
181	id = srcu_read_lock(&srcu);	457	id = srcu_read_lock(&srcu);
182	hlist_for_each_entry_rcu(mn, &range->mm->mmu_notifier_mm->list, hlist) {	458	hlist_for_each_entry_rcu(mn, &mmn_mm->list, hlist) {
183	if (mn->ops->invalidate_range_start) {	459	if (mn->ops->invalidate_range_start) {
184	int _ret;	460	int _ret;
185		461
@@ -203,15 +479,30 @@ int __mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range) · 17 unmodified lines
203	return ret;	479	return ret;
204	}	480	}
205		481
206	void __mmu_notifier_invalidate_range_end(struct mmu_notifier_range *range,	482	int __mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range)
207	bool only_end)	483	{
		484	struct mmu_notifier_mm *mmn_mm = range->mm->mmu_notifier_mm;
		485	int ret;
		486
		487	if (mmn_mm->has_itree) {
		488	ret = mn_itree_invalidate(mmn_mm, range);
		489	if (ret)
		490	return ret;
		491	}
		492	if (!hlist_empty(&mmn_mm->list))
		493	return mn_hlist_invalidate_range_start(mmn_mm, range);
		494	return 0;
		495	}
		496
		497	static void mn_hlist_invalidate_end(struct mmu_notifier_mm *mmn_mm,
		498	struct mmu_notifier_range *range,
		499	bool only_end)
208	{	500	{
209	struct mmu_notifier *mn;	501	struct mmu_notifier *mn;
210	int id;	502	int id;
211		503
212	lock_map_acquire(&__mmu_notifier_invalidate_range_start_map);
213	id = srcu_read_lock(&srcu);	504	id = srcu_read_lock(&srcu);
214	hlist_for_each_entry_rcu(mn, &range->mm->mmu_notifier_mm->list, hlist) {	505	hlist_for_each_entry_rcu(mn, &mmn_mm->list, hlist) {
215	/*	506	/*
216	* Call invalidate_range here too to avoid the need for the	507	* Call invalidate_range here too to avoid the need for the
217	* subsystem of having to register an invalidate_range_end	508	* subsystem of having to register an invalidate_range_end
@@ -238,6 +529,19 @@ void __mmu_notifier_invalidate_range_end(struct mmu_notifier_range *range, · 20 unmodified lines
238	}	529	}
239	}	530	}
240	srcu_read_unlock(&srcu, id);	531	srcu_read_unlock(&srcu, id);
		532	}
		533
		534	void __mmu_notifier_invalidate_range_end(struct mmu_notifier_range *range,
		535	bool only_end)
		536	{
		537	struct mmu_notifier_mm *mmn_mm = range->mm->mmu_notifier_mm;
		538
		539	lock_map_acquire(&__mmu_notifier_invalidate_range_start_map);
		540	if (mmn_mm->has_itree)
		541	mn_itree_inv_end(mmn_mm);
		542
		543	if (!hlist_empty(&mmn_mm->list))
		544	mn_hlist_invalidate_end(mmn_mm, range, only_end);
241	lock_map_release(&__mmu_notifier_invalidate_range_start_map);	545	lock_map_release(&__mmu_notifier_invalidate_range_start_map);
242	}	546	}
243		547
@@ -256,8 +560,9 @@ void __mmu_notifier_invalidate_range(struct mm_struct *mm, · 12 unmodified lines
256	}	560	}
257		561
258	/*	562	/*
259	* Same as mmu_notifier_register but here the caller must hold the	563	* Same as mmu_notifier_register but here the caller must hold the mmap_sem in
260	* mmap_sem in write mode.	564	* write mode. A NULL mn signals the notifier is being registered for itree
		565	* mode.
261	*/	566	*/
262	int __mmu_notifier_register(struct mmu_notifier mn, struct mm_struct mm)	567	int __mmu_notifier_register(struct mmu_notifier mn, struct mm_struct mm)
263	{	568	{
@@ -274,9 +579,6 @@ int __mmu_notifier_register(struct mmu_notifier mn, struct mm_struct mm) · 10 unmodified lines
274	fs_reclaim_release(GFP_KERNEL);	579	fs_reclaim_release(GFP_KERNEL);
275	}	580	}
276		581
277	mn->mm = mm;
278	mn->users = 1;
279
280	if (!mm->mmu_notifier_mm) {	582	if (!mm->mmu_notifier_mm) {
281	/*	583	/*
282	* kmalloc cannot be called under mm_take_all_locks(), but we	584	* kmalloc cannot be called under mm_take_all_locks(), but we
@@ -284,21 +586,22 @@ int __mmu_notifier_register(struct mmu_notifier mn, struct mm_struct mm) · 1 unmodified line
284	* the write side of the mmap_sem.	586	* the write side of the mmap_sem.
285	*/	587	*/
286	mmu_notifier_mm =	588	mmu_notifier_mm =
287	kmalloc(sizeof(struct mmu_notifier_mm), GFP_KERNEL);	589	kzalloc(sizeof(struct mmu_notifier_mm), GFP_KERNEL);
288	if (!mmu_notifier_mm)	590	if (!mmu_notifier_mm)
289	return -ENOMEM;	591	return -ENOMEM;
290		592
291	INIT_HLIST_HEAD(&mmu_notifier_mm->list);	593	INIT_HLIST_HEAD(&mmu_notifier_mm->list);
292	spin_lock_init(&mmu_notifier_mm->lock);	594	spin_lock_init(&mmu_notifier_mm->lock);
		595	mmu_notifier_mm->invalidate_seq = 2;
		596	mmu_notifier_mm->itree = RB_ROOT_CACHED;
		597	init_waitqueue_head(&mmu_notifier_mm->wq);
		598	INIT_HLIST_HEAD(&mmu_notifier_mm->deferred_list);
293	}	599	}
294		600
295	ret = mm_take_all_locks(mm);	601	ret = mm_take_all_locks(mm);
296	if (unlikely(ret))	602	if (unlikely(ret))
297	goto out_clean;	603	goto out_clean;
298		604
299	/* Pairs with the mmdrop in mmu_notifier_unregister_* */
300	mmgrab(mm);
301
302	/*	605	/*
303	* Serialize the update against mmu_notifier_unregister. A	606	* Serialize the update against mmu_notifier_unregister. A
304	* side note: mmu_notifier_release can't run concurrently with	607	* side note: mmu_notifier_release can't run concurrently with
@@ -306,13 +609,28 @@ int __mmu_notifier_register(struct mmu_notifier mn, struct mm_struct mm) · 1 unmodified line
306	* current->mm or explicitly with get_task_mm() or similar).	609	* current->mm or explicitly with get_task_mm() or similar).
307	* We can't race against any other mmu notifier method either	610	* We can't race against any other mmu notifier method either
308	* thanks to mm_take_all_locks().	611	* thanks to mm_take_all_locks().
		612	*
		613	* release semantics on the initialization of the mmu_notifier_mm's
		614	* contents are provided for unlocked readers. acquire can only be
		615	* used while holding the mmgrab or mmget, and is safe because once
		616	* created the mmu_notififer_mm is not freed until the mm is
		617	* destroyed. As above, users holding the mmap_sem or one of the
		618	* mm_take_all_locks() do not need to use acquire semantics.
309	*/	619	*/
310	if (mmu_notifier_mm)	620	if (mmu_notifier_mm)
311	mm->mmu_notifier_mm = mmu_notifier_mm;	621	smp_store_release(&mm->mmu_notifier_mm, mmu_notifier_mm);
312		622
313	spin_lock(&mm->mmu_notifier_mm->lock);	623	if (mn) {
314	hlist_add_head_rcu(&mn->hlist, &mm->mmu_notifier_mm->list);	624	/* Pairs with the mmdrop in mmu_notifier_unregister_* */
315	spin_unlock(&mm->mmu_notifier_mm->lock);	625	mmgrab(mm);
		626	mn->mm = mm;
		627	mn->users = 1;
		628
		629	spin_lock(&mm->mmu_notifier_mm->lock);
		630	hlist_add_head_rcu(&mn->hlist, &mm->mmu_notifier_mm->list);
		631	spin_unlock(&mm->mmu_notifier_mm->lock);
		632	} else
		633	mm->mmu_notifier_mm->has_itree = true;
316		634
317	mm_drop_all_locks(mm);	635	mm_drop_all_locks(mm);
318	BUG_ON(atomic_read(&mm->mm_users) <= 0);	636	BUG_ON(atomic_read(&mm->mm_users) <= 0);
@@ -529,6 +847,180 @@ out_unlock: · 210 unmodified lines
529	}	847	}
530	EXPORT_SYMBOL_GPL(mmu_notifier_put);	848	EXPORT_SYMBOL_GPL(mmu_notifier_put);
531		849
		850	static int __mmu_interval_notifier_insert(
		851	struct mmu_interval_notifier mni, struct mm_struct mm,
		852	struct mmu_notifier_mm *mmn_mm, unsigned long start,
		853	unsigned long length, const struct mmu_interval_notifier_ops *ops)
		854	{
		855	mni->mm = mm;
		856	mni->ops = ops;
		857	RB_CLEAR_NODE(&mni->interval_tree.rb);
		858	mni->interval_tree.start = start;
		859	/*
		860	* Note that the representation of the intervals in the interval tree
		861	* considers the ending point as contained in the interval.
		862	*/
		863	if (length == 0 \|\|
		864	check_add_overflow(start, length - 1, &mni->interval_tree.last))
		865	return -EOVERFLOW;
		866
		867	/* Must call with a mmget() held */
		868	if (WARN_ON(atomic_read(&mm->mm_count) <= 0))
		869	return -EINVAL;
		870
		871	/* pairs with mmdrop in mmu_interval_notifier_remove() */
		872	mmgrab(mm);
		873
		874	/*
		875	* If some invalidate_range_start/end region is going on in parallel
		876	* we don't know what VA ranges are affected, so we must assume this
		877	* new range is included.
		878	*
		879	* If the itree is invalidating then we are not allowed to change
		880	* it. Retrying until invalidation is done is tricky due to the
		881	* possibility for live lock, instead defer the add to
		882	* mn_itree_inv_end() so this algorithm is deterministic.
		883	*
		884	* In all cases the value for the mni->invalidate_seq should be
		885	* odd, see mmu_interval_read_begin()
		886	*/
		887	spin_lock(&mmn_mm->lock);
		888	if (mmn_mm->active_invalidate_ranges) {
		889	if (mn_itree_is_invalidating(mmn_mm))
		890	hlist_add_head(&mni->deferred_item,
		891	&mmn_mm->deferred_list);
		892	else {
		893	mmn_mm->invalidate_seq \|= 1;
		894	interval_tree_insert(&mni->interval_tree,
		895	&mmn_mm->itree);
		896	}
		897	mni->invalidate_seq = mmn_mm->invalidate_seq;
		898	} else {
		899	WARN_ON(mn_itree_is_invalidating(mmn_mm));
		900	/*
		901	* The starting seq for a mni not under invalidation should be
		902	* odd, not equal to the current invalidate_seq and
		903	* invalidate_seq should not 'wrap' to the new seq any time
		904	* soon.
		905	*/
		906	mni->invalidate_seq = mmn_mm->invalidate_seq - 1;
		907	interval_tree_insert(&mni->interval_tree, &mmn_mm->itree);
		908	}
		909	spin_unlock(&mmn_mm->lock);
		910	return 0;
		911	}
		912
		913	/**
		914	* mmu_interval_notifier_insert - Insert an interval notifier
		915	* @mni: Interval notifier to register
		916	* @start: Starting virtual address to monitor
		917	* @length: Length of the range to monitor
		918	* @mm : mm_struct to attach to
		919	*
		920	* This function subscribes the interval notifier for notifications from the
		921	* mm. Upon return the ops related to mmu_interval_notifier will be called
		922	* whenever an event that intersects with the given range occurs.
		923	*
		924	* Upon return the range_notifier may not be present in the interval tree yet.
		925	* The caller must use the normal interval notifier read flow via
		926	* mmu_interval_read_begin() to establish SPTEs for this range.
		927	*/
		928	int mmu_interval_notifier_insert(struct mmu_interval_notifier *mni,
		929	struct mm_struct *mm, unsigned long start,
		930	unsigned long length,
		931	const struct mmu_interval_notifier_ops *ops)
		932	{
		933	struct mmu_notifier_mm *mmn_mm;
		934	int ret;
		935
		936	might_lock(&mm->mmap_sem);
		937
		938	mmn_mm = smp_load_acquire(&mm->mmu_notifier_mm);
		939	if (!mmn_mm \|\| !mmn_mm->has_itree) {
		940	ret = mmu_notifier_register(NULL, mm);
		941	if (ret)
		942	return ret;
		943	mmn_mm = mm->mmu_notifier_mm;
		944	}
		945	return __mmu_interval_notifier_insert(mni, mm, mmn_mm, start, length,
		946	ops);
		947	}
		948	EXPORT_SYMBOL_GPL(mmu_interval_notifier_insert);
		949
		950	int mmu_interval_notifier_insert_locked(
		951	struct mmu_interval_notifier mni, struct mm_struct mm,
		952	unsigned long start, unsigned long length,
		953	const struct mmu_interval_notifier_ops *ops)
		954	{
		955	struct mmu_notifier_mm *mmn_mm;
		956	int ret;
		957
		958	lockdep_assert_held_write(&mm->mmap_sem);
		959
		960	mmn_mm = mm->mmu_notifier_mm;
		961	if (!mmn_mm \|\| !mmn_mm->has_itree) {
		962	ret = __mmu_notifier_register(NULL, mm);
		963	if (ret)
		964	return ret;
		965	mmn_mm = mm->mmu_notifier_mm;
		966	}
		967	return __mmu_interval_notifier_insert(mni, mm, mmn_mm, start, length,
		968	ops);
		969	}
		970	EXPORT_SYMBOL_GPL(mmu_interval_notifier_insert_locked);
		971
		972	/**
		973	* mmu_interval_notifier_remove - Remove a interval notifier
		974	* @mni: Interval notifier to unregister
		975	*
		976	* This function must be paired with mmu_interval_notifier_insert(). It cannot
		977	* be called from any ops callback.
		978	*
		979	* Once this returns ops callbacks are no longer running on other CPUs and
		980	* will not be called in future.
		981	*/
		982	void mmu_interval_notifier_remove(struct mmu_interval_notifier *mni)
		983	{
		984	struct mm_struct *mm = mni->mm;
		985	struct mmu_notifier_mm *mmn_mm = mm->mmu_notifier_mm;
		986	unsigned long seq = 0;
		987
		988	might_sleep();
		989
		990	spin_lock(&mmn_mm->lock);
		991	if (mn_itree_is_invalidating(mmn_mm)) {
		992	/*
		993	* remove is being called after insert put this on the
		994	* deferred list, but before the deferred list was processed.
		995	*/
		996	if (RB_EMPTY_NODE(&mni->interval_tree.rb)) {
		997	hlist_del(&mni->deferred_item);
		998	} else {
		999	hlist_add_head(&mni->deferred_item,
		1000	&mmn_mm->deferred_list);
		1001	seq = mmn_mm->invalidate_seq;
		1002	}
		1003	} else {
		1004	WARN_ON(RB_EMPTY_NODE(&mni->interval_tree.rb));
		1005	interval_tree_remove(&mni->interval_tree, &mmn_mm->itree);
		1006	}
		1007	spin_unlock(&mmn_mm->lock);
		1008
		1009	/*
		1010	* The possible sleep on progress in the invalidation requires the
		1011	* caller not hold any locks held by invalidation callbacks.
		1012	*/
		1013	lock_map_acquire(&__mmu_notifier_invalidate_range_start_map);
		1014	lock_map_release(&__mmu_notifier_invalidate_range_start_map);
		1015	if (seq)
		1016	wait_event(mmn_mm->wq,
		1017	READ_ONCE(mmn_mm->invalidate_seq) != seq);
		1018
		1019	/* pairs with mmgrab in mmu_interval_notifier_insert() */
		1020	mmdrop(mm);
		1021	}
		1022	EXPORT_SYMBOL_GPL(mmu_interval_notifier_remove);
		1023
532	/**	1024	/**
533	* mmu_notifier_synchronize - Ensure all mmu_notifiers are freed	1025	* mmu_notifier_synchronize - Ensure all mmu_notifiers are freed
534	*	1026	*

mm/mmu_notifier: add an interval tree notifier

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code