mm/mmu_notifier: add an interval tree notifier

Of the 13 users of mmu_notifiers, 8 of them use only invalidate_range_start/end() and immediately intersect the mmu_notifier_range with some kind of internal list of VAs. 4 use an interval tree (i915_gem, radeon_mn, umem_odp, hfi1). 4 use a linked list of some kind (scif_dma, vhost, gntdev, hmm) And the remaining 5 either don't use invalidate_range_start() or do some special thing with it. It turns out that building a correct scheme with an interval tree is pretty complicated, particularly if the use case is synchronizing against another thread doing get_user_pages(). Many of these implementations have various subtle and difficult to fix races. This approach puts the interval tree as common code at the top of the mmu notifier call tree and implements a shareable locking scheme. It includes: - An interval tree tracking VA ranges, with per-range callbacks - A read/write locking scheme for the interval tree that avoids sleeping in the notifier path (for OOM killer) - A sequence counter based collision-retry locking scheme to tell device page fault that a VA range is being concurrently invalidated. This is based on various ideas: - hmm accumulates invalidated VA ranges and releases them when all invalidates are done, via active_invalidate_ranges count. This approach avoids having to intersect the interval tree twice (as umem_odp does) at the potential cost of a longer device page fault. - kvm/umem_odp use a sequence counter to drive the collision retry, via invalidate_seq - a deferred work todo list on unlock scheme like RTNL, via deferred_list. This makes adding/removing interval tree members more deterministic - seqlock, except this version makes the seqlock idea multi-holder on the write side by protecting it with active_invalidate_ranges and a spinlock To minimize MM overhead when only the interval tree is being used, the entire SRCU and hlist overheads are dropped using some simple branches. Similarly the interval tree overhead is dropped when in hlist mode. The overhead from the mandatory spinlock is broadly the same as most of existing users which already had a lock (or two) of some sort on the invalidation path.

Jason Gunthorpe committed 7 years ago

Commit 99cb2521 parent 56f434f3 files · +620 −26Browse files

Unified · Split

3 files

include/linux/mmu_notifier.h +101 −0
mm/Kconfig +1 −0
mm/mmu_notifier.c +518 −26

include/linux/mmu_notifier.h+101−0

@@ -6,10 +6,12 @@ · 5 unmodified lines
6	6	#include <linux/spinlock.h>
7	7	#include <linux/mm_types.h>
8	8	#include <linux/srcu.h>
	9	+#include <linux/interval_tree.h>
9	10
10	11	struct mmu_notifier_mm;
11	12	struct mmu_notifier;
12	13	struct mmu_notifier_range;
	14	+struct mmu_interval_notifier;
13	15
14	16	/**
15	17	* enum mmu_notifier_event - reason for the mmu notifier callback
@@ -32,6 +34,9 @@ struct mmu_notifier_range; · 16 unmodified lines
32	34	* access flags). User should soft dirty the page in the end callback to make
33	35	* sure that anyone relying on soft dirtyness catch pages that might be written
34	36	* through non CPU mappings.
	37	+ *
	38	+ * @MMU_NOTIFY_RELEASE: used during mmu_interval_notifier invalidate to signal
	39	+ * that the mm refcount is zero and the range is no longer accessible.
35	40	*/
36	41	enum mmu_notifier_event {
37	42	MMU_NOTIFY_UNMAP = 0,
@@ -39,6 +44,7 @@ enum mmu_notifier_event { · 1 unmodified line
39	44	MMU_NOTIFY_PROTECTION_VMA,
40	45	MMU_NOTIFY_PROTECTION_PAGE,
41	46	MMU_NOTIFY_SOFT_DIRTY,
	47	+ MMU_NOTIFY_RELEASE,
42	48	};
43	49
44	50	#define MMU_NOTIFIER_RANGE_BLOCKABLE (1 << 0)
@@ -222,6 +228,26 @@ struct mmu_notifier { · 177 unmodified lines
222	228	unsigned int users;
223	229	};
224	230
	231	+/**
	232	+ * struct mmu_interval_notifier_ops
	233	+ * @invalidate: Upon return the caller must stop using any SPTEs within this
	234	+ * range. This function can sleep. Return false only if sleeping
	235	+ * was required but mmu_notifier_range_blockable(range) is false.
	236	+ */
	237	+struct mmu_interval_notifier_ops {
	238	+ bool (invalidate)(struct mmu_interval_notifier mni,
	239	+ const struct mmu_notifier_range *range,
	240	+ unsigned long cur_seq);
	241	+};
	242	+
	243	+struct mmu_interval_notifier {
	244	+ struct interval_tree_node interval_tree;
	245	+ const struct mmu_interval_notifier_ops *ops;
	246	+ struct mm_struct *mm;
	247	+ struct hlist_node deferred_item;
	248	+ unsigned long invalidate_seq;
	249	+};
	250	+
225	251	#ifdef CONFIG_MMU_NOTIFIER
226	252
227	253	#ifdef CONFIG_LOCKDEP
@@ -263,6 +289,81 @@ extern int __mmu_notifier_register(struct mmu_notifier *mn, · 35 unmodified lines
263	289	struct mm_struct *mm);
264	290	extern void mmu_notifier_unregister(struct mmu_notifier *mn,
265	291	struct mm_struct *mm);
	292	+
	293	+unsigned long mmu_interval_read_begin(struct mmu_interval_notifier *mni);
	294	+int mmu_interval_notifier_insert(struct mmu_interval_notifier *mni,
	295	+ struct mm_struct *mm, unsigned long start,
	296	+ unsigned long length,
	297	+ const struct mmu_interval_notifier_ops *ops);
	298	+int mmu_interval_notifier_insert_locked(
	299	+ struct mmu_interval_notifier mni, struct mm_struct mm,
	300	+ unsigned long start, unsigned long length,
	301	+ const struct mmu_interval_notifier_ops *ops);
	302	+void mmu_interval_notifier_remove(struct mmu_interval_notifier *mni);
	303	+
	304	+/**
	305	+ * mmu_interval_set_seq - Save the invalidation sequence
	306	+ * @mni - The mni passed to invalidate
	307	+ * @cur_seq - The cur_seq passed to the invalidate() callback
	308	+ *
	309	+ * This must be called unconditionally from the invalidate callback of a
	310	+ * struct mmu_interval_notifier_ops under the same lock that is used to call
	311	+ * mmu_interval_read_retry(). It updates the sequence number for later use by
	312	+ * mmu_interval_read_retry(). The provided cur_seq will always be odd.
	313	+ *
	314	+ * If the caller does not call mmu_interval_read_begin() or
	315	+ * mmu_interval_read_retry() then this call is not required.
	316	+ */
	317	+static inline void mmu_interval_set_seq(struct mmu_interval_notifier *mni,
	318	+ unsigned long cur_seq)
	319	+{
	320	+ WRITE_ONCE(mni->invalidate_seq, cur_seq);
	321	+}
	322	+
	323	+/**
	324	+ * mmu_interval_read_retry - End a read side critical section against a VA range
	325	+ * mni: The range
	326	+ * seq: The return of the paired mmu_interval_read_begin()
	327	+ *
	328	+ * This MUST be called under a user provided lock that is also held
	329	+ * unconditionally by op->invalidate() when it calls mmu_interval_set_seq().
	330	+ *
	331	+ * Each call should be paired with a single mmu_interval_read_begin() and
	332	+ * should be used to conclude the read side.
	333	+ *
	334	+ * Returns true if an invalidation collided with this critical section, and
	335	+ * the caller should retry.
	336	+ */
	337	+static inline bool mmu_interval_read_retry(struct mmu_interval_notifier *mni,
	338	+ unsigned long seq)
	339	+{
	340	+ return mni->invalidate_seq != seq;
	341	+}
	342	+
	343	+/**
	344	+ * mmu_interval_check_retry - Test if a collision has occurred
	345	+ * mni: The range
	346	+ * seq: The return of the matching mmu_interval_read_begin()
	347	+ *
	348	+ * This can be used in the critical section between mmu_interval_read_begin()
	349	+ * and mmu_interval_read_retry(). A return of true indicates an invalidation
	350	+ * has collided with this critical region and a future
	351	+ * mmu_interval_read_retry() will return true.
	352	+ *
	353	+ * False is not reliable and only suggests a collision may not have
	354	+ * occured. It can be called many times and does not have to hold the user
	355	+ * provided lock.
	356	+ *
	357	+ * This call can be used as part of loops and other expensive operations to
	358	+ * expedite a retry.
	359	+ */
	360	+static inline bool mmu_interval_check_retry(struct mmu_interval_notifier *mni,
	361	+ unsigned long seq)
	362	+{
	363	+ /* Pairs with the WRITE_ONCE in mmu_interval_set_seq() */
	364	+ return READ_ONCE(mni->invalidate_seq) != seq;
	365	+}
	366	+
266	367	extern void __mmu_notifier_mm_destroy(struct mm_struct *mm);
267	368	extern void __mmu_notifier_release(struct mm_struct *mm);
268	369	extern int __mmu_notifier_clear_flush_young(struct mm_struct *mm,

mm/Kconfig+1−0

@@ -284,6 +284,7 @@ config VIRT_TO_BUS · 283 unmodified lines
284	284	config MMU_NOTIFIER
285	285	bool
286	286	select SRCU
	287	+ select INTERVAL_TREE
287	288
288	289	config KSM
289	290	bool "Enable KSM for page merging"

mm/mmu_notifier.c+518−26

@@ -12,6 +12,7 @@ · 11 unmodified lines
12	12	#include <linux/export.h>
13	13	#include <linux/mm.h>
14	14	#include <linux/err.h>
	15	+#include <linux/interval_tree.h>
15	16	#include <linux/srcu.h>
16	17	#include <linux/rcupdate.h>
17	18	#include <linux/sched.h>
@@ -36,10 +37,245 @@ struct lockdep_map __mmu_notifier_invalidate_range_start_map = { · 18 unmodified lines
36	37	struct mmu_notifier_mm {
37	38	/* all mmu notifiers registered in this mm are queued in this list */
38	39	struct hlist_head list;
	40	+ bool has_itree;
39	41	/* to serialize the list modifications and hlist_unhashed */
40	42	spinlock_t lock;
	43	+ unsigned long invalidate_seq;
	44	+ unsigned long active_invalidate_ranges;
	45	+ struct rb_root_cached itree;
	46	+ wait_queue_head_t wq;
	47	+ struct hlist_head deferred_list;
41	48	};
42	49
	50	+/*
	51	+ * This is a collision-retry read-side/write-side 'lock', a lot like a
	52	+ * seqcount, however this allows multiple write-sides to hold it at
	53	+ * once. Conceptually the write side is protecting the values of the PTEs in
	54	+ * this mm, such that PTES cannot be read into SPTEs (shadow PTEs) while any
	55	+ * writer exists.
	56	+ *
	57	+ * Note that the core mm creates nested invalidate_range_start()/end() regions
	58	+ * within the same thread, and runs invalidate_range_start()/end() in parallel
	59	+ * on multiple CPUs. This is designed to not reduce concurrency or block
	60	+ * progress on the mm side.
	61	+ *
	62	+ * As a secondary function, holding the full write side also serves to prevent
	63	+ * writers for the itree, this is an optimization to avoid extra locking
	64	+ * during invalidate_range_start/end notifiers.
	65	+ *
	66	+ * The write side has two states, fully excluded:
	67	+ * - mm->active_invalidate_ranges != 0
	68	+ * - mnn->invalidate_seq & 1 == True (odd)
	69	+ * - some range on the mm_struct is being invalidated
	70	+ * - the itree is not allowed to change
	71	+ *
	72	+ * And partially excluded:
	73	+ * - mm->active_invalidate_ranges != 0
	74	+ * - mnn->invalidate_seq & 1 == False (even)
	75	+ * - some range on the mm_struct is being invalidated
	76	+ * - the itree is allowed to change
	77	+ *
	78	+ * Operations on mmu_notifier_mm->invalidate_seq (under spinlock):
	79	+ * seq \|= 1 # Begin writing
	80	+ * seq++ # Release the writing state
	81	+ * seq & 1 # True if a writer exists
	82	+ *
	83	+ * The later state avoids some expensive work on inv_end in the common case of
	84	+ * no mni monitoring the VA.
	85	+ */
	86	+static bool mn_itree_is_invalidating(struct mmu_notifier_mm *mmn_mm)
	87	+{
	88	+ lockdep_assert_held(&mmn_mm->lock);
	89	+ return mmn_mm->invalidate_seq & 1;
	90	+}
	91	+
	92	+static struct mmu_interval_notifier *
	93	+mn_itree_inv_start_range(struct mmu_notifier_mm *mmn_mm,
	94	+ const struct mmu_notifier_range *range,
	95	+ unsigned long *seq)
	96	+{
	97	+ struct interval_tree_node *node;
	98	+ struct mmu_interval_notifier *res = NULL;
	99	+
	100	+ spin_lock(&mmn_mm->lock);
	101	+ mmn_mm->active_invalidate_ranges++;
	102	+ node = interval_tree_iter_first(&mmn_mm->itree, range->start,
	103	+ range->end - 1);
	104	+ if (node) {
	105	+ mmn_mm->invalidate_seq \|= 1;
	106	+ res = container_of(node, struct mmu_interval_notifier,
	107	+ interval_tree);
	108	+ }
	109	+
	110	+ *seq = mmn_mm->invalidate_seq;
	111	+ spin_unlock(&mmn_mm->lock);
	112	+ return res;
	113	+}
	114	+
	115	+static struct mmu_interval_notifier *
	116	+mn_itree_inv_next(struct mmu_interval_notifier *mni,
	117	+ const struct mmu_notifier_range *range)
	118	+{
	119	+ struct interval_tree_node *node;
	120	+
	121	+ node = interval_tree_iter_next(&mni->interval_tree, range->start,
	122	+ range->end - 1);
	123	+ if (!node)
	124	+ return NULL;
	125	+ return container_of(node, struct mmu_interval_notifier, interval_tree);
	126	+}
	127	+
	128	+static void mn_itree_inv_end(struct mmu_notifier_mm *mmn_mm)
	129	+{
	130	+ struct mmu_interval_notifier *mni;
	131	+ struct hlist_node *next;
	132	+
	133	+ spin_lock(&mmn_mm->lock);
	134	+ if (--mmn_mm->active_invalidate_ranges \|\|
	135	+ !mn_itree_is_invalidating(mmn_mm)) {
	136	+ spin_unlock(&mmn_mm->lock);
	137	+ return;
	138	+ }
	139	+
	140	+ /* Make invalidate_seq even */
	141	+ mmn_mm->invalidate_seq++;
	142	+
	143	+ /*
	144	+ * The inv_end incorporates a deferred mechanism like rtnl_unlock().
	145	+ * Adds and removes are queued until the final inv_end happens then
	146	+ * they are progressed. This arrangement for tree updates is used to
	147	+ * avoid using a blocking lock during invalidate_range_start.
	148	+ */
	149	+ hlist_for_each_entry_safe(mni, next, &mmn_mm->deferred_list,
	150	+ deferred_item) {
	151	+ if (RB_EMPTY_NODE(&mni->interval_tree.rb))
	152	+ interval_tree_insert(&mni->interval_tree,
	153	+ &mmn_mm->itree);
	154	+ else
	155	+ interval_tree_remove(&mni->interval_tree,
	156	+ &mmn_mm->itree);
	157	+ hlist_del(&mni->deferred_item);
	158	+ }
	159	+ spin_unlock(&mmn_mm->lock);
	160	+
	161	+ wake_up_all(&mmn_mm->wq);
	162	+}
	163	+
	164	+/**
	165	+ * mmu_interval_read_begin - Begin a read side critical section against a VA
	166	+ * range
	167	+ * mni: The range to use
	168	+ *
	169	+ * mmu_iterval_read_begin()/mmu_iterval_read_retry() implement a
	170	+ * collision-retry scheme similar to seqcount for the VA range under mni. If
	171	+ * the mm invokes invalidation during the critical section then
	172	+ * mmu_interval_read_retry() will return true.
	173	+ *
	174	+ * This is useful to obtain shadow PTEs where teardown or setup of the SPTEs
	175	+ * require a blocking context. The critical region formed by this can sleep,
	176	+ * and the required 'user_lock' can also be a sleeping lock.
	177	+ *
	178	+ * The caller is required to provide a 'user_lock' to serialize both teardown
	179	+ * and setup.
	180	+ *
	181	+ * The return value should be passed to mmu_interval_read_retry().
	182	+ */
	183	+unsigned long mmu_interval_read_begin(struct mmu_interval_notifier *mni)
	184	+{
	185	+ struct mmu_notifier_mm *mmn_mm = mni->mm->mmu_notifier_mm;
	186	+ unsigned long seq;
	187	+ bool is_invalidating;
	188	+
	189	+ /*
	190	+ * If the mni has a different seq value under the user_lock than we
	191	+ * started with then it has collided.
	192	+ *
	193	+ * If the mni currently has the same seq value as the mmn_mm seq, then
	194	+ * it is currently between invalidate_start/end and is colliding.
	195	+ *
	196	+ * The locking looks broadly like this:
	197	+ * mn_tree_invalidate_start(): mmu_interval_read_begin():
	198	+ * spin_lock
	199	+ * seq = READ_ONCE(mni->invalidate_seq);
	200	+ * seq == mmn_mm->invalidate_seq
	201	+ * spin_unlock
	202	+ * spin_lock
	203	+ * seq = ++mmn_mm->invalidate_seq
	204	+ * spin_unlock
	205	+ * op->invalidate_range():
	206	+ * user_lock
	207	+ * mmu_interval_set_seq()
	208	+ * mni->invalidate_seq = seq
	209	+ * user_unlock
	210	+ *
	211	+ * [Required: mmu_interval_read_retry() == true]
	212	+ *
	213	+ * mn_itree_inv_end():
	214	+ * spin_lock
	215	+ * seq = ++mmn_mm->invalidate_seq
	216	+ * spin_unlock
	217	+ *
	218	+ * user_lock
	219	+ * mmu_interval_read_retry():
	220	+ * mni->invalidate_seq != seq
	221	+ * user_unlock
	222	+ *
	223	+ * Barriers are not needed here as any races here are closed by an
	224	+ * eventual mmu_interval_read_retry(), which provides a barrier via the
	225	+ * user_lock.
	226	+ */
	227	+ spin_lock(&mmn_mm->lock);
	228	+ /* Pairs with the WRITE_ONCE in mmu_interval_set_seq() */
	229	+ seq = READ_ONCE(mni->invalidate_seq);
	230	+ is_invalidating = seq == mmn_mm->invalidate_seq;
	231	+ spin_unlock(&mmn_mm->lock);
	232	+
	233	+ /*
	234	+ * mni->invalidate_seq must always be set to an odd value via
	235	+ * mmu_interval_set_seq() using the provided cur_seq from
	236	+ * mn_itree_inv_start_range(). This ensures that if seq does wrap we
	237	+ * will always clear the below sleep in some reasonable time as
	238	+ * mmn_mm->invalidate_seq is even in the idle state.
	239	+ */
	240	+ lock_map_acquire(&__mmu_notifier_invalidate_range_start_map);
	241	+ lock_map_release(&__mmu_notifier_invalidate_range_start_map);
	242	+ if (is_invalidating)
	243	+ wait_event(mmn_mm->wq,
	244	+ READ_ONCE(mmn_mm->invalidate_seq) != seq);
	245	+
	246	+ /*
	247	+ * Notice that mmu_interval_read_retry() can already be true at this
	248	+ * point, avoiding loops here allows the caller to provide a global
	249	+ * time bound.
	250	+ */
	251	+
	252	+ return seq;
	253	+}
	254	+EXPORT_SYMBOL_GPL(mmu_interval_read_begin);
	255	+
	256	+static void mn_itree_release(struct mmu_notifier_mm *mmn_mm,
	257	+ struct mm_struct *mm)
	258	+{
	259	+ struct mmu_notifier_range range = {
	260	+ .flags = MMU_NOTIFIER_RANGE_BLOCKABLE,
	261	+ .event = MMU_NOTIFY_RELEASE,
	262	+ .mm = mm,
	263	+ .start = 0,
	264	+ .end = ULONG_MAX,
	265	+ };
	266	+ struct mmu_interval_notifier *mni;
	267	+ unsigned long cur_seq;
	268	+ bool ret;
	269	+
	270	+ for (mni = mn_itree_inv_start_range(mmn_mm, &range, &cur_seq); mni;
	271	+ mni = mn_itree_inv_next(mni, &range)) {
	272	+ ret = mni->ops->invalidate(mni, &range, cur_seq);
	273	+ WARN_ON(!ret);
	274	+ }
	275	+
	276	+ mn_itree_inv_end(mmn_mm);
	277	+}
	278	+
43	279	/*
44	280	* This function can't run concurrently against mmu_notifier_register
45	281	* because mm->mm_users > 0 during mmu_notifier_register and exit_mmap
@@ -52,7 +288,8 @@ struct mmu_notifier_mm { · 6 unmodified lines
52	288	* can't go away from under us as exit_mmap holds an mm_count pin
53	289	* itself.
54	290	*/
55		−void __mmu_notifier_release(struct mm_struct *mm)
	291	+static void mn_hlist_release(struct mmu_notifier_mm *mmn_mm,
	292	+ struct mm_struct *mm)
56	293	{
57	294	struct mmu_notifier *mn;
58	295	int id;
@@ -62,7 +299,7 @@ void __mmu_notifier_release(struct mm_struct *mm) · 3 unmodified lines
62	299	* ->release returns.
63	300	*/
64	301	id = srcu_read_lock(&srcu);
65		− hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist)
	302	+ hlist_for_each_entry_rcu(mn, &mmn_mm->list, hlist)
66	303	/*
67	304	* If ->release runs before mmu_notifier_unregister it must be
68	305	* handled, as it's the only way for the driver to flush all
@@ -72,10 +309,9 @@ void __mmu_notifier_release(struct mm_struct *mm) · 3 unmodified lines
72	309	if (mn->ops->release)
73	310	mn->ops->release(mn, mm);
74	311
75		− spin_lock(&mm->mmu_notifier_mm->lock);
76		− while (unlikely(!hlist_empty(&mm->mmu_notifier_mm->list))) {
77		− mn = hlist_entry(mm->mmu_notifier_mm->list.first,
78		− struct mmu_notifier,
	312	+ spin_lock(&mmn_mm->lock);
	313	+ while (unlikely(!hlist_empty(&mmn_mm->list))) {
	314	+ mn = hlist_entry(mmn_mm->list.first, struct mmu_notifier,
79	315	hlist);
80	316	/*
81	317	* We arrived before mmu_notifier_unregister so
@@ -85,7 +321,7 @@ void __mmu_notifier_release(struct mm_struct *mm) · 3 unmodified lines
85	321	*/
86	322	hlist_del_init_rcu(&mn->hlist);
87	323	}
88		− spin_unlock(&mm->mmu_notifier_mm->lock);
	324	+ spin_unlock(&mmn_mm->lock);
89	325	srcu_read_unlock(&srcu, id);
90	326
91	327	/*
@@ -100,6 +336,17 @@ void __mmu_notifier_release(struct mm_struct *mm) · 8 unmodified lines
100	336	synchronize_srcu(&srcu);
101	337	}
102	338
	339	+void __mmu_notifier_release(struct mm_struct *mm)
	340	+{
	341	+ struct mmu_notifier_mm *mmn_mm = mm->mmu_notifier_mm;
	342	+
	343	+ if (mmn_mm->has_itree)
	344	+ mn_itree_release(mmn_mm, mm);
	345	+
	346	+ if (!hlist_empty(&mmn_mm->list))
	347	+ mn_hlist_release(mmn_mm, mm);
	348	+}
	349	+
103	350	/*
104	351	* If no young bitflag is supported by the hardware, ->clear_flush_young can
105	352	* unmap the address and return 1 or 0 depending if the mapping previously
@@ -172,14 +419,43 @@ void __mmu_notifier_change_pte(struct mm_struct *mm, unsigned long address, · 66 unmodified lines
172	419	srcu_read_unlock(&srcu, id);
173	420	}
174	421
175		−int __mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range)
	422	+static int mn_itree_invalidate(struct mmu_notifier_mm *mmn_mm,
	423	+ const struct mmu_notifier_range *range)
	424	+{
	425	+ struct mmu_interval_notifier *mni;
	426	+ unsigned long cur_seq;
	427	+
	428	+ for (mni = mn_itree_inv_start_range(mmn_mm, range, &cur_seq); mni;
	429	+ mni = mn_itree_inv_next(mni, range)) {
	430	+ bool ret;
	431	+
	432	+ ret = mni->ops->invalidate(mni, range, cur_seq);
	433	+ if (!ret) {
	434	+ if (WARN_ON(mmu_notifier_range_blockable(range)))
	435	+ continue;
	436	+ goto out_would_block;
	437	+ }
	438	+ }
	439	+ return 0;
	440	+
	441	+out_would_block:
	442	+ /*
	443	+ * On -EAGAIN the non-blocking caller is not allowed to call
	444	+ * invalidate_range_end()
	445	+ */
	446	+ mn_itree_inv_end(mmn_mm);
	447	+ return -EAGAIN;
	448	+}
	449	+
	450	+static int mn_hlist_invalidate_range_start(struct mmu_notifier_mm *mmn_mm,
	451	+ struct mmu_notifier_range *range)
176	452	{
177	453	struct mmu_notifier *mn;
178	454	int ret = 0;
179	455	int id;
180	456
181	457	id = srcu_read_lock(&srcu);
182		− hlist_for_each_entry_rcu(mn, &range->mm->mmu_notifier_mm->list, hlist) {
	458	+ hlist_for_each_entry_rcu(mn, &mmn_mm->list, hlist) {
183	459	if (mn->ops->invalidate_range_start) {
184	460	int _ret;
185	461
@@ -203,15 +479,30 @@ int __mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range) · 17 unmodified lines
203	479	return ret;
204	480	}
205	481
206		−void __mmu_notifier_invalidate_range_end(struct mmu_notifier_range *range,
207		− bool only_end)
	482	+int __mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range)
	483	+{
	484	+ struct mmu_notifier_mm *mmn_mm = range->mm->mmu_notifier_mm;
	485	+ int ret;
	486	+
	487	+ if (mmn_mm->has_itree) {
	488	+ ret = mn_itree_invalidate(mmn_mm, range);
	489	+ if (ret)
	490	+ return ret;
	491	+ }
	492	+ if (!hlist_empty(&mmn_mm->list))
	493	+ return mn_hlist_invalidate_range_start(mmn_mm, range);
	494	+ return 0;
	495	+}
	496	+
	497	+static void mn_hlist_invalidate_end(struct mmu_notifier_mm *mmn_mm,
	498	+ struct mmu_notifier_range *range,
	499	+ bool only_end)
208	500	{
209	501	struct mmu_notifier *mn;
210	502	int id;
211	503
212		− lock_map_acquire(&__mmu_notifier_invalidate_range_start_map);
213	504	id = srcu_read_lock(&srcu);
214		− hlist_for_each_entry_rcu(mn, &range->mm->mmu_notifier_mm->list, hlist) {
	505	+ hlist_for_each_entry_rcu(mn, &mmn_mm->list, hlist) {
215	506	/*
216	507	* Call invalidate_range here too to avoid the need for the
217	508	* subsystem of having to register an invalidate_range_end
@@ -238,6 +529,19 @@ void __mmu_notifier_invalidate_range_end(struct mmu_notifier_range *range, · 20 unmodified lines
238	529	}
239	530	}
240	531	srcu_read_unlock(&srcu, id);
	532	+}
	533	+
	534	+void __mmu_notifier_invalidate_range_end(struct mmu_notifier_range *range,
	535	+ bool only_end)
	536	+{
	537	+ struct mmu_notifier_mm *mmn_mm = range->mm->mmu_notifier_mm;
	538	+
	539	+ lock_map_acquire(&__mmu_notifier_invalidate_range_start_map);
	540	+ if (mmn_mm->has_itree)
	541	+ mn_itree_inv_end(mmn_mm);
	542	+
	543	+ if (!hlist_empty(&mmn_mm->list))
	544	+ mn_hlist_invalidate_end(mmn_mm, range, only_end);
241	545	lock_map_release(&__mmu_notifier_invalidate_range_start_map);
242	546	}
243	547
@@ -256,8 +560,9 @@ void __mmu_notifier_invalidate_range(struct mm_struct *mm, · 12 unmodified lines
256	560	}
257	561
258	562	/*
259		− * Same as mmu_notifier_register but here the caller must hold the
260		− * mmap_sem in write mode.
	563	+ * Same as mmu_notifier_register but here the caller must hold the mmap_sem in
	564	+ * write mode. A NULL mn signals the notifier is being registered for itree
	565	+ * mode.
261	566	*/
262	567	int __mmu_notifier_register(struct mmu_notifier mn, struct mm_struct mm)
263	568	{
@@ -274,9 +579,6 @@ int __mmu_notifier_register(struct mmu_notifier mn, struct mm_struct mm) · 10 unmodified lines
274	579	fs_reclaim_release(GFP_KERNEL);
275	580	}
276	581
277		− mn->mm = mm;
278		− mn->users = 1;
279		−
280	582	if (!mm->mmu_notifier_mm) {
281	583	/*
282	584	* kmalloc cannot be called under mm_take_all_locks(), but we
@@ -284,21 +586,22 @@ int __mmu_notifier_register(struct mmu_notifier mn, struct mm_struct mm) · 1 unmodified line
284	586	* the write side of the mmap_sem.
285	587	*/
286	588	mmu_notifier_mm =
287		− kmalloc(sizeof(struct mmu_notifier_mm), GFP_KERNEL);
	589	+ kzalloc(sizeof(struct mmu_notifier_mm), GFP_KERNEL);
288	590	if (!mmu_notifier_mm)
289	591	return -ENOMEM;
290	592
291	593	INIT_HLIST_HEAD(&mmu_notifier_mm->list);
292	594	spin_lock_init(&mmu_notifier_mm->lock);
	595	+ mmu_notifier_mm->invalidate_seq = 2;
	596	+ mmu_notifier_mm->itree = RB_ROOT_CACHED;
	597	+ init_waitqueue_head(&mmu_notifier_mm->wq);
	598	+ INIT_HLIST_HEAD(&mmu_notifier_mm->deferred_list);
293	599	}
294	600
295	601	ret = mm_take_all_locks(mm);
296	602	if (unlikely(ret))
297	603	goto out_clean;
298	604
299		− /* Pairs with the mmdrop in mmu_notifier_unregister_* */
300		− mmgrab(mm);
301		−
302	605	/*
303	606	* Serialize the update against mmu_notifier_unregister. A
304	607	* side note: mmu_notifier_release can't run concurrently with
@@ -306,13 +609,28 @@ int __mmu_notifier_register(struct mmu_notifier mn, struct mm_struct mm) · 1 unmodified line
306	609	* current->mm or explicitly with get_task_mm() or similar).
307	610	* We can't race against any other mmu notifier method either
308	611	* thanks to mm_take_all_locks().
	612	+ *
	613	+ * release semantics on the initialization of the mmu_notifier_mm's
	614	+ * contents are provided for unlocked readers. acquire can only be
	615	+ * used while holding the mmgrab or mmget, and is safe because once
	616	+ * created the mmu_notififer_mm is not freed until the mm is
	617	+ * destroyed. As above, users holding the mmap_sem or one of the
	618	+ * mm_take_all_locks() do not need to use acquire semantics.
309	619	*/
310	620	if (mmu_notifier_mm)
311		− mm->mmu_notifier_mm = mmu_notifier_mm;
	621	+ smp_store_release(&mm->mmu_notifier_mm, mmu_notifier_mm);
312	622
313		− spin_lock(&mm->mmu_notifier_mm->lock);
314		− hlist_add_head_rcu(&mn->hlist, &mm->mmu_notifier_mm->list);
315		− spin_unlock(&mm->mmu_notifier_mm->lock);
	623	+ if (mn) {
	624	+ /* Pairs with the mmdrop in mmu_notifier_unregister_* */
	625	+ mmgrab(mm);
	626	+ mn->mm = mm;
	627	+ mn->users = 1;
	628	+
	629	+ spin_lock(&mm->mmu_notifier_mm->lock);
	630	+ hlist_add_head_rcu(&mn->hlist, &mm->mmu_notifier_mm->list);
	631	+ spin_unlock(&mm->mmu_notifier_mm->lock);
	632	+ } else
	633	+ mm->mmu_notifier_mm->has_itree = true;
316	634
317	635	mm_drop_all_locks(mm);
318	636	BUG_ON(atomic_read(&mm->mm_users) <= 0);
@@ -529,6 +847,180 @@ out_unlock: · 210 unmodified lines
529	847	}
530	848	EXPORT_SYMBOL_GPL(mmu_notifier_put);
531	849
	850	+static int __mmu_interval_notifier_insert(
	851	+ struct mmu_interval_notifier mni, struct mm_struct mm,
	852	+ struct mmu_notifier_mm *mmn_mm, unsigned long start,
	853	+ unsigned long length, const struct mmu_interval_notifier_ops *ops)
	854	+{
	855	+ mni->mm = mm;
	856	+ mni->ops = ops;
	857	+ RB_CLEAR_NODE(&mni->interval_tree.rb);
	858	+ mni->interval_tree.start = start;
	859	+ /*
	860	+ * Note that the representation of the intervals in the interval tree
	861	+ * considers the ending point as contained in the interval.
	862	+ */
	863	+ if (length == 0 \|\|
	864	+ check_add_overflow(start, length - 1, &mni->interval_tree.last))
	865	+ return -EOVERFLOW;
	866	+
	867	+ /* Must call with a mmget() held */
	868	+ if (WARN_ON(atomic_read(&mm->mm_count) <= 0))
	869	+ return -EINVAL;
	870	+
	871	+ /* pairs with mmdrop in mmu_interval_notifier_remove() */
	872	+ mmgrab(mm);
	873	+
	874	+ /*
	875	+ * If some invalidate_range_start/end region is going on in parallel
	876	+ * we don't know what VA ranges are affected, so we must assume this
	877	+ * new range is included.
	878	+ *
	879	+ * If the itree is invalidating then we are not allowed to change
	880	+ * it. Retrying until invalidation is done is tricky due to the
	881	+ * possibility for live lock, instead defer the add to
	882	+ * mn_itree_inv_end() so this algorithm is deterministic.
	883	+ *
	884	+ * In all cases the value for the mni->invalidate_seq should be
	885	+ * odd, see mmu_interval_read_begin()
	886	+ */
	887	+ spin_lock(&mmn_mm->lock);
	888	+ if (mmn_mm->active_invalidate_ranges) {
	889	+ if (mn_itree_is_invalidating(mmn_mm))
	890	+ hlist_add_head(&mni->deferred_item,
	891	+ &mmn_mm->deferred_list);
	892	+ else {
	893	+ mmn_mm->invalidate_seq \|= 1;
	894	+ interval_tree_insert(&mni->interval_tree,
	895	+ &mmn_mm->itree);
	896	+ }
	897	+ mni->invalidate_seq = mmn_mm->invalidate_seq;
	898	+ } else {
	899	+ WARN_ON(mn_itree_is_invalidating(mmn_mm));
	900	+ /*
	901	+ * The starting seq for a mni not under invalidation should be
	902	+ * odd, not equal to the current invalidate_seq and
	903	+ * invalidate_seq should not 'wrap' to the new seq any time
	904	+ * soon.
	905	+ */
	906	+ mni->invalidate_seq = mmn_mm->invalidate_seq - 1;
	907	+ interval_tree_insert(&mni->interval_tree, &mmn_mm->itree);
	908	+ }
	909	+ spin_unlock(&mmn_mm->lock);
	910	+ return 0;
	911	+}
	912	+
	913	+/**
	914	+ * mmu_interval_notifier_insert - Insert an interval notifier
	915	+ * @mni: Interval notifier to register
	916	+ * @start: Starting virtual address to monitor
	917	+ * @length: Length of the range to monitor
	918	+ * @mm : mm_struct to attach to
	919	+ *
	920	+ * This function subscribes the interval notifier for notifications from the
	921	+ * mm. Upon return the ops related to mmu_interval_notifier will be called
	922	+ * whenever an event that intersects with the given range occurs.
	923	+ *
	924	+ * Upon return the range_notifier may not be present in the interval tree yet.
	925	+ * The caller must use the normal interval notifier read flow via
	926	+ * mmu_interval_read_begin() to establish SPTEs for this range.
	927	+ */
	928	+int mmu_interval_notifier_insert(struct mmu_interval_notifier *mni,
	929	+ struct mm_struct *mm, unsigned long start,
	930	+ unsigned long length,
	931	+ const struct mmu_interval_notifier_ops *ops)
	932	+{
	933	+ struct mmu_notifier_mm *mmn_mm;
	934	+ int ret;
	935	+
	936	+ might_lock(&mm->mmap_sem);
	937	+
	938	+ mmn_mm = smp_load_acquire(&mm->mmu_notifier_mm);
	939	+ if (!mmn_mm \|\| !mmn_mm->has_itree) {
	940	+ ret = mmu_notifier_register(NULL, mm);
	941	+ if (ret)
	942	+ return ret;
	943	+ mmn_mm = mm->mmu_notifier_mm;
	944	+ }
	945	+ return __mmu_interval_notifier_insert(mni, mm, mmn_mm, start, length,
	946	+ ops);
	947	+}
	948	+EXPORT_SYMBOL_GPL(mmu_interval_notifier_insert);
	949	+
	950	+int mmu_interval_notifier_insert_locked(
	951	+ struct mmu_interval_notifier mni, struct mm_struct mm,
	952	+ unsigned long start, unsigned long length,
	953	+ const struct mmu_interval_notifier_ops *ops)
	954	+{
	955	+ struct mmu_notifier_mm *mmn_mm;
	956	+ int ret;
	957	+
	958	+ lockdep_assert_held_write(&mm->mmap_sem);
	959	+
	960	+ mmn_mm = mm->mmu_notifier_mm;
	961	+ if (!mmn_mm \|\| !mmn_mm->has_itree) {
	962	+ ret = __mmu_notifier_register(NULL, mm);
	963	+ if (ret)
	964	+ return ret;
	965	+ mmn_mm = mm->mmu_notifier_mm;
	966	+ }
	967	+ return __mmu_interval_notifier_insert(mni, mm, mmn_mm, start, length,
	968	+ ops);
	969	+}
	970	+EXPORT_SYMBOL_GPL(mmu_interval_notifier_insert_locked);
	971	+
	972	+/**
	973	+ * mmu_interval_notifier_remove - Remove a interval notifier
	974	+ * @mni: Interval notifier to unregister
	975	+ *
	976	+ * This function must be paired with mmu_interval_notifier_insert(). It cannot
	977	+ * be called from any ops callback.
	978	+ *
	979	+ * Once this returns ops callbacks are no longer running on other CPUs and
	980	+ * will not be called in future.
	981	+ */
	982	+void mmu_interval_notifier_remove(struct mmu_interval_notifier *mni)
	983	+{
	984	+ struct mm_struct *mm = mni->mm;
	985	+ struct mmu_notifier_mm *mmn_mm = mm->mmu_notifier_mm;
	986	+ unsigned long seq = 0;
	987	+
	988	+ might_sleep();
	989	+
	990	+ spin_lock(&mmn_mm->lock);
	991	+ if (mn_itree_is_invalidating(mmn_mm)) {
	992	+ /*
	993	+ * remove is being called after insert put this on the
	994	+ * deferred list, but before the deferred list was processed.
	995	+ */
	996	+ if (RB_EMPTY_NODE(&mni->interval_tree.rb)) {
	997	+ hlist_del(&mni->deferred_item);
	998	+ } else {
	999	+ hlist_add_head(&mni->deferred_item,
	1000	+ &mmn_mm->deferred_list);
	1001	+ seq = mmn_mm->invalidate_seq;
	1002	+ }
	1003	+ } else {
	1004	+ WARN_ON(RB_EMPTY_NODE(&mni->interval_tree.rb));
	1005	+ interval_tree_remove(&mni->interval_tree, &mmn_mm->itree);
	1006	+ }
	1007	+ spin_unlock(&mmn_mm->lock);
	1008	+
	1009	+ /*
	1010	+ * The possible sleep on progress in the invalidation requires the
	1011	+ * caller not hold any locks held by invalidation callbacks.
	1012	+ */
	1013	+ lock_map_acquire(&__mmu_notifier_invalidate_range_start_map);
	1014	+ lock_map_release(&__mmu_notifier_invalidate_range_start_map);
	1015	+ if (seq)
	1016	+ wait_event(mmn_mm->wq,
	1017	+ READ_ONCE(mmn_mm->invalidate_seq) != seq);
	1018	+
	1019	+ /* pairs with mmgrab in mmu_interval_notifier_insert() */
	1020	+ mmdrop(mm);
	1021	+}
	1022	+EXPORT_SYMBOL_GPL(mmu_interval_notifier_remove);
	1023	+
532	1024	/**
533	1025	* mmu_notifier_synchronize - Ensure all mmu_notifiers are freed
534	1026	*

mm/mmu_notifier: add an interval tree notifier

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code