mm: Introduce mm_struct.has_pinned

(Commit message majorly collected from Jason Gunthorpe) Reduce the chance of false positive from page_maybe_dma_pinned() by keeping track if the mm_struct has ever been used with pin_user_pages(). This allows cases that might drive up the page ref_count to avoid any penalty from handling dma_pinned pages. Future work is planned, to provide a more sophisticated solution, likely to turn it into a real counter. For now, make it atomic_t but use it as a boolean for simplicity.

Peter Xu committed 6 years ago

Commit 008cfe41 parent a1bffa43 files · +17 −0Browse files

Unified · Split

3 files

include/linux/mm_types.h +10 −0
kernel/fork.c +1 −0
mm/gup.c +6 −0

include/linux/mm_types.h+10−0

@@ -436,6 +436,16 @@ struct mm_struct { · 435 unmodified lines
436	436	*/
437	437	atomic_t mm_count;
438	438
	439	+ /**
	440	+ * @has_pinned: Whether this mm has pinned any pages. This can
	441	+ * be either replaced in the future by @pinned_vm when it
	442	+ * becomes stable, or grow into a counter on its own. We're
	443	+ * aggresive on this bit now - even if the pinned pages were
	444	+ * unpinned later on, we'll still keep this bit set for the
	445	+ * lifecycle of this mm just for simplicity.
	446	+ */
	447	+ atomic_t has_pinned;
	448	+
439	449	#ifdef CONFIG_MMU
440	450	atomic_long_t pgtables_bytes; /* PTE page table pages */
441	451	#endif

kernel/fork.c+1−0

@@ -1011,6 +1011,7 @@ static struct mm_struct mm_init(struct mm_struct mm, struct task_struct *p, · 1010 unmodified lines
1011	1011	mm_pgtables_bytes_init(mm);
1012	1012	mm->map_count = 0;
1013	1013	mm->locked_vm = 0;
	1014	+ atomic_set(&mm->has_pinned, 0);
1014	1015	atomic64_set(&mm->pinned_vm, 0);
1015	1016	memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));
1016	1017	spin_lock_init(&mm->page_table_lock);

mm/gup.c+6−0

@@ -1255,6 +1255,9 @@ static __always_inline long __get_user_pages_locked(struct mm_struct *mm, · 1254 unmodified lines
1255	1255	BUG_ON(*locked != 1);
1256	1256	}
1257	1257
	1258	+ if (flags & FOLL_PIN)
	1259	+ atomic_set(&current->mm->has_pinned, 1);
	1260	+
1258	1261	/*
1259	1262	* FOLL_PIN and FOLL_GET are mutually exclusive. Traditional behavior
1260	1263	* is to set FOLL_GET if the caller wants pages[] filled in (but has
@@ -2660,6 +2663,9 @@ static int internal_get_user_pages_fast(unsigned long start, int nr_pages, · 1399 unmodified lines
2660	2663	FOLL_FAST_ONLY)))
2661	2664	return -EINVAL;
2662	2665
	2666	+ if (gup_flags & FOLL_PIN)
	2667	+ atomic_set(&current->mm->has_pinned, 1);
	2668	+
2663	2669	if (!(gup_flags & FOLL_FAST_ONLY))
2664	2670	might_lock_read(&current->mm->mmap_lock);
2665	2671

mm: Introduce mm_struct.has_pinned

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code