userfaultfd: add UFFDIO_REGISTER_MODE_RWP and UFFDIO_RWPROTECT plumbing

Add the userspace interface for read-write protection tracking: - UFFDIO_REGISTER_MODE_RWP register a range for RWP tracking - UFFD_FEATURE_RWP capability bit - UFFDIO_RWPROTECT install / remove RWP on a range Introduce CONFIG_USERFAULTFD_RWP, auto-selected on 64-bit kernels with ARCH_HAS_PTE_PROTNONE and HAVE_ARCH_USERFAULTFD_WP. The symbol gates VM_UFFD_RWP (previously aliased to VM_NONE) and the smaps/trace-flag hooks added in the preparatory patches; without it the UAPI bits added here have nothing to drive and would be unreachable. Registration sets VM_UFFD_RWP on the VMA. Combining MODE_WP with MODE_RWP is rejected because both modes claim the uffd PTE bit. UFFDIO_RWPROTECT is the bidirectional counterpart of UFFDIO_WRITEPROTECT: - MODE_RWP change_protection() with MM_CP_UFFD_RWP installs PAGE_NONE and sets the uffd bit on present PTEs - !MODE_RWP change_protection() with MM_CP_UFFD_RWP_RESOLVE restores vma->vm_page_prot and clears the bit userfaultfd_clear_vma() runs the same resolve pass on unregister so RWP state cannot outlive the uffd. Re-registering a range must not drop a mode that installs per-PTE markers (WP or RWP); doing so returns -EBUSY. This also closes a pre-existing window where re-registering without MODE_WP would strand uffd-wp markers: before, those caused extra write-faults but were otherwise benign; with RWP preservation in place, a subsequent mprotect() on a VM_UFFD_RWP VMA would silently promote the stale markers to RWP. The feature is not yet advertised. UFFDIO_REGISTER_MODE_RWP, UFFD_FEATURE_RWP, and _UFFDIO_RWPROTECT are intentionally absent from UFFD_API_REGISTER_MODES, UFFD_API_FEATURES, and UFFD_API_RANGE_IOCTLS, so UFFDIO_API masks them out and the register-mode validator rejects the bit. The follow-up patch adds fault dispatch and exposes the UAPI.

Kiryl Shutsemau (Meta) committed 4 months ago

Commit 6eab8f21 parent 7974c235 files · +228 −3Browse files

Unified · Split

5 files

Documentation/admin-guide/mm/userfaultfd.rst +10 −0
include/linux/userfaultfd_k.h +2 −0
include/uapi/linux/userfaultfd.h +19 −0
mm/Kconfig +9 −0
mm/userfaultfd.c +188 −3

Documentation/admin-guide/mm/userfaultfd.rst+10−0

@@ -131,6 +131,16 @@ userfaults on the range registered. Not all ioctls will necessarily be · 130 unmodified lines
131	supported for all memory types (e.g. anonymous memory vs. shmem vs.	131	supported for all memory types (e.g. anonymous memory vs. shmem vs.
132	hugetlbfs), or all types of intercepted faults.	132	hugetlbfs), or all types of intercepted faults.
133		133
		134	.. note::
		135
		136	Re-registering an already-registered range must not drop any of the
		137	modes that install per-PTE markers — currently
		138	``UFFDIO_REGISTER_MODE_WP`` and ``UFFDIO_REGISTER_MODE_RWP``. Doing
		139	so would strand markers with no flag to describe them, so the call
		140	is rejected with ``-EBUSY``; userspace must issue
		141	``UFFDIO_UNREGISTER`` first. This differs from older kernels, which
		142	silently replaced the mode bits on re-registration.
		143
134	Userland can use the ``uffdio_register.ioctls`` to manage the virtual	144	Userland can use the ``uffdio_register.ioctls`` to manage the virtual
135	address space in the background (to add or potentially also remove	145	address space in the background (to add or potentially also remove
136	memory from the ``userfaultfd`` registered range). This means a userfault	146	memory from the ``userfaultfd`` registered range). This means a userfault

include/linux/userfaultfd_k.h+2−0

@@ -150,6 +150,8 @@ static inline uffd_flags_t uffd_flags_set_mode(uffd_flags_t flags, enum mfill_at · 149 unmodified lines
150		150
151	extern long uffd_wp_range(struct vm_area_struct *vma,	151	extern long uffd_wp_range(struct vm_area_struct *vma,
152	unsigned long start, unsigned long len, bool enable_wp);	152	unsigned long start, unsigned long len, bool enable_wp);
		153	extern int mrwprotect_range(struct userfaultfd_ctx *ctx, unsigned long start,
		154	unsigned long len, bool enable_rwp);
153		155
154	/* move_pages */	156	/* move_pages */
155	void double_pt_lock(spinlock_t ptl1, spinlock_t ptl2);	157	void double_pt_lock(spinlock_t ptl1, spinlock_t ptl2);

include/uapi/linux/userfaultfd.h+19−0

 #define _UFFDIO_WRITEPROTECT		(0x06)
 #define _UFFDIO_CONTINUE		(0x07)
 #define _UFFDIO_POISON			(0x08)
+#define _UFFDIO_RWPROTECT		(0x09)
 #define _UFFDIO_API			(0x3F)
 /* userfaultfd ioctl ids */
 				      struct uffdio_continue)
 #define UFFDIO_POISON		_IOWR(UFFDIO, _UFFDIO_POISON, \
 				      struct uffdio_poison)
+#define UFFDIO_RWPROTECT	_IOWR(UFFDIO, _UFFDIO_RWPROTECT,	\
+				      struct uffdio_rwprotect)
 /* read() structure */
 struct uffd_msg {
 #define UFFD_PAGEFAULT_FLAG_WRITE	(1<<0)	/* If this was a write fault */
 #define UFFD_PAGEFAULT_FLAG_WP		(1<<1)	/* If reason is VM_UFFD_WP */
 #define UFFD_PAGEFAULT_FLAG_MINOR	(1<<2)	/* If reason is VM_UFFD_MINOR */
+#define UFFD_PAGEFAULT_FLAG_RWP		(1<<3)	/* If reason is VM_UFFD_RWP */
 struct uffdio_api {
 	/* userland asks for an API number and the features to enable */
 	 *
 	 * UFFD_FEATURE_MOVE indicates that the kernel supports moving an
 	 * existing page contents from userspace.
+	 *
+	 * UFFD_FEATURE_RWP indicates that the kernel supports
+	 * UFFDIO_REGISTER_MODE_RWP for read-write protection tracking.
+	 * Pages are made inaccessible via UFFDIO_RWPROTECT and faults
+	 * are delivered when the pages are re-accessed.
 	 */
 #define UFFD_FEATURE_PAGEFAULT_FLAG_WP		(1<<0)
 #define UFFD_FEATURE_EVENT_FORK			(1<<1)
 #define UFFD_FEATURE_POISON			(1<<14)
 #define UFFD_FEATURE_WP_ASYNC			(1<<15)
 #define UFFD_FEATURE_MOVE			(1<<16)
+#define UFFD_FEATURE_RWP			(1<<17)
 	__u64 features;
 	__u64 ioctls;
 #define UFFDIO_REGISTER_MODE_MISSING	((__u64)1<<0)
 #define UFFDIO_REGISTER_MODE_WP		((__u64)1<<1)
 #define UFFDIO_REGISTER_MODE_MINOR	((__u64)1<<2)
+#define UFFDIO_REGISTER_MODE_RWP	((__u64)1<<3)
 	__u64 mode;
 	/*
 	__s64 updated;
 };
+struct uffdio_rwprotect {
+	struct uffdio_range range;
+	/* !RWP means undo RWP-protection */
+#define UFFDIO_RWPROTECT_MODE_RWP		((__u64)1<<0)
+#define UFFDIO_RWPROTECT_MODE_DONTWAKE		((__u64)1<<1)
+	__u64 mode;
+};
 struct uffdio_move {
 	__u64 dst;
 	__u64 src;

mm/Kconfig+9−0

@@ -1386,6 +1386,15 @@ config HAVE_ARCH_USERFAULTFD_MINOR · 1385 unmodified lines
1386	help	1386	help
1387	Arch has userfaultfd minor fault support	1387	Arch has userfaultfd minor fault support
1388		1388
		1389	config USERFAULTFD_RWP
		1390	def_bool y
		1391	depends on 64BIT && ARCH_HAS_PTE_PROTNONE && HAVE_ARCH_USERFAULTFD_WP
		1392	help
		1393	Userfaultfd read-write protection (UFFDIO_RWPROTECT) delivers a
		1394	userfaultfd notification on every access -- read or write -- to a
		1395	protected range, letting userspace observe the working set of a
		1396	process.
		1397
1389	menuconfig USERFAULTFD	1398	menuconfig USERFAULTFD
1390	bool "Enable userfaultfd() system call"	1399	bool "Enable userfaultfd() system call"
1391	depends on MMU	1400	depends on MMU

mm/userfaultfd.c+188−3

@@ -1164,6 +1164,75 @@ out_unlock: · 1163 unmodified lines
1164	return err;	1164	return err;
1165	}	1165	}
1166		1166
		1167	int mrwprotect_range(struct userfaultfd_ctx *ctx, unsigned long start,
		1168	unsigned long len, bool enable_rwp)
		1169	{
		1170	struct mm_struct *dst_mm = ctx->mm;
		1171	unsigned long end = start + len;
		1172	struct vm_area_struct *dst_vma;
		1173	unsigned int mm_cp_flags;
		1174	struct mmu_gather tlb;
		1175	bool found = false;
		1176	VMA_ITERATOR(vmi, dst_mm, start);
		1177
		1178	VM_WARN_ON_ONCE(start & ~PAGE_MASK);
		1179	VM_WARN_ON_ONCE(len & ~PAGE_MASK);
		1180	VM_WARN_ON_ONCE(start + len <= start);
		1181
		1182	guard(mmap_read_lock)(dst_mm);
		1183	guard(rwsem_read)(&ctx->map_changing_lock);
		1184
		1185	if (atomic_read(&ctx->mmap_changing))
		1186	return -EAGAIN;
		1187
		1188	if (enable_rwp)
		1189	mm_cp_flags = MM_CP_UFFD_RWP;
		1190	else
		1191	mm_cp_flags = MM_CP_UFFD_RWP_RESOLVE;
		1192
		1193	/*
		1194	* Pre-scan the range: validate every spanned VMA before applying
		1195	* any change_protection() so a partial failure cannot leave the
		1196	* process with only a prefix of the range re-protected.
		1197	*/
		1198	for_each_vma_range(vmi, dst_vma, end) {
		1199	if (!userfaultfd_rwp(dst_vma))
		1200	return -ENOENT;
		1201
		1202	if (is_vm_hugetlb_page(dst_vma)) {
		1203	unsigned long page_mask;
		1204
		1205	page_mask = vma_kernel_pagesize(dst_vma) - 1;
		1206	if ((start & page_mask) \|\| (len & page_mask))
		1207	return -EINVAL;
		1208	}
		1209	found = true;
		1210	}
		1211	if (!found)
		1212	return -ENOENT;
		1213
		1214	vma_iter_set(&vmi, start);
		1215	tlb_gather_mmu(&tlb, dst_mm);
		1216	for_each_vma_range(vmi, dst_vma, end) {
		1217	unsigned long vma_start = max(dst_vma->vm_start, start);
		1218	unsigned long vma_end = min(dst_vma->vm_end, end);
		1219	unsigned int flags = mm_cp_flags;
		1220
		1221	/*
		1222	* On resolve, try to upgrade writability per-VMA --
		1223	* MM_CP_TRY_CHANGE_WRITABLE WARNs in
		1224	* maybe_change_pte_writable() if the VMA is not VM_WRITE,
		1225	* and RWP can be registered on PROT_READ-only mappings.
		1226	*/
		1227	if (!enable_rwp && vma_wants_manual_pte_write_upgrade(dst_vma))
		1228	flags \|= MM_CP_TRY_CHANGE_WRITABLE;
		1229
		1230	change_protection(&tlb, dst_vma, vma_start, vma_end, flags);
		1231	}
		1232	tlb_finish_mmu(&tlb);
		1233
		1234	return 0;
		1235	}
1167		1236
1168	void double_pt_lock(spinlock_t *ptl1,	1237	void double_pt_lock(spinlock_t *ptl1,
1169	spinlock_t *ptl2)	1238	spinlock_t *ptl2)
@@ -2207,9 +2276,22 @@ static struct vm_area_struct userfaultfd_clear_vma(struct vma_iterator vmi, · 1037 unmodified lines
2207	if (start == vma->vm_start && end == vma->vm_end)	2276	if (start == vma->vm_start && end == vma->vm_end)
2208	give_up_on_oom = true;	2277	give_up_on_oom = true;
2209		2278
2210	/* Reset ptes for the whole vma range if wr-protected */	2279	/* Clear the uffd bit and/or restore protnone PTEs */
2211	if (userfaultfd_wp(vma))	2280	if (userfaultfd_protected(vma)) {
2212	uffd_wp_range(vma, start, end - start, false);	2281	unsigned int mm_cp_flags = 0;
		2282	struct mmu_gather tlb;
		2283
		2284	if (userfaultfd_wp(vma))
		2285	mm_cp_flags \|= MM_CP_UFFD_WP_RESOLVE;
		2286	if (userfaultfd_rwp(vma))
		2287	mm_cp_flags \|= MM_CP_UFFD_RWP_RESOLVE;
		2288	if (vma_wants_manual_pte_write_upgrade(vma))
		2289	mm_cp_flags \|= MM_CP_TRY_CHANGE_WRITABLE;
		2290
		2291	tlb_gather_mmu(&tlb, vma->vm_mm);
		2292	change_protection(&tlb, vma, start, end, mm_cp_flags);
		2293	tlb_finish_mmu(&tlb);
		2294	}
2213		2295
2214	ret = vma_modify_flags_uffd(vmi, prev, vma, start, end,	2296	ret = vma_modify_flags_uffd(vmi, prev, vma, start, end,
2215	&new_vma_flags, NULL_VM_UFFD_CTX,	2297	&new_vma_flags, NULL_VM_UFFD_CTX,
@@ -2258,6 +2340,14 @@ static int userfaultfd_register_range(struct userfaultfd_ctx *ctx, · 42 unmodified lines
2258	vma_test_all_mask(vma, vma_flags))	2340	vma_test_all_mask(vma, vma_flags))
2259	goto skip;	2341	goto skip;
2260		2342
		2343	/*
		2344	* Pre-scan in userfaultfd_register() already rejected mode
		2345	* switches that would drop VM_UFFD_WP or VM_UFFD_RWP, so a
		2346	* stray bit here is a bug.
		2347	*/
		2348	VM_WARN_ON_ONCE(vma->vm_userfaultfd_ctx.ctx == ctx &&
		2349	vma->vm_flags & (VM_UFFD_WP \| VM_UFFD_RWP) & ~vm_flags);
		2350
2261	if (vma->vm_start > start)	2351	if (vma->vm_start > start)
2262	start = vma->vm_start;	2352	start = vma->vm_start;
2263	vma_end = min(end, vma->vm_end);	2353	vma_end = min(end, vma->vm_end);
@@ -2524,6 +2614,8 @@ static inline struct uffd_msg userfault_msg(unsigned long address, · 260 unmodified lines
2524	msg.arg.pagefault.flags \|= UFFD_PAGEFAULT_FLAG_WRITE;	2614	msg.arg.pagefault.flags \|= UFFD_PAGEFAULT_FLAG_WRITE;
2525	if (reason & VM_UFFD_WP)	2615	if (reason & VM_UFFD_WP)
2526	msg.arg.pagefault.flags \|= UFFD_PAGEFAULT_FLAG_WP;	2616	msg.arg.pagefault.flags \|= UFFD_PAGEFAULT_FLAG_WP;
		2617	if (reason & VM_UFFD_RWP)
		2618	msg.arg.pagefault.flags \|= UFFD_PAGEFAULT_FLAG_RWP;
2527	if (reason & VM_UFFD_MINOR)	2619	if (reason & VM_UFFD_MINOR)
2528	msg.arg.pagefault.flags \|= UFFD_PAGEFAULT_FLAG_MINOR;	2620	msg.arg.pagefault.flags \|= UFFD_PAGEFAULT_FLAG_MINOR;
2529	if (features & UFFD_FEATURE_THREAD_ID)	2621	if (features & UFFD_FEATURE_THREAD_ID)
@@ -3623,6 +3715,22 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, · 1093 unmodified lines
3623		3715
3624	vm_flags \|= VM_UFFD_WP;	3716	vm_flags \|= VM_UFFD_WP;
3625	}	3717	}
		3718	if (uffdio_register.mode & UFFDIO_REGISTER_MODE_RWP) {
		3719	if (!pgtable_supports_uffd() \|\| VM_UFFD_RWP == VM_NONE)
		3720	goto out;
		3721	if (!(ctx->features & UFFD_FEATURE_RWP))
		3722	goto out;
		3723	vm_flags \|= VM_UFFD_RWP;
		3724	}
		3725
		3726	/*
		3727	* WP and RWP share the uffd PTE bit and
		3728	* cannot coexist in the same VMA — the bit would carry ambiguous
		3729	* semantics. Reject the combination up front.
		3730	*/
		3731	if ((vm_flags & VM_UFFD_WP) && (vm_flags & VM_UFFD_RWP))
		3732	goto out;
		3733
3626	if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR) {	3734	if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR) {
3627	#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR	3735	#ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
3628	goto out;	3736	goto out;
@@ -3677,6 +3785,17 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, · 48 unmodified lines
3677	if (!vma_can_userfault(cur, vm_flags, wp_async))	3785	if (!vma_can_userfault(cur, vm_flags, wp_async))
3678	goto out_unlock;	3786	goto out_unlock;
3679		3787
		3788	/*
		3789	* RWP uses protnone as an access-tracking marker. PROT_NONE
		3790	* VMAs have vm_page_prot == PAGE_NONE, so RWP resolution
		3791	* cannot make a page accessible again. Reject at register
		3792	* time only: a VMA that later becomes inaccessible via
		3793	* mprotect() must still be unregisterable, so this is not
		3794	* part of vma_can_userfault().
		3795	*/
		3796	if ((vm_flags & VM_UFFD_RWP) && !vma_is_accessible(cur))
		3797	goto out_unlock;
		3798
3680	/*	3799	/*
3681	* UFFDIO_COPY will fill file holes even without	3800	* UFFDIO_COPY will fill file holes even without
3682	* PROT_WRITE. This check enforces that if this is a	3801	* PROT_WRITE. This check enforces that if this is a
@@ -3716,6 +3835,16 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, · 33 unmodified lines
3716	cur->vm_userfaultfd_ctx.ctx != ctx)	3835	cur->vm_userfaultfd_ctx.ctx != ctx)
3717	goto out_unlock;	3836	goto out_unlock;
3718		3837
		3838	/*
		3839	* Mode switches that drop VM_UFFD_WP or VM_UFFD_RWP would
		3840	* leave PTE markers without the flag that describes them;
		3841	* subsequent mprotect() would then promote stale markers
		3842	* into the other mode. Require an unregister first.
		3843	*/
		3844	if (cur->vm_userfaultfd_ctx.ctx == ctx &&
		3845	cur->vm_flags & (VM_UFFD_WP \| VM_UFFD_RWP) & ~vm_flags)
		3846	goto out_unlock;
		3847
3719	/*	3848	/*
3720	* Note vmas containing huge pages	3849	* Note vmas containing huge pages
3721	*/	3850	*/
@@ -3749,6 +3878,10 @@ out_unlock: · 27 unmodified lines
3749	if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR))	3878	if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR))
3750	ioctls_out &= ~((__u64)1 << _UFFDIO_CONTINUE);	3879	ioctls_out &= ~((__u64)1 << _UFFDIO_CONTINUE);
3751		3880
		3881	/* RWPROTECT is only supported for RWP ranges */
		3882	if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_RWP))
		3883	ioctls_out &= ~((__u64)1 << _UFFDIO_RWPROTECT);
		3884
3752	/*	3885	/*
3753	* Now that we scanned all vmas we can already tell	3886	* Now that we scanned all vmas we can already tell
3754	* userland which ioctls methods are guaranteed to	3887	* userland which ioctls methods are guaranteed to
@@ -4096,6 +4229,55 @@ static int userfaultfd_writeprotect(struct userfaultfd_ctx *ctx, · 341 unmodified lines
4096	return ret;	4229	return ret;
4097	}	4230	}
4098		4231
		4232	static int userfaultfd_rwprotect(struct userfaultfd_ctx *ctx,
		4233	unsigned long arg)
		4234	{
		4235	int ret;
		4236	struct uffdio_rwprotect uffdio_rwp;
		4237	struct userfaultfd_wake_range range;
		4238	bool mode_rwp, mode_dontwake;
		4239
		4240	if (atomic_read(&ctx->mmap_changing))
		4241	return -EAGAIN;
		4242
		4243	if (copy_from_user(&uffdio_rwp, (void __user *)arg,
		4244	sizeof(uffdio_rwp)))
		4245	return -EFAULT;
		4246
		4247	ret = validate_range(ctx->mm, uffdio_rwp.range.start,
		4248	uffdio_rwp.range.len);
		4249	if (ret)
		4250	return ret;
		4251
		4252	if (uffdio_rwp.mode & ~(UFFDIO_RWPROTECT_MODE_DONTWAKE \|
		4253	UFFDIO_RWPROTECT_MODE_RWP))
		4254	return -EINVAL;
		4255
		4256	mode_rwp = uffdio_rwp.mode & UFFDIO_RWPROTECT_MODE_RWP;
		4257	mode_dontwake = uffdio_rwp.mode & UFFDIO_RWPROTECT_MODE_DONTWAKE;
		4258
		4259	if (mode_rwp && mode_dontwake)
		4260	return -EINVAL;
		4261
		4262	if (mmget_not_zero(ctx->mm)) {
		4263	ret = mrwprotect_range(ctx, uffdio_rwp.range.start,
		4264	uffdio_rwp.range.len, mode_rwp);
		4265	mmput(ctx->mm);
		4266	} else {
		4267	return -ESRCH;
		4268	}
		4269
		4270	if (ret)
		4271	return ret;
		4272
		4273	if (!mode_rwp && !mode_dontwake) {
		4274	range.start = uffdio_rwp.range.start;
		4275	range.len = uffdio_rwp.range.len;
		4276	wake_userfault(ctx, &range);
		4277	}
		4278	return ret;
		4279	}
		4280
4099	static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg)	4281	static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg)
4100	{	4282	{
4101	__s64 ret;	4283	__s64 ret;
@@ -4402,6 +4584,9 @@ static long userfaultfd_ioctl(struct file *file, unsigned cmd, · 300 unmodified lines
4402	case UFFDIO_POISON:	4584	case UFFDIO_POISON:
4403	ret = userfaultfd_poison(ctx, arg);	4585	ret = userfaultfd_poison(ctx, arg);
4404	break;	4586	break;
		4587	case UFFDIO_RWPROTECT:
		4588	ret = userfaultfd_rwprotect(ctx, arg);
		4589	break;
4405	}	4590	}
4406	return ret;	4591	return ret;
4407	}	4592	}

userfaultfd: add UFFDIO_REGISTER_MODE_RWP and UFFDIO_RWPROTECT plumbing

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code