History

userfaultfd: add UFFDIO_REGISTER_MODE_RWP and UFFDIO_RWPROTECT plumbing

Add the userspace interface for read-write protection tracking: - UFFDIO_REGISTER_MODE_RWP register a range for RWP tracking - UFFD_FEATURE_RWP capability bit - UFFDIO_RWPROTECT install / remove RWP on a range Introduce CONFIG_USERFAULTFD_RWP, auto-selected on 64-bit kernels with ARCH_HAS_PTE_PROTNONE and HAVE_ARCH_USERFAULTFD_WP. The symbol gates VM_UFFD_RWP (previously aliased to VM_NONE) and the smaps/trace-flag hooks added in the preparatory patches; without it the UAPI bits added here have nothing to drive and would be unreachable. Registration sets VM_UFFD_RWP on the VMA. Combining MODE_WP with MODE_RWP is rejected because both modes claim the uffd PTE bit. UFFDIO_RWPROTECT is the bidirectional counterpart of UFFDIO_WRITEPROTECT: - MODE_RWP change_protection() with MM_CP_UFFD_RWP installs PAGE_NONE and sets the uffd bit on present PTEs - !MODE_RWP change_protection() with MM_CP_UFFD_RWP_RESOLVE restores vma->vm_page_prot and clears the bit userfaultfd_clear_vma() runs the same resolve pass on unregister so RWP state cannot outlive the uffd. Re-registering a range must not drop a mode that installs per-PTE markers (WP or RWP); doing so returns -EBUSY. This also closes a pre-existing window where re-registering without MODE_WP would strand uffd-wp markers: before, those caused extra write-faults but were otherwise benign; with RWP preservation in place, a subsequent mprotect() on a VM_UFFD_RWP VMA would silently promote the stale markers to RWP. The feature is not yet advertised. UFFDIO_REGISTER_MODE_RWP, UFFD_FEATURE_RWP, and _UFFDIO_RWPROTECT are intentionally absent from UFFD_API_REGISTER_MODES, UFFD_API_FEATURES, and UFFD_API_RANGE_IOCTLS, so UFFDIO_API masks them out and the register-mode validator rejects the bit. The follow-up patch adds fault dispatch and exposes the UAPI.

Kiryl Shutsemau (Meta) committed 4 months ago

Commit 6eab8f21 parent 7974c235 files · +228 −3Browse files

Unified · Split

5 files
  1. Documentation/admin-guide/mm/userfaultfd.rst +10 −0
  2. include/linux/userfaultfd_k.h +2 −0
  3. include/uapi/linux/userfaultfd.h +19 −0
  4. mm/Kconfig +9 −0
  5. mm/userfaultfd.c +188 −3
Documentation/admin-guide/mm/userfaultfd.rst+10−0
@@ -131,6 +131,16 @@ userfaults on the range registered. Not all ioctls will necessarily be · 130 unmodified lines
131131 supported for all memory types (e.g. anonymous memory vs. shmem vs.
132132 hugetlbfs), or all types of intercepted faults.
133133
134+.. note::
135+
136+ Re-registering an already-registered range must not drop any of the
137+ modes that install per-PTE markers — currently
138+ ``UFFDIO_REGISTER_MODE_WP`` and ``UFFDIO_REGISTER_MODE_RWP``. Doing
139+ so would strand markers with no flag to describe them, so the call
140+ is rejected with ``-EBUSY``; userspace must issue
141+ ``UFFDIO_UNREGISTER`` first. This differs from older kernels, which
142+ silently replaced the mode bits on re-registration.
143+
134144 Userland can use the ``uffdio_register.ioctls`` to manage the virtual
135145 address space in the background (to add or potentially also remove
136146 memory from the ``userfaultfd`` registered range). This means a userfault
include/linux/userfaultfd_k.h+2−0
@@ -150,6 +150,8 @@ static inline uffd_flags_t uffd_flags_set_mode(uffd_flags_t flags, enum mfill_at · 149 unmodified lines
150150
151151 extern long uffd_wp_range(struct vm_area_struct *vma,
152152 unsigned long start, unsigned long len, bool enable_wp);
153+extern int mrwprotect_range(struct userfaultfd_ctx *ctx, unsigned long start,
154+ unsigned long len, bool enable_rwp);
153155
154156 /* move_pages */
155157 void double_pt_lock(spinlock_t *ptl1, spinlock_t *ptl2);
include/uapi/linux/userfaultfd.h+19−0
@@ -79,6 +79,7 @@ · 78 unmodified lines
7979 #define _UFFDIO_WRITEPROTECT (0x06)
8080 #define _UFFDIO_CONTINUE (0x07)
8181 #define _UFFDIO_POISON (0x08)
82+#define _UFFDIO_RWPROTECT (0x09)
8283 #define _UFFDIO_API (0x3F)
8384
8485 /* userfaultfd ioctl ids */
@@ -103,6 +104,8 @@ · 18 unmodified lines
103104 struct uffdio_continue)
104105 #define UFFDIO_POISON _IOWR(UFFDIO, _UFFDIO_POISON, \
105106 struct uffdio_poison)
107+#define UFFDIO_RWPROTECT _IOWR(UFFDIO, _UFFDIO_RWPROTECT, \
108+ struct uffdio_rwprotect)
106109
107110 /* read() structure */
108111 struct uffd_msg {
@@ -158,6 +161,7 @@ struct uffd_msg { · 49 unmodified lines
158161 #define UFFD_PAGEFAULT_FLAG_WRITE (1<<0) /* If this was a write fault */
159162 #define UFFD_PAGEFAULT_FLAG_WP (1<<1) /* If reason is VM_UFFD_WP */
160163 #define UFFD_PAGEFAULT_FLAG_MINOR (1<<2) /* If reason is VM_UFFD_MINOR */
164+#define UFFD_PAGEFAULT_FLAG_RWP (1<<3) /* If reason is VM_UFFD_RWP */
161165
162166 struct uffdio_api {
163167 /* userland asks for an API number and the features to enable */
@@ -230,6 +234,11 @@ struct uffdio_api { · 66 unmodified lines
230234 *
231235 * UFFD_FEATURE_MOVE indicates that the kernel supports moving an
232236 * existing page contents from userspace.
237+ *
238+ * UFFD_FEATURE_RWP indicates that the kernel supports
239+ * UFFDIO_REGISTER_MODE_RWP for read-write protection tracking.
240+ * Pages are made inaccessible via UFFDIO_RWPROTECT and faults
241+ * are delivered when the pages are re-accessed.
233242 */
234243 #define UFFD_FEATURE_PAGEFAULT_FLAG_WP (1<<0)
235244 #define UFFD_FEATURE_EVENT_FORK (1<<1)
@@ -248,6 +257,7 @@ struct uffdio_api { · 12 unmodified lines
248257 #define UFFD_FEATURE_POISON (1<<14)
249258 #define UFFD_FEATURE_WP_ASYNC (1<<15)
250259 #define UFFD_FEATURE_MOVE (1<<16)
260+#define UFFD_FEATURE_RWP (1<<17)
251261 __u64 features;
252262
253263 __u64 ioctls;
@@ -263,6 +273,7 @@ struct uffdio_register { · 9 unmodified lines
263273 #define UFFDIO_REGISTER_MODE_MISSING ((__u64)1<<0)
264274 #define UFFDIO_REGISTER_MODE_WP ((__u64)1<<1)
265275 #define UFFDIO_REGISTER_MODE_MINOR ((__u64)1<<2)
276+#define UFFDIO_REGISTER_MODE_RWP ((__u64)1<<3)
266277 __u64 mode;
267278
268279 /*
@@ -356,6 +367,14 @@ struct uffdio_poison { · 87 unmodified lines
356367 __s64 updated;
357368 };
358369
370+struct uffdio_rwprotect {
371+ struct uffdio_range range;
372+ /* !RWP means undo RWP-protection */
373+#define UFFDIO_RWPROTECT_MODE_RWP ((__u64)1<<0)
374+#define UFFDIO_RWPROTECT_MODE_DONTWAKE ((__u64)1<<1)
375+ __u64 mode;
376+};
377+
359378 struct uffdio_move {
360379 __u64 dst;
361380 __u64 src;
mm/Kconfig+9−0
@@ -1386,6 +1386,15 @@ config HAVE_ARCH_USERFAULTFD_MINOR · 1385 unmodified lines
13861386 help
13871387 Arch has userfaultfd minor fault support
13881388
1389+config USERFAULTFD_RWP
1390+ def_bool y
1391+ depends on 64BIT && ARCH_HAS_PTE_PROTNONE && HAVE_ARCH_USERFAULTFD_WP
1392+ help
1393+ Userfaultfd read-write protection (UFFDIO_RWPROTECT) delivers a
1394+ userfaultfd notification on every access -- read or write -- to a
1395+ protected range, letting userspace observe the working set of a
1396+ process.
1397+
13891398 menuconfig USERFAULTFD
13901399 bool "Enable userfaultfd() system call"
13911400 depends on MMU
mm/userfaultfd.c+188−3
@@ -1164,6 +1164,75 @@ out_unlock: · 1163 unmodified lines
11641164 return err;
11651165 }
11661166
1167+int mrwprotect_range(struct userfaultfd_ctx *ctx, unsigned long start,
1168+ unsigned long len, bool enable_rwp)
1169+{
1170+ struct mm_struct *dst_mm = ctx->mm;
1171+ unsigned long end = start + len;
1172+ struct vm_area_struct *dst_vma;
1173+ unsigned int mm_cp_flags;
1174+ struct mmu_gather tlb;
1175+ bool found = false;
1176+ VMA_ITERATOR(vmi, dst_mm, start);
1177+
1178+ VM_WARN_ON_ONCE(start & ~PAGE_MASK);
1179+ VM_WARN_ON_ONCE(len & ~PAGE_MASK);
1180+ VM_WARN_ON_ONCE(start + len <= start);
1181+
1182+ guard(mmap_read_lock)(dst_mm);
1183+ guard(rwsem_read)(&ctx->map_changing_lock);
1184+
1185+ if (atomic_read(&ctx->mmap_changing))
1186+ return -EAGAIN;
1187+
1188+ if (enable_rwp)
1189+ mm_cp_flags = MM_CP_UFFD_RWP;
1190+ else
1191+ mm_cp_flags = MM_CP_UFFD_RWP_RESOLVE;
1192+
1193+ /*
1194+ * Pre-scan the range: validate every spanned VMA before applying
1195+ * any change_protection() so a partial failure cannot leave the
1196+ * process with only a prefix of the range re-protected.
1197+ */
1198+ for_each_vma_range(vmi, dst_vma, end) {
1199+ if (!userfaultfd_rwp(dst_vma))
1200+ return -ENOENT;
1201+
1202+ if (is_vm_hugetlb_page(dst_vma)) {
1203+ unsigned long page_mask;
1204+
1205+ page_mask = vma_kernel_pagesize(dst_vma) - 1;
1206+ if ((start & page_mask) || (len & page_mask))
1207+ return -EINVAL;
1208+ }
1209+ found = true;
1210+ }
1211+ if (!found)
1212+ return -ENOENT;
1213+
1214+ vma_iter_set(&vmi, start);
1215+ tlb_gather_mmu(&tlb, dst_mm);
1216+ for_each_vma_range(vmi, dst_vma, end) {
1217+ unsigned long vma_start = max(dst_vma->vm_start, start);
1218+ unsigned long vma_end = min(dst_vma->vm_end, end);
1219+ unsigned int flags = mm_cp_flags;
1220+
1221+ /*
1222+ * On resolve, try to upgrade writability per-VMA --
1223+ * MM_CP_TRY_CHANGE_WRITABLE WARNs in
1224+ * maybe_change_pte_writable() if the VMA is not VM_WRITE,
1225+ * and RWP can be registered on PROT_READ-only mappings.
1226+ */
1227+ if (!enable_rwp && vma_wants_manual_pte_write_upgrade(dst_vma))
1228+ flags |= MM_CP_TRY_CHANGE_WRITABLE;
1229+
1230+ change_protection(&tlb, dst_vma, vma_start, vma_end, flags);
1231+ }
1232+ tlb_finish_mmu(&tlb);
1233+
1234+ return 0;
1235+}
11671236
11681237 void double_pt_lock(spinlock_t *ptl1,
11691238 spinlock_t *ptl2)
@@ -2207,9 +2276,22 @@ static struct vm_area_struct *userfaultfd_clear_vma(struct vma_iterator *vmi, · 1037 unmodified lines
22072276 if (start == vma->vm_start && end == vma->vm_end)
22082277 give_up_on_oom = true;
22092278
2210− /* Reset ptes for the whole vma range if wr-protected */
2211− if (userfaultfd_wp(vma))
2212− uffd_wp_range(vma, start, end - start, false);
2279+ /* Clear the uffd bit and/or restore protnone PTEs */
2280+ if (userfaultfd_protected(vma)) {
2281+ unsigned int mm_cp_flags = 0;
2282+ struct mmu_gather tlb;
2283+
2284+ if (userfaultfd_wp(vma))
2285+ mm_cp_flags |= MM_CP_UFFD_WP_RESOLVE;
2286+ if (userfaultfd_rwp(vma))
2287+ mm_cp_flags |= MM_CP_UFFD_RWP_RESOLVE;
2288+ if (vma_wants_manual_pte_write_upgrade(vma))
2289+ mm_cp_flags |= MM_CP_TRY_CHANGE_WRITABLE;
2290+
2291+ tlb_gather_mmu(&tlb, vma->vm_mm);
2292+ change_protection(&tlb, vma, start, end, mm_cp_flags);
2293+ tlb_finish_mmu(&tlb);
2294+ }
22132295
22142296 ret = vma_modify_flags_uffd(vmi, prev, vma, start, end,
22152297 &new_vma_flags, NULL_VM_UFFD_CTX,
@@ -2258,6 +2340,14 @@ static int userfaultfd_register_range(struct userfaultfd_ctx *ctx, · 42 unmodified lines
22582340 vma_test_all_mask(vma, vma_flags))
22592341 goto skip;
22602342
2343+ /*
2344+ * Pre-scan in userfaultfd_register() already rejected mode
2345+ * switches that would drop VM_UFFD_WP or VM_UFFD_RWP, so a
2346+ * stray bit here is a bug.
2347+ */
2348+ VM_WARN_ON_ONCE(vma->vm_userfaultfd_ctx.ctx == ctx &&
2349+ vma->vm_flags & (VM_UFFD_WP | VM_UFFD_RWP) & ~vm_flags);
2350+
22612351 if (vma->vm_start > start)
22622352 start = vma->vm_start;
22632353 vma_end = min(end, vma->vm_end);
@@ -2524,6 +2614,8 @@ static inline struct uffd_msg userfault_msg(unsigned long address, · 260 unmodified lines
25242614 msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WRITE;
25252615 if (reason & VM_UFFD_WP)
25262616 msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WP;
2617+ if (reason & VM_UFFD_RWP)
2618+ msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_RWP;
25272619 if (reason & VM_UFFD_MINOR)
25282620 msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_MINOR;
25292621 if (features & UFFD_FEATURE_THREAD_ID)
@@ -3623,6 +3715,22 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, · 1093 unmodified lines
36233715
36243716 vm_flags |= VM_UFFD_WP;
36253717 }
3718+ if (uffdio_register.mode & UFFDIO_REGISTER_MODE_RWP) {
3719+ if (!pgtable_supports_uffd() || VM_UFFD_RWP == VM_NONE)
3720+ goto out;
3721+ if (!(ctx->features & UFFD_FEATURE_RWP))
3722+ goto out;
3723+ vm_flags |= VM_UFFD_RWP;
3724+ }
3725+
3726+ /*
3727+ * WP and RWP share the uffd PTE bit and
3728+ * cannot coexist in the same VMA — the bit would carry ambiguous
3729+ * semantics. Reject the combination up front.
3730+ */
3731+ if ((vm_flags & VM_UFFD_WP) && (vm_flags & VM_UFFD_RWP))
3732+ goto out;
3733+
36263734 if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR) {
36273735 #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
36283736 goto out;
@@ -3677,6 +3785,17 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, · 48 unmodified lines
36773785 if (!vma_can_userfault(cur, vm_flags, wp_async))
36783786 goto out_unlock;
36793787
3788+ /*
3789+ * RWP uses protnone as an access-tracking marker. PROT_NONE
3790+ * VMAs have vm_page_prot == PAGE_NONE, so RWP resolution
3791+ * cannot make a page accessible again. Reject at register
3792+ * time only: a VMA that later becomes inaccessible via
3793+ * mprotect() must still be unregisterable, so this is not
3794+ * part of vma_can_userfault().
3795+ */
3796+ if ((vm_flags & VM_UFFD_RWP) && !vma_is_accessible(cur))
3797+ goto out_unlock;
3798+
36803799 /*
36813800 * UFFDIO_COPY will fill file holes even without
36823801 * PROT_WRITE. This check enforces that if this is a
@@ -3716,6 +3835,16 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, · 33 unmodified lines
37163835 cur->vm_userfaultfd_ctx.ctx != ctx)
37173836 goto out_unlock;
37183837
3838+ /*
3839+ * Mode switches that drop VM_UFFD_WP or VM_UFFD_RWP would
3840+ * leave PTE markers without the flag that describes them;
3841+ * subsequent mprotect() would then promote stale markers
3842+ * into the other mode. Require an unregister first.
3843+ */
3844+ if (cur->vm_userfaultfd_ctx.ctx == ctx &&
3845+ cur->vm_flags & (VM_UFFD_WP | VM_UFFD_RWP) & ~vm_flags)
3846+ goto out_unlock;
3847+
37193848 /*
37203849 * Note vmas containing huge pages
37213850 */
@@ -3749,6 +3878,10 @@ out_unlock: · 27 unmodified lines
37493878 if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR))
37503879 ioctls_out &= ~((__u64)1 << _UFFDIO_CONTINUE);
37513880
3881+ /* RWPROTECT is only supported for RWP ranges */
3882+ if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_RWP))
3883+ ioctls_out &= ~((__u64)1 << _UFFDIO_RWPROTECT);
3884+
37523885 /*
37533886 * Now that we scanned all vmas we can already tell
37543887 * userland which ioctls methods are guaranteed to
@@ -4096,6 +4229,55 @@ static int userfaultfd_writeprotect(struct userfaultfd_ctx *ctx, · 341 unmodified lines
40964229 return ret;
40974230 }
40984231
4232+static int userfaultfd_rwprotect(struct userfaultfd_ctx *ctx,
4233+ unsigned long arg)
4234+{
4235+ int ret;
4236+ struct uffdio_rwprotect uffdio_rwp;
4237+ struct userfaultfd_wake_range range;
4238+ bool mode_rwp, mode_dontwake;
4239+
4240+ if (atomic_read(&ctx->mmap_changing))
4241+ return -EAGAIN;
4242+
4243+ if (copy_from_user(&uffdio_rwp, (void __user *)arg,
4244+ sizeof(uffdio_rwp)))
4245+ return -EFAULT;
4246+
4247+ ret = validate_range(ctx->mm, uffdio_rwp.range.start,
4248+ uffdio_rwp.range.len);
4249+ if (ret)
4250+ return ret;
4251+
4252+ if (uffdio_rwp.mode & ~(UFFDIO_RWPROTECT_MODE_DONTWAKE |
4253+ UFFDIO_RWPROTECT_MODE_RWP))
4254+ return -EINVAL;
4255+
4256+ mode_rwp = uffdio_rwp.mode & UFFDIO_RWPROTECT_MODE_RWP;
4257+ mode_dontwake = uffdio_rwp.mode & UFFDIO_RWPROTECT_MODE_DONTWAKE;
4258+
4259+ if (mode_rwp && mode_dontwake)
4260+ return -EINVAL;
4261+
4262+ if (mmget_not_zero(ctx->mm)) {
4263+ ret = mrwprotect_range(ctx, uffdio_rwp.range.start,
4264+ uffdio_rwp.range.len, mode_rwp);
4265+ mmput(ctx->mm);
4266+ } else {
4267+ return -ESRCH;
4268+ }
4269+
4270+ if (ret)
4271+ return ret;
4272+
4273+ if (!mode_rwp && !mode_dontwake) {
4274+ range.start = uffdio_rwp.range.start;
4275+ range.len = uffdio_rwp.range.len;
4276+ wake_userfault(ctx, &range);
4277+ }
4278+ return ret;
4279+}
4280+
40994281 static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg)
41004282 {
41014283 __s64 ret;
@@ -4402,6 +4584,9 @@ static long userfaultfd_ioctl(struct file *file, unsigned cmd, · 300 unmodified lines
44024584 case UFFDIO_POISON:
44034585 ret = userfaultfd_poison(ctx, arg);
44044586 break;
4587+ case UFFDIO_RWPROTECT:
4588+ ret = userfaultfd_rwprotect(ctx, arg);
4589+ break;
44054590 }
44064591 return ret;
44074592 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t