History

mm/userfaultfd: add RWP fault delivery and expose UFFDIO_REGISTER_MODE_RWP

Wire the fault side of read-write protection tracking and turn the userspace interface on. An RWP-protected PTE is PAGE_NONE with the uffd bit set. The PROT_NONE triggers a fault on any access; the uffd bit distinguishes it from plain mprotect(PROT_NONE) or NUMA hinting. Fault dispatch, per level: PTE handle_pte_fault() -> do_uffd_rwp() PMD __handle_mm_fault() -> do_huge_pmd_uffd_rwp() hugetlb hugetlb_fault() -> hugetlb_handle_userfault() The RWP branches gate on userfaultfd_pte_rwp() / userfaultfd_huge_pmd_rwp() (VM_UFFD_RWP plus the uffd bit) and fall through to do_numa_page() / do_huge_pmd_numa_page() otherwise. Each delivers a UFFD_PAGEFAULT_FLAG_RWP message through handle_userfault(); the handler resolves it with UFFDIO_RWPROTECT clearing MODE_RWP. userfaultfd_must_wait() and userfaultfd_huge_must_wait() add matching protnone+uffd waiters so sync-mode fault handlers block correctly. Expose the UAPI: UFFDIO_REGISTER_MODE_RWP -> UFFD_API_REGISTER_MODES UFFD_FEATURE_RWP -> UFFD_API_FEATURES _UFFDIO_RWPROTECT -> UFFD_API_RANGE_IOCTLS UFFD_API_RANGE_IOCTLS_BASIC UFFD_FEATURE_RWP is masked out at UFFDIO_API time when PROT_NONE is not available or VM_UFFD_RWP aliases VM_NONE (32-bit), so userspace never sees an advertised-but-broken feature. Works on anonymous, shmem, and hugetlb memory.

Kiryl Shutsemau (Meta) committed 4 months ago

Commit 2d427bb1 parent 6eab8f27 files · +114 −8Browse files

Unified · Split

7 files
  1. include/linux/huge_mm.h +7 −0
  2. include/linux/userfaultfd_k.h +24 −0
  3. include/uapi/linux/userfaultfd.h +8 −4
  4. mm/huge_memory.c +5 −0
  5. mm/hugetlb.c +11 −0
  6. mm/memory.c +29 −2
  7. mm/userfaultfd.c +30 −2
include/linux/huge_mm.h+7−0
@@ -529,6 +529,8 @@ static inline bool folio_test_pmd_mappable(struct folio *folio) · 528 unmodified lines
529529
530530 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf);
531531
532+vm_fault_t do_huge_pmd_uffd_rwp(struct vm_fault *vmf);
533+
532534 vm_fault_t do_huge_pmd_device_private(struct vm_fault *vmf);
533535
534536 extern struct folio *huge_zero_folio;
@@ -723,6 +725,11 @@ static inline spinlock_t *pud_trans_huge_lock(pud_t *pud, · 188 unmodified lines
723725 return NULL;
724726 }
725727
728+static inline vm_fault_t do_huge_pmd_uffd_rwp(struct vm_fault *vmf)
729+{
730+ return 0;
731+}
732+
726733 static inline vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf)
727734 {
728735 return 0;
include/linux/userfaultfd_k.h+24−0
@@ -241,6 +241,18 @@ static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, · 240 unmodified lines
241241 return userfaultfd_wp(vma) && pmd_uffd(pmd);
242242 }
243243
244+static inline bool userfaultfd_pte_rwp(struct vm_area_struct *vma,
245+ pte_t pte)
246+{
247+ return userfaultfd_rwp(vma) && pte_uffd(pte);
248+}
249+
250+static inline bool userfaultfd_huge_pmd_rwp(struct vm_area_struct *vma,
251+ pmd_t pmd)
252+{
253+ return userfaultfd_rwp(vma) && pmd_uffd(pmd);
254+}
255+
244256 static inline bool userfaultfd_armed(struct vm_area_struct *vma)
245257 {
246258 return vma_test_any_mask(vma, __VMA_UFFD_FLAGS);
@@ -371,6 +383,18 @@ static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, · 124 unmodified lines
371383 return false;
372384 }
373385
386+static inline bool userfaultfd_pte_rwp(struct vm_area_struct *vma,
387+ pte_t pte)
388+{
389+ return false;
390+}
391+
392+static inline bool userfaultfd_huge_pmd_rwp(struct vm_area_struct *vma,
393+ pmd_t pmd)
394+{
395+ return false;
396+}
397+
374398 static inline bool userfaultfd_armed(struct vm_area_struct *vma)
375399 {
376400 return false;
include/uapi/linux/userfaultfd.h+8−4
@@ -25,7 +25,8 @@ · 24 unmodified lines
2525 #define UFFD_API ((__u64)0xAA)
2626 #define UFFD_API_REGISTER_MODES (UFFDIO_REGISTER_MODE_MISSING | \
2727 UFFDIO_REGISTER_MODE_WP | \
28− UFFDIO_REGISTER_MODE_MINOR)
28+ UFFDIO_REGISTER_MODE_MINOR | \
29+ UFFDIO_REGISTER_MODE_RWP)
2930 #define UFFD_API_FEATURES (UFFD_FEATURE_PAGEFAULT_FLAG_WP | \
3031 UFFD_FEATURE_EVENT_FORK | \
3132 UFFD_FEATURE_EVENT_REMAP | \
@@ -42,7 +43,8 @@ · 10 unmodified lines
4243 UFFD_FEATURE_WP_UNPOPULATED | \
4344 UFFD_FEATURE_POISON | \
4445 UFFD_FEATURE_WP_ASYNC | \
45− UFFD_FEATURE_MOVE)
46+ UFFD_FEATURE_MOVE | \
47+ UFFD_FEATURE_RWP)
4648 #define UFFD_API_IOCTLS \
4749 ((__u64)1 << _UFFDIO_REGISTER | \
4850 (__u64)1 << _UFFDIO_UNREGISTER | \
@@ -54,13 +56,15 @@ · 5 unmodified lines
5456 (__u64)1 << _UFFDIO_MOVE | \
5557 (__u64)1 << _UFFDIO_WRITEPROTECT | \
5658 (__u64)1 << _UFFDIO_CONTINUE | \
57− (__u64)1 << _UFFDIO_POISON)
59+ (__u64)1 << _UFFDIO_POISON | \
60+ (__u64)1 << _UFFDIO_RWPROTECT)
5861 #define UFFD_API_RANGE_IOCTLS_BASIC \
5962 ((__u64)1 << _UFFDIO_WAKE | \
6063 (__u64)1 << _UFFDIO_COPY | \
6164 (__u64)1 << _UFFDIO_WRITEPROTECT | \
6265 (__u64)1 << _UFFDIO_CONTINUE | \
63− (__u64)1 << _UFFDIO_POISON)
66+ (__u64)1 << _UFFDIO_POISON | \
67+ (__u64)1 << _UFFDIO_RWPROTECT)
6468
6569 /*
6670 * Valid ioctl command number range with this API is from 0x00 to
mm/huge_memory.c+5−0
@@ -2202,6 +2202,11 @@ static inline bool can_change_pmd_writable(struct vm_area_struct *vma, · 2201 unmodified lines
22022202 return pmd_dirty(pmd);
22032203 }
22042204
2205+vm_fault_t do_huge_pmd_uffd_rwp(struct vm_fault *vmf)
2206+{
2207+ return handle_userfault(vmf, VM_UFFD_RWP);
2208+}
2209+
22052210 /* NUMA hinting page fault entry point for trans huge pmds */
22062211 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf)
22072212 {
mm/hugetlb.c+11−0
@@ -6095,6 +6095,17 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, · 6094 unmodified lines
60956095 goto out_mutex;
60966096 }
60976097
6098+ /*
6099+ * Protnone hugetlb PTEs with the uffd bit are used by
6100+ * userfaultfd RWP for access tracking. Plain PROT_NONE (without the
6101+ * marker) is not an RWP fault and is not expected on hugetlb (no
6102+ * NUMA hinting), so let normal hugetlb fault handling proceed.
6103+ */
6104+ if (pte_protnone(vmf.orig_pte) && vma_is_accessible(vma) &&
6105+ userfaultfd_rwp(vma) && huge_pte_uffd(vmf.orig_pte)) {
6106+ return hugetlb_handle_userfault(&vmf, mapping, VM_UFFD_RWP);
6107+ }
6108+
60986109 /*
60996110 * If we are going to COW/unshare the mapping later, we examine the
61006111 * pending reservations for this page now. This will ensure that any
mm/memory.c+29−2
@@ -6119,6 +6119,16 @@ static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_stru · 6118 unmodified lines
61196119 if (!pte_present(ptent) || !pte_protnone(ptent))
61206120 continue;
61216121
6122+ /*
6123+ * RWP-armed PTEs are also protnone but carry _PAGE_UFFD as a
6124+ * marker. Leave them alone -- rewriting to vm_page_prot would
6125+ * stop the RWP trap. Gate on userfaultfd_rwp(vma) too:
6126+ * NUMA balancing preserves _PAGE_UFFD on UFFD_WP-marked PTEs
6127+ * when applying PROT_NONE, and those still need rebuilding.
6128+ */
6129+ if (userfaultfd_rwp(vma) && pte_uffd(ptent))
6130+ continue;
6131+
61226132 if (pfn_folio(pte_pfn(ptent)) != folio)
61236133 continue;
61246134
@@ -6134,6 +6144,12 @@ static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_stru · 9 unmodified lines
61346144 }
61356145 }
61366146
6147+static vm_fault_t do_uffd_rwp(struct vm_fault *vmf)
6148+{
6149+ pte_unmap(vmf->pte);
6150+ return handle_userfault(vmf, VM_UFFD_RWP);
6151+}
6152+
61376153 static vm_fault_t do_numa_page(struct vm_fault *vmf)
61386154 {
61396155 struct vm_area_struct *vma = vmf->vma;
@@ -6409,8 +6425,16 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) · 269 unmodified lines
64096425 if (!pte_present(vmf->orig_pte))
64106426 return do_swap_page(vmf);
64116427
6412− if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma))
6428+ if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma)) {
6429+ /*
6430+ * RWP-protected PTEs are protnone plus the uffd bit. On a
6431+ * VM_UFFD_RWP VMA, a protnone PTE without the uffd bit is
6432+ * NUMA hinting and must still fall through to do_numa_page().
6433+ */
6434+ if (userfaultfd_pte_rwp(vmf->vma, vmf->orig_pte))
6435+ return do_uffd_rwp(vmf);
64136436 return do_numa_page(vmf);
6437+ }
64146438
64156439 spin_lock(vmf->ptl);
64166440 entry = vmf->orig_pte;
@@ -6524,8 +6548,11 @@ retry_pud: · 107 unmodified lines
65246548 return 0;
65256549 }
65266550 if (pmd_trans_huge(vmf.orig_pmd)) {
6527− if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma))
6551+ if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma)) {
6552+ if (userfaultfd_huge_pmd_rwp(vma, vmf.orig_pmd))
6553+ return do_huge_pmd_uffd_rwp(&vmf);
65286554 return do_huge_pmd_numa_page(&vmf);
6555+ }
65296556
65306557 if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) &&
65316558 !pmd_write(vmf.orig_pmd)) {
mm/userfaultfd.c+30−2
@@ -2669,6 +2669,12 @@ static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx, · 2668 unmodified lines
26692669 */
26702670 if (!huge_pte_write(pte) && (reason & VM_UFFD_WP))
26712671 return true;
2672+ /*
2673+ * PTE is still RW-protected (protnone with uffd bit), wait for
2674+ * resolution. Plain PROT_NONE without the marker is not an RWP fault.
2675+ */
2676+ if (pte_protnone(pte) && huge_pte_uffd(pte) && (reason & VM_UFFD_RWP))
2677+ return true;
26722678
26732679 return false;
26742680 }
@@ -2729,8 +2735,14 @@ again: · 54 unmodified lines
27292735 if (!pmd_present(_pmd))
27302736 return false;
27312737
2732− if (pmd_trans_huge(_pmd))
2733− return !pmd_write(_pmd) && (reason & VM_UFFD_WP);
2738+ if (pmd_trans_huge(_pmd)) {
2739+ if (!pmd_write(_pmd) && (reason & VM_UFFD_WP))
2740+ return true;
2741+ if (pmd_protnone(_pmd) && pmd_uffd(_pmd) &&
2742+ (reason & VM_UFFD_RWP))
2743+ return true;
2744+ return false;
2745+ }
27342746
27352747 pte = pte_offset_map(pmd, address);
27362748 if (!pte)
@@ -2766,6 +2778,13 @@ again: · 29 unmodified lines
27662778 */
27672779 if (!pte_write(ptent) && (reason & VM_UFFD_WP))
27682780 goto out;
2781+ /*
2782+ * PTE is still RW-protected (protnone with uffd bit), wait for
2783+ * userspace to resolve. Plain PROT_NONE without the marker is not
2784+ * an RWP fault.
2785+ */
2786+ if (pte_protnone(ptent) && pte_uffd(ptent) && (reason & VM_UFFD_RWP))
2787+ goto out;
27692788
27702789 ret = false;
27712790 out:
@@ -4518,6 +4537,15 @@ static int userfaultfd_api(struct userfaultfd_ctx *ctx, · 1746 unmodified lines
45184537 uffdio_api.features &= ~UFFD_FEATURE_WP_UNPOPULATED;
45194538 uffdio_api.features &= ~UFFD_FEATURE_WP_ASYNC;
45204539 }
4540+ /*
4541+ * RWP needs both PROT_NONE support and the uffd-wp PTE bit. The
4542+ * VM_UFFD_RWP check covers compile-time unavailability; the
4543+ * pgtable_supports_uffd() check covers runtime (e.g. riscv
4544+ * without the SVRSW60T59B extension) where the PTE bit is declared
4545+ * but not actually usable.
4546+ */
4547+ if (VM_UFFD_RWP == VM_NONE || !pgtable_supports_uffd())
4548+ uffdio_api.features &= ~UFFD_FEATURE_RWP;
45214549
45224550 ret = -EINVAL;
45234551 if (features & ~uffdio_api.features)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t