Wire the fault side of read-write protection tracking and turn the userspace interface on. An RWP-protected PTE is PAGE_NONE with the uffd bit set. The PROT_NONE triggers a fault on any access; the uffd bit distinguishes it from plain mprotect(PROT_NONE) or NUMA hinting. Fault dispatch, per level: PTE handle_pte_fault() -> do_uffd_rwp() PMD __handle_mm_fault() -> do_huge_pmd_uffd_rwp() hugetlb hugetlb_fault() -> hugetlb_handle_userfault() The RWP branches gate on userfaultfd_pte_rwp() / userfaultfd_huge_pmd_rwp() (VM_UFFD_RWP plus the uffd bit) and fall through to do_numa_page() / do_huge_pmd_numa_page() otherwise. Each delivers a UFFD_PAGEFAULT_FLAG_RWP message through handle_userfault(); the handler resolves it with UFFDIO_RWPROTECT clearing MODE_RWP. userfaultfd_must_wait() and userfaultfd_huge_must_wait() add matching protnone+uffd waiters so sync-mode fault handlers block correctly. Expose the UAPI: UFFDIO_REGISTER_MODE_RWP -> UFFD_API_REGISTER_MODES UFFD_FEATURE_RWP -> UFFD_API_FEATURES _UFFDIO_RWPROTECT -> UFFD_API_RANGE_IOCTLS UFFD_API_RANGE_IOCTLS_BASIC UFFD_FEATURE_RWP is masked out at UFFDIO_API time when PROT_NONE is not available or VM_UFFD_RWP aliases VM_NONE (32-bit), so userspace never sees an advertised-but-broken feature. Works on anonymous, shmem, and hugetlb memory.
Unified · Split
| @@ -529,6 +529,8 @@ static inline bool folio_test_pmd_mappable(struct folio *folio) · 528 unmodified lines | |||
| 529 | 529 | ||
| 530 | vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf); | 530 | vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf); |
| 531 | 531 | ||
| 532 | vm_fault_t do_huge_pmd_uffd_rwp(struct vm_fault *vmf); | ||
| 533 | |||
| 532 | vm_fault_t do_huge_pmd_device_private(struct vm_fault *vmf); | 534 | vm_fault_t do_huge_pmd_device_private(struct vm_fault *vmf); |
| 533 | 535 | ||
| 534 | extern struct folio *huge_zero_folio; | 536 | extern struct folio *huge_zero_folio; |
| @@ -723,6 +725,11 @@ static inline spinlock_t *pud_trans_huge_lock(pud_t *pud, · 188 unmodified lines | |||
| 723 | return NULL; | 725 | return NULL; |
| 724 | } | 726 | } |
| 725 | 727 | ||
| 728 | static inline vm_fault_t do_huge_pmd_uffd_rwp(struct vm_fault *vmf) | ||
| 729 | { | ||
| 730 | return 0; | ||
| 731 | } | ||
| 732 | |||
| 726 | static inline vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) | 733 | static inline vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) |
| 727 | { | 734 | { |
| 728 | return 0; | 735 | return 0; |
| @@ -241,6 +241,18 @@ static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, · 240 unmodified lines | |||
| 241 | return userfaultfd_wp(vma) && pmd_uffd(pmd); | 241 | return userfaultfd_wp(vma) && pmd_uffd(pmd); |
| 242 | } | 242 | } |
| 243 | 243 | ||
| 244 | static inline bool userfaultfd_pte_rwp(struct vm_area_struct *vma, | ||
| 245 | pte_t pte) | ||
| 246 | { | ||
| 247 | return userfaultfd_rwp(vma) && pte_uffd(pte); | ||
| 248 | } | ||
| 249 | |||
| 250 | static inline bool userfaultfd_huge_pmd_rwp(struct vm_area_struct *vma, | ||
| 251 | pmd_t pmd) | ||
| 252 | { | ||
| 253 | return userfaultfd_rwp(vma) && pmd_uffd(pmd); | ||
| 254 | } | ||
| 255 | |||
| 244 | static inline bool userfaultfd_armed(struct vm_area_struct *vma) | 256 | static inline bool userfaultfd_armed(struct vm_area_struct *vma) |
| 245 | { | 257 | { |
| 246 | return vma_test_any_mask(vma, __VMA_UFFD_FLAGS); | 258 | return vma_test_any_mask(vma, __VMA_UFFD_FLAGS); |
| @@ -371,6 +383,18 @@ static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, · 124 unmodified lines | |||
| 371 | return false; | 383 | return false; |
| 372 | } | 384 | } |
| 373 | 385 | ||
| 386 | static inline bool userfaultfd_pte_rwp(struct vm_area_struct *vma, | ||
| 387 | pte_t pte) | ||
| 388 | { | ||
| 389 | return false; | ||
| 390 | } | ||
| 391 | |||
| 392 | static inline bool userfaultfd_huge_pmd_rwp(struct vm_area_struct *vma, | ||
| 393 | pmd_t pmd) | ||
| 394 | { | ||
| 395 | return false; | ||
| 396 | } | ||
| 397 | |||
| 374 | static inline bool userfaultfd_armed(struct vm_area_struct *vma) | 398 | static inline bool userfaultfd_armed(struct vm_area_struct *vma) |
| 375 | { | 399 | { |
| 376 | return false; | 400 | return false; |
| @@ -25,7 +25,8 @@ · 24 unmodified lines | |||
| 25 | #define UFFD_API ((__u64)0xAA) | 25 | #define UFFD_API ((__u64)0xAA) |
| 26 | #define UFFD_API_REGISTER_MODES (UFFDIO_REGISTER_MODE_MISSING | \ | 26 | #define UFFD_API_REGISTER_MODES (UFFDIO_REGISTER_MODE_MISSING | \ |
| 27 | UFFDIO_REGISTER_MODE_WP | \ | 27 | UFFDIO_REGISTER_MODE_WP | \ |
| 28 | UFFDIO_REGISTER_MODE_MINOR) | 28 | UFFDIO_REGISTER_MODE_MINOR | \ |
| 29 | UFFDIO_REGISTER_MODE_RWP) | ||
| 29 | #define UFFD_API_FEATURES (UFFD_FEATURE_PAGEFAULT_FLAG_WP | \ | 30 | #define UFFD_API_FEATURES (UFFD_FEATURE_PAGEFAULT_FLAG_WP | \ |
| 30 | UFFD_FEATURE_EVENT_FORK | \ | 31 | UFFD_FEATURE_EVENT_FORK | \ |
| 31 | UFFD_FEATURE_EVENT_REMAP | \ | 32 | UFFD_FEATURE_EVENT_REMAP | \ |
| @@ -42,7 +43,8 @@ · 10 unmodified lines | |||
| 42 | UFFD_FEATURE_WP_UNPOPULATED | \ | 43 | UFFD_FEATURE_WP_UNPOPULATED | \ |
| 43 | UFFD_FEATURE_POISON | \ | 44 | UFFD_FEATURE_POISON | \ |
| 44 | UFFD_FEATURE_WP_ASYNC | \ | 45 | UFFD_FEATURE_WP_ASYNC | \ |
| 45 | UFFD_FEATURE_MOVE) | 46 | UFFD_FEATURE_MOVE | \ |
| 47 | UFFD_FEATURE_RWP) | ||
| 46 | #define UFFD_API_IOCTLS \ | 48 | #define UFFD_API_IOCTLS \ |
| 47 | ((__u64)1 << _UFFDIO_REGISTER | \ | 49 | ((__u64)1 << _UFFDIO_REGISTER | \ |
| 48 | (__u64)1 << _UFFDIO_UNREGISTER | \ | 50 | (__u64)1 << _UFFDIO_UNREGISTER | \ |
| @@ -54,13 +56,15 @@ · 5 unmodified lines | |||
| 54 | (__u64)1 << _UFFDIO_MOVE | \ | 56 | (__u64)1 << _UFFDIO_MOVE | \ |
| 55 | (__u64)1 << _UFFDIO_WRITEPROTECT | \ | 57 | (__u64)1 << _UFFDIO_WRITEPROTECT | \ |
| 56 | (__u64)1 << _UFFDIO_CONTINUE | \ | 58 | (__u64)1 << _UFFDIO_CONTINUE | \ |
| 57 | (__u64)1 << _UFFDIO_POISON) | 59 | (__u64)1 << _UFFDIO_POISON | \ |
| 60 | (__u64)1 << _UFFDIO_RWPROTECT) | ||
| 58 | #define UFFD_API_RANGE_IOCTLS_BASIC \ | 61 | #define UFFD_API_RANGE_IOCTLS_BASIC \ |
| 59 | ((__u64)1 << _UFFDIO_WAKE | \ | 62 | ((__u64)1 << _UFFDIO_WAKE | \ |
| 60 | (__u64)1 << _UFFDIO_COPY | \ | 63 | (__u64)1 << _UFFDIO_COPY | \ |
| 61 | (__u64)1 << _UFFDIO_WRITEPROTECT | \ | 64 | (__u64)1 << _UFFDIO_WRITEPROTECT | \ |
| 62 | (__u64)1 << _UFFDIO_CONTINUE | \ | 65 | (__u64)1 << _UFFDIO_CONTINUE | \ |
| 63 | (__u64)1 << _UFFDIO_POISON) | 66 | (__u64)1 << _UFFDIO_POISON | \ |
| 67 | (__u64)1 << _UFFDIO_RWPROTECT) | ||
| 64 | 68 | ||
| 65 | /* | 69 | /* |
| 66 | * Valid ioctl command number range with this API is from 0x00 to | 70 | * Valid ioctl command number range with this API is from 0x00 to |
| @@ -2202,6 +2202,11 @@ static inline bool can_change_pmd_writable(struct vm_area_struct *vma, · 2201 unmodified lines | |||
| 2202 | return pmd_dirty(pmd); | 2202 | return pmd_dirty(pmd); |
| 2203 | } | 2203 | } |
| 2204 | 2204 | ||
| 2205 | vm_fault_t do_huge_pmd_uffd_rwp(struct vm_fault *vmf) | ||
| 2206 | { | ||
| 2207 | return handle_userfault(vmf, VM_UFFD_RWP); | ||
| 2208 | } | ||
| 2209 | |||
| 2205 | /* NUMA hinting page fault entry point for trans huge pmds */ | 2210 | /* NUMA hinting page fault entry point for trans huge pmds */ |
| 2206 | vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) | 2211 | vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) |
| 2207 | { | 2212 | { |
| @@ -6095,6 +6095,17 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, · 6094 unmodified lines | |||
| 6095 | goto out_mutex; | 6095 | goto out_mutex; |
| 6096 | } | 6096 | } |
| 6097 | 6097 | ||
| 6098 | /* | ||
| 6099 | * Protnone hugetlb PTEs with the uffd bit are used by | ||
| 6100 | * userfaultfd RWP for access tracking. Plain PROT_NONE (without the | ||
| 6101 | * marker) is not an RWP fault and is not expected on hugetlb (no | ||
| 6102 | * NUMA hinting), so let normal hugetlb fault handling proceed. | ||
| 6103 | */ | ||
| 6104 | if (pte_protnone(vmf.orig_pte) && vma_is_accessible(vma) && | ||
| 6105 | userfaultfd_rwp(vma) && huge_pte_uffd(vmf.orig_pte)) { | ||
| 6106 | return hugetlb_handle_userfault(&vmf, mapping, VM_UFFD_RWP); | ||
| 6107 | } | ||
| 6108 | |||
| 6098 | /* | 6109 | /* |
| 6099 | * If we are going to COW/unshare the mapping later, we examine the | 6110 | * If we are going to COW/unshare the mapping later, we examine the |
| 6100 | * pending reservations for this page now. This will ensure that any | 6111 | * pending reservations for this page now. This will ensure that any |
| @@ -6119,6 +6119,16 @@ static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_stru · 6118 unmodified lines | |||
| 6119 | if (!pte_present(ptent) || !pte_protnone(ptent)) | 6119 | if (!pte_present(ptent) || !pte_protnone(ptent)) |
| 6120 | continue; | 6120 | continue; |
| 6121 | 6121 | ||
| 6122 | /* | ||
| 6123 | * RWP-armed PTEs are also protnone but carry _PAGE_UFFD as a | ||
| 6124 | * marker. Leave them alone -- rewriting to vm_page_prot would | ||
| 6125 | * stop the RWP trap. Gate on userfaultfd_rwp(vma) too: | ||
| 6126 | * NUMA balancing preserves _PAGE_UFFD on UFFD_WP-marked PTEs | ||
| 6127 | * when applying PROT_NONE, and those still need rebuilding. | ||
| 6128 | */ | ||
| 6129 | if (userfaultfd_rwp(vma) && pte_uffd(ptent)) | ||
| 6130 | continue; | ||
| 6131 | |||
| 6122 | if (pfn_folio(pte_pfn(ptent)) != folio) | 6132 | if (pfn_folio(pte_pfn(ptent)) != folio) |
| 6123 | continue; | 6133 | continue; |
| 6124 | 6134 | ||
| @@ -6134,6 +6144,12 @@ static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_stru · 9 unmodified lines | |||
| 6134 | } | 6144 | } |
| 6135 | } | 6145 | } |
| 6136 | 6146 | ||
| 6147 | static vm_fault_t do_uffd_rwp(struct vm_fault *vmf) | ||
| 6148 | { | ||
| 6149 | pte_unmap(vmf->pte); | ||
| 6150 | return handle_userfault(vmf, VM_UFFD_RWP); | ||
| 6151 | } | ||
| 6152 | |||
| 6137 | static vm_fault_t do_numa_page(struct vm_fault *vmf) | 6153 | static vm_fault_t do_numa_page(struct vm_fault *vmf) |
| 6138 | { | 6154 | { |
| 6139 | struct vm_area_struct *vma = vmf->vma; | 6155 | struct vm_area_struct *vma = vmf->vma; |
| @@ -6409,8 +6425,16 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) · 269 unmodified lines | |||
| 6409 | if (!pte_present(vmf->orig_pte)) | 6425 | if (!pte_present(vmf->orig_pte)) |
| 6410 | return do_swap_page(vmf); | 6426 | return do_swap_page(vmf); |
| 6411 | 6427 | ||
| 6412 | if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma)) | 6428 | if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma)) { |
| 6429 | /* | ||
| 6430 | * RWP-protected PTEs are protnone plus the uffd bit. On a | ||
| 6431 | * VM_UFFD_RWP VMA, a protnone PTE without the uffd bit is | ||
| 6432 | * NUMA hinting and must still fall through to do_numa_page(). | ||
| 6433 | */ | ||
| 6434 | if (userfaultfd_pte_rwp(vmf->vma, vmf->orig_pte)) | ||
| 6435 | return do_uffd_rwp(vmf); | ||
| 6413 | return do_numa_page(vmf); | 6436 | return do_numa_page(vmf); |
| 6437 | } | ||
| 6414 | 6438 | ||
| 6415 | spin_lock(vmf->ptl); | 6439 | spin_lock(vmf->ptl); |
| 6416 | entry = vmf->orig_pte; | 6440 | entry = vmf->orig_pte; |
| @@ -6524,8 +6548,11 @@ retry_pud: · 107 unmodified lines | |||
| 6524 | return 0; | 6548 | return 0; |
| 6525 | } | 6549 | } |
| 6526 | if (pmd_trans_huge(vmf.orig_pmd)) { | 6550 | if (pmd_trans_huge(vmf.orig_pmd)) { |
| 6527 | if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma)) | 6551 | if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma)) { |
| 6552 | if (userfaultfd_huge_pmd_rwp(vma, vmf.orig_pmd)) | ||
| 6553 | return do_huge_pmd_uffd_rwp(&vmf); | ||
| 6528 | return do_huge_pmd_numa_page(&vmf); | 6554 | return do_huge_pmd_numa_page(&vmf); |
| 6555 | } | ||
| 6529 | 6556 | ||
| 6530 | if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) && | 6557 | if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) && |
| 6531 | !pmd_write(vmf.orig_pmd)) { | 6558 | !pmd_write(vmf.orig_pmd)) { |
| @@ -2669,6 +2669,12 @@ static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx, · 2668 unmodified lines | |||
| 2669 | */ | 2669 | */ |
| 2670 | if (!huge_pte_write(pte) && (reason & VM_UFFD_WP)) | 2670 | if (!huge_pte_write(pte) && (reason & VM_UFFD_WP)) |
| 2671 | return true; | 2671 | return true; |
| 2672 | /* | ||
| 2673 | * PTE is still RW-protected (protnone with uffd bit), wait for | ||
| 2674 | * resolution. Plain PROT_NONE without the marker is not an RWP fault. | ||
| 2675 | */ | ||
| 2676 | if (pte_protnone(pte) && huge_pte_uffd(pte) && (reason & VM_UFFD_RWP)) | ||
| 2677 | return true; | ||
| 2672 | 2678 | ||
| 2673 | return false; | 2679 | return false; |
| 2674 | } | 2680 | } |
| @@ -2729,8 +2735,14 @@ again: · 54 unmodified lines | |||
| 2729 | if (!pmd_present(_pmd)) | 2735 | if (!pmd_present(_pmd)) |
| 2730 | return false; | 2736 | return false; |
| 2731 | 2737 | ||
| 2732 | if (pmd_trans_huge(_pmd)) | 2738 | if (pmd_trans_huge(_pmd)) { |
| 2733 | return !pmd_write(_pmd) && (reason & VM_UFFD_WP); | 2739 | if (!pmd_write(_pmd) && (reason & VM_UFFD_WP)) |
| 2740 | return true; | ||
| 2741 | if (pmd_protnone(_pmd) && pmd_uffd(_pmd) && | ||
| 2742 | (reason & VM_UFFD_RWP)) | ||
| 2743 | return true; | ||
| 2744 | return false; | ||
| 2745 | } | ||
| 2734 | 2746 | ||
| 2735 | pte = pte_offset_map(pmd, address); | 2747 | pte = pte_offset_map(pmd, address); |
| 2736 | if (!pte) | 2748 | if (!pte) |
| @@ -2766,6 +2778,13 @@ again: · 29 unmodified lines | |||
| 2766 | */ | 2778 | */ |
| 2767 | if (!pte_write(ptent) && (reason & VM_UFFD_WP)) | 2779 | if (!pte_write(ptent) && (reason & VM_UFFD_WP)) |
| 2768 | goto out; | 2780 | goto out; |
| 2781 | /* | ||
| 2782 | * PTE is still RW-protected (protnone with uffd bit), wait for | ||
| 2783 | * userspace to resolve. Plain PROT_NONE without the marker is not | ||
| 2784 | * an RWP fault. | ||
| 2785 | */ | ||
| 2786 | if (pte_protnone(ptent) && pte_uffd(ptent) && (reason & VM_UFFD_RWP)) | ||
| 2787 | goto out; | ||
| 2769 | 2788 | ||
| 2770 | ret = false; | 2789 | ret = false; |
| 2771 | out: | 2790 | out: |
| @@ -4518,6 +4537,15 @@ static int userfaultfd_api(struct userfaultfd_ctx *ctx, · 1746 unmodified lines | |||
| 4518 | uffdio_api.features &= ~UFFD_FEATURE_WP_UNPOPULATED; | 4537 | uffdio_api.features &= ~UFFD_FEATURE_WP_UNPOPULATED; |
| 4519 | uffdio_api.features &= ~UFFD_FEATURE_WP_ASYNC; | 4538 | uffdio_api.features &= ~UFFD_FEATURE_WP_ASYNC; |
| 4520 | } | 4539 | } |
| 4540 | /* | ||
| 4541 | * RWP needs both PROT_NONE support and the uffd-wp PTE bit. The | ||
| 4542 | * VM_UFFD_RWP check covers compile-time unavailability; the | ||
| 4543 | * pgtable_supports_uffd() check covers runtime (e.g. riscv | ||
| 4544 | * without the SVRSW60T59B extension) where the PTE bit is declared | ||
| 4545 | * but not actually usable. | ||
| 4546 | */ | ||
| 4547 | if (VM_UFFD_RWP == VM_NONE || !pgtable_supports_uffd()) | ||
| 4548 | uffdio_api.features &= ~UFFD_FEATURE_RWP; | ||
| 4521 | 4549 | ||
| 4522 | ret = -EINVAL; | 4550 | ret = -EINVAL; |
| 4523 | if (features & ~uffdio_api.features) | 4551 | if (features & ~uffdio_api.features) |