History

mm/pagemap: add PAGE_IS_ACCESSED for RWP tracking

PAGEMAP_SCAN already reports PAGE_IS_WRITTEN from the inverted uffd PTE bit, targeting the UFFDIO_WRITEPROTECT workflow. UFFDIO_RWPROTECT reuses the same PTE bit as a marker for read-write protection, but "has been written" and "has been accessed" are distinct semantic signals — they happen to share one PTE bit today only because the two implementations share infrastructure. Give RWP its own pagemap category so the UAPI does not conflate them: PAGE_IS_WRITTEN reported on VM_UFFD_WP VMAs, !pte_uffd(pte) PAGE_IS_ACCESSED reported on VM_UFFD_RWP VMAs, !pte_uffd(pte) Both still read the same PTE bit today, but each is scoped to the VMA whose registered mode makes the bit meaningful. If a future implementation moves RWP to a separate PTE bit, only PAGE_IS_ACCESSED switches over. This is a UAPI narrowing. Outside VM_UFFD_WP VMAs the uffd bit is always clear, so PAGEMAP_SCAN used to flag PAGE_IS_WRITTEN on every present PTE there — a meaningless duplicate of PAGE_IS_PRESENT. Now PAGE_IS_WRITTEN fires only inside VM_UFFD_WP VMAs. pagemap_hugetlb_category() now takes the vma like its PTE/PMD peers.

Kiryl Shutsemau (Meta) committed 4 months ago

Commit 33896481 parent 2d427bb4 files · +67 −29Browse files

Unified · Split

4 files
  1. Documentation/admin-guide/mm/pagemap.rst +10 −3
  2. fs/proc/task_mmu.c +55 −26
  3. include/uapi/linux/fs.h +1 −0
  4. tools/include/uapi/linux/fs.h +1 −0
Documentation/admin-guide/mm/pagemap.rst+10−3
@@ -19,8 +19,11 @@ There are four components to pagemap: · 18 unmodified lines
1919 * Bit 55 pte is soft-dirty (see
2020 Documentation/admin-guide/mm/soft-dirty.rst)
2121 * Bit 56 page exclusively mapped (since 4.2)
22− * Bit 57 pte is uffd-wp write-protected (since 5.13) (see
23− Documentation/admin-guide/mm/userfaultfd.rst)
22+ * Bit 57 pte is tracked by userfaultfd (since 5.13) — in a
23+ ``VM_UFFD_WP`` VMA this indicates a write-protected PTE; in a
24+ ``VM_UFFD_RWP`` VMA it indicates an RWP-protected PTE. WP and
25+ RWP are mutually exclusive per VMA, so the meaning is
26+ unambiguous. See Documentation/admin-guide/mm/userfaultfd.rst.
2427 * Bit 58 pte is a guard region (since 6.15) (see madvise (2) man page)
2528 * Bits 59-60 zero
2629 * Bit 61 page is file-page or shared-anon (since 3.5)
@@ -244,7 +247,8 @@ in this IOCTL: · 217 unmodified lines
244247 Following flags about pages are currently supported:
245248
246249 - ``PAGE_IS_WPALLOWED`` - Page has async-write-protection enabled
247−- ``PAGE_IS_WRITTEN`` - Page has been written to from the time it was write protected
250+- ``PAGE_IS_WRITTEN`` - Page in a ``UFFDIO_REGISTER_MODE_WP`` VMA has been
251+ written to since it was write-protected. Only reported inside such VMAs.
248252 - ``PAGE_IS_FILE`` - Page is file backed
249253 - ``PAGE_IS_PRESENT`` - Page is present in the memory
250254 - ``PAGE_IS_SWAPPED`` - Page is in swapped
@@ -252,6 +256,9 @@ Following flags about pages are currently supported: · 1 unmodified line
252256 - ``PAGE_IS_HUGE`` - Page is PMD-mapped THP or Hugetlb backed
253257 - ``PAGE_IS_SOFT_DIRTY`` - Page is soft-dirty
254258 - ``PAGE_IS_GUARD`` - Page is a part of a guard region
259+- ``PAGE_IS_ACCESSED`` - Page in a ``UFFDIO_REGISTER_MODE_RWP`` VMA has been
260+ accessed since RWP was applied. Only reported inside such VMAs. See
261+ Documentation/admin-guide/mm/userfaultfd.rst for the RWP workflow.
255262
256263 The ``struct pm_scan_arg`` is used as the argument of the IOCTL.
257264
fs/proc/task_mmu.c+55−26
@@ -2284,7 +2284,7 @@ static const struct mm_walk_ops pagemap_ops = { · 2283 unmodified lines
22842284 * Bits 5-54 swap offset if swapped
22852285 * Bit 55 pte is soft-dirty (see Documentation/admin-guide/mm/soft-dirty.rst)
22862286 * Bit 56 page exclusively mapped
2287− * Bit 57 pte is uffd-wp write-protected
2287+ * Bit 57 pte is tracked by userfaultfd (uffd-wp or RWP)
22882288 * Bit 58 pte is a guard region
22892289 * Bits 59-60 zero
22902290 * Bit 61 page is file-page or shared-anon
@@ -2419,7 +2419,7 @@ static int pagemap_release(struct inode *inode, struct file *file) · 128 unmodified lines
24192419 PAGE_IS_FILE | PAGE_IS_PRESENT | \
24202420 PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | \
24212421 PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY | \
2422− PAGE_IS_GUARD)
2422+ PAGE_IS_GUARD | PAGE_IS_ACCESSED)
24232423 #define PM_SCAN_FLAGS (PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC)
24242424
24252425 struct pagemap_scan_private {
@@ -2438,15 +2438,17 @@ static unsigned long pagemap_page_category(struct pagemap_scan_private *p, · 12 unmodified lines
24382438
24392439 if (pte_none(pte)) {
24402440 /*
2441− * An unpopulated pte carries no uffd-wp marker, i.e. it is not
2442− * write-protected, the same condition under which the present
2443− * and swap cases below report PAGE_IS_WRITTEN. Report it here
2444− * too so this generic path agrees with the PAGE_IS_WRITTEN fast
2445− * path in pagemap_scan_pmd_entry(), which reports pte_none as
2446− * written and, under PM_SCAN_WP_MATCHING, arms a marker. The
2447− * fast path applies no VMA test, so neither does this.
2441+ * An unpopulated pte carries no uffd bit, i.e. it is not
2442+ * write-protected. The PAGE_IS_WRITTEN fast path in
2443+ * pagemap_scan_pmd_entry() is now gated on a VM_UFFD_WP VMA;
2444+ * gate the pte_none report here the same way so the two paths
2445+ * still agree. RWP has no such fast path and an unpopulated
2446+ * page is not part of the RWP working set, so it is reported as
2447+ * neither.
24482448 */
2449− return PAGE_IS_WRITTEN;
2449+ if (userfaultfd_wp(vma))
2450+ return PAGE_IS_WRITTEN;
2451+ return 0;
24502452 }
24512453
24522454 if (pte_present(pte)) {
@@ -2454,8 +2456,12 @@ static unsigned long pagemap_page_category(struct pagemap_scan_private *p, · 1 unmodified line
24542456
24552457 categories = PAGE_IS_PRESENT;
24562458
2457− if (!pte_uffd(pte))
2458− categories |= PAGE_IS_WRITTEN;
2459+ if (!pte_uffd(pte)) {
2460+ if (userfaultfd_wp(vma))
2461+ categories |= PAGE_IS_WRITTEN;
2462+ if (userfaultfd_rwp(vma))
2463+ categories |= PAGE_IS_ACCESSED;
2464+ }
24592465
24602466 if (p->masks_of_interest & PAGE_IS_FILE) {
24612467 page = vm_normal_page(vma, addr, pte);
@@ -2472,8 +2478,12 @@ static unsigned long pagemap_page_category(struct pagemap_scan_private *p, · 10 unmodified lines
24722478
24732479 categories = PAGE_IS_SWAPPED;
24742480
2475− if (!pte_swp_uffd_any(pte))
2476− categories |= PAGE_IS_WRITTEN;
2481+ if (!pte_swp_uffd_any(pte)) {
2482+ if (userfaultfd_wp(vma))
2483+ categories |= PAGE_IS_WRITTEN;
2484+ if (userfaultfd_rwp(vma))
2485+ categories |= PAGE_IS_ACCESSED;
2486+ }
24772487
24782488 entry = softleaf_from_pte(pte);
24792489 if (softleaf_is_guard_marker(entry))
@@ -2522,8 +2532,12 @@ static unsigned long pagemap_thp_category(struct pagemap_scan_private *p, · 42 unmodified lines
25222532 struct page *page;
25232533
25242534 categories |= PAGE_IS_PRESENT;
2525− if (!pmd_uffd(pmd))
2526− categories |= PAGE_IS_WRITTEN;
2535+ if (!pmd_uffd(pmd)) {
2536+ if (userfaultfd_wp(vma))
2537+ categories |= PAGE_IS_WRITTEN;
2538+ if (userfaultfd_rwp(vma))
2539+ categories |= PAGE_IS_ACCESSED;
2540+ }
25272541
25282542 if (p->masks_of_interest & PAGE_IS_FILE) {
25292543 page = vm_normal_page_pmd(vma, addr, pmd);
@@ -2537,8 +2551,12 @@ static unsigned long pagemap_thp_category(struct pagemap_scan_private *p, · 7 unmodified lines
25372551 categories |= PAGE_IS_SOFT_DIRTY;
25382552 } else {
25392553 categories |= PAGE_IS_SWAPPED;
2540− if (!pmd_swp_uffd(pmd))
2541− categories |= PAGE_IS_WRITTEN;
2554+ if (!pmd_swp_uffd(pmd)) {
2555+ if (userfaultfd_wp(vma))
2556+ categories |= PAGE_IS_WRITTEN;
2557+ if (userfaultfd_rwp(vma))
2558+ categories |= PAGE_IS_ACCESSED;
2559+ }
25422560 if (pmd_swp_soft_dirty(pmd))
25432561 categories |= PAGE_IS_SOFT_DIRTY;
25442562
@@ -2571,7 +2589,8 @@ static void make_uffd_wp_pmd(struct vm_area_struct *vma, · 26 unmodified lines
25712589 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
25722590
25732591 #ifdef CONFIG_HUGETLB_PAGE
2574−static unsigned long pagemap_hugetlb_category(pte_t pte)
2592+static unsigned long pagemap_hugetlb_category(struct vm_area_struct *vma,
2593+ pte_t pte)
25752594 {
25762595 unsigned long categories = PAGE_IS_HUGE;
25772596
@@ -2586,8 +2605,12 @@ static unsigned long pagemap_hugetlb_category(pte_t pte) · 8 unmodified lines
25862605 if (pte_present(pte)) {
25872606 categories |= PAGE_IS_PRESENT;
25882607
2589− if (!huge_pte_uffd(pte))
2590− categories |= PAGE_IS_WRITTEN;
2608+ if (!huge_pte_uffd(pte)) {
2609+ if (userfaultfd_wp(vma))
2610+ categories |= PAGE_IS_WRITTEN;
2611+ if (userfaultfd_rwp(vma))
2612+ categories |= PAGE_IS_ACCESSED;
2613+ }
25912614 if (!PageAnon(pte_page(pte)))
25922615 categories |= PAGE_IS_FILE;
25932616 if (is_zero_pfn(pte_pfn(pte)))
@@ -2597,8 +2620,12 @@ static unsigned long pagemap_hugetlb_category(pte_t pte) · 3 unmodified lines
25972620 } else {
25982621 categories |= PAGE_IS_SWAPPED;
25992622
2600− if (!pte_swp_uffd_any(pte))
2601− categories |= PAGE_IS_WRITTEN;
2623+ if (!pte_swp_uffd_any(pte)) {
2624+ if (userfaultfd_wp(vma))
2625+ categories |= PAGE_IS_WRITTEN;
2626+ if (userfaultfd_rwp(vma))
2627+ categories |= PAGE_IS_ACCESSED;
2628+ }
26022629 if (pte_swp_soft_dirty(pte))
26032630 categories |= PAGE_IS_SOFT_DIRTY;
26042631 }
@@ -2874,7 +2901,8 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, · 269 unmodified lines
28742901 goto flush_and_return;
28752902 }
28762903
2877− if (!p->arg.category_anyof_mask && !p->arg.category_inverted &&
2904+ if (userfaultfd_wp(vma) && !p->arg.category_anyof_mask &&
2905+ !p->arg.category_inverted &&
28782906 p->arg.category_mask == PAGE_IS_WRITTEN &&
28792907 p->arg.return_mask == PAGE_IS_WRITTEN) {
28802908 for (addr = start; addr < end; pte++, addr += PAGE_SIZE) {
@@ -2949,7 +2977,8 @@ static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask, · 68 unmodified lines
29492977 /* Go the short route when not write-protecting pages. */
29502978
29512979 pte = huge_ptep_get(walk->mm, start, ptep);
2952− categories = p->cur_vma_category | pagemap_hugetlb_category(pte);
2980+ categories = p->cur_vma_category |
2981+ pagemap_hugetlb_category(vma, pte);
29532982
29542983 if (!pagemap_scan_is_interesting_page(categories, p))
29552984 return 0;
@@ -2961,7 +2990,7 @@ static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask, · 5 unmodified lines
29612990 ptl = huge_pte_lock(hstate_vma(vma), vma->vm_mm, ptep);
29622991
29632992 pte = huge_ptep_get(walk->mm, start, ptep);
2964− categories = p->cur_vma_category | pagemap_hugetlb_category(pte);
2993+ categories = p->cur_vma_category | pagemap_hugetlb_category(vma, pte);
29652994
29662995 if (!pagemap_scan_is_interesting_page(categories, p))
29672996 goto out_unlock;
include/uapi/linux/fs.h+1−0
@@ -471,6 +471,7 @@ typedef int __bitwise __kernel_rwf_t; · 470 unmodified lines
471471 #define PAGE_IS_HUGE (1 << 6)
472472 #define PAGE_IS_SOFT_DIRTY (1 << 7)
473473 #define PAGE_IS_GUARD (1 << 8)
474+#define PAGE_IS_ACCESSED (1 << 9)
474475
475476 /*
476477 * struct page_region - Page region with flags
tools/include/uapi/linux/fs.h+1−0
@@ -364,6 +364,7 @@ typedef int __bitwise __kernel_rwf_t; · 363 unmodified lines
364364 #define PAGE_IS_HUGE (1 << 6)
365365 #define PAGE_IS_SOFT_DIRTY (1 << 7)
366366 #define PAGE_IS_GUARD (1 << 8)
367+#define PAGE_IS_ACCESSED (1 << 9)
367368
368369 /*
369370 * struct page_region - Page region with flags