History

mm/pagemap: add PAGE_IS_ACCESSED for RWP tracking

PAGEMAP_SCAN already reports PAGE_IS_WRITTEN from the inverted uffd PTE bit, targeting the UFFDIO_WRITEPROTECT workflow. UFFDIO_RWPROTECT reuses the same PTE bit as a marker for read-write protection, but "has been written" and "has been accessed" are distinct semantic signals — they happen to share one PTE bit today only because the two implementations share infrastructure. Give RWP its own pagemap category so the UAPI does not conflate them: PAGE_IS_WRITTEN reported on VM_UFFD_WP VMAs, !pte_uffd(pte) PAGE_IS_ACCESSED reported on VM_UFFD_RWP VMAs, !pte_uffd(pte) Both still read the same PTE bit today, but each is scoped to the VMA whose registered mode makes the bit meaningful. If a future implementation moves RWP to a separate PTE bit, only PAGE_IS_ACCESSED switches over. This is a UAPI narrowing. Outside VM_UFFD_WP VMAs the uffd bit is always clear, so PAGEMAP_SCAN used to flag PAGE_IS_WRITTEN on every present PTE there — a meaningless duplicate of PAGE_IS_PRESENT. Now PAGE_IS_WRITTEN fires only inside VM_UFFD_WP VMAs. pagemap_hugetlb_category() now takes the vma like its PTE/PMD peers.

Kiryl Shutsemau (Meta) committed 4 months ago

Commit 33896481 parent 2d427bb4 files · +67 −29Browse files

Showing the changes to fs/proc (1 of 4 files). Show all

Unified · Split

fs/proc/task_mmu.c+55−26
@@ -2284,7 +2284,7 @@ static const struct mm_walk_ops pagemap_ops = { · 2283 unmodified lines
22842284 * Bits 5-54 swap offset if swapped
22852285 * Bit 55 pte is soft-dirty (see Documentation/admin-guide/mm/soft-dirty.rst)
22862286 * Bit 56 page exclusively mapped
2287− * Bit 57 pte is uffd-wp write-protected
2287+ * Bit 57 pte is tracked by userfaultfd (uffd-wp or RWP)
22882288 * Bit 58 pte is a guard region
22892289 * Bits 59-60 zero
22902290 * Bit 61 page is file-page or shared-anon
@@ -2419,7 +2419,7 @@ static int pagemap_release(struct inode *inode, struct file *file) · 128 unmodified lines
24192419 PAGE_IS_FILE | PAGE_IS_PRESENT | \
24202420 PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | \
24212421 PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY | \
2422− PAGE_IS_GUARD)
2422+ PAGE_IS_GUARD | PAGE_IS_ACCESSED)
24232423 #define PM_SCAN_FLAGS (PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC)
24242424
24252425 struct pagemap_scan_private {
@@ -2438,15 +2438,17 @@ static unsigned long pagemap_page_category(struct pagemap_scan_private *p, · 12 unmodified lines
24382438
24392439 if (pte_none(pte)) {
24402440 /*
2441− * An unpopulated pte carries no uffd-wp marker, i.e. it is not
2442− * write-protected, the same condition under which the present
2443− * and swap cases below report PAGE_IS_WRITTEN. Report it here
2444− * too so this generic path agrees with the PAGE_IS_WRITTEN fast
2445− * path in pagemap_scan_pmd_entry(), which reports pte_none as
2446− * written and, under PM_SCAN_WP_MATCHING, arms a marker. The
2447− * fast path applies no VMA test, so neither does this.
2441+ * An unpopulated pte carries no uffd bit, i.e. it is not
2442+ * write-protected. The PAGE_IS_WRITTEN fast path in
2443+ * pagemap_scan_pmd_entry() is now gated on a VM_UFFD_WP VMA;
2444+ * gate the pte_none report here the same way so the two paths
2445+ * still agree. RWP has no such fast path and an unpopulated
2446+ * page is not part of the RWP working set, so it is reported as
2447+ * neither.
24482448 */
2449− return PAGE_IS_WRITTEN;
2449+ if (userfaultfd_wp(vma))
2450+ return PAGE_IS_WRITTEN;
2451+ return 0;
24502452 }
24512453
24522454 if (pte_present(pte)) {
@@ -2454,8 +2456,12 @@ static unsigned long pagemap_page_category(struct pagemap_scan_private *p, · 1 unmodified line
24542456
24552457 categories = PAGE_IS_PRESENT;
24562458
2457− if (!pte_uffd(pte))
2458− categories |= PAGE_IS_WRITTEN;
2459+ if (!pte_uffd(pte)) {
2460+ if (userfaultfd_wp(vma))
2461+ categories |= PAGE_IS_WRITTEN;
2462+ if (userfaultfd_rwp(vma))
2463+ categories |= PAGE_IS_ACCESSED;
2464+ }
24592465
24602466 if (p->masks_of_interest & PAGE_IS_FILE) {
24612467 page = vm_normal_page(vma, addr, pte);
@@ -2472,8 +2478,12 @@ static unsigned long pagemap_page_category(struct pagemap_scan_private *p, · 10 unmodified lines
24722478
24732479 categories = PAGE_IS_SWAPPED;
24742480
2475− if (!pte_swp_uffd_any(pte))
2476− categories |= PAGE_IS_WRITTEN;
2481+ if (!pte_swp_uffd_any(pte)) {
2482+ if (userfaultfd_wp(vma))
2483+ categories |= PAGE_IS_WRITTEN;
2484+ if (userfaultfd_rwp(vma))
2485+ categories |= PAGE_IS_ACCESSED;
2486+ }
24772487
24782488 entry = softleaf_from_pte(pte);
24792489 if (softleaf_is_guard_marker(entry))
@@ -2522,8 +2532,12 @@ static unsigned long pagemap_thp_category(struct pagemap_scan_private *p, · 42 unmodified lines
25222532 struct page *page;
25232533
25242534 categories |= PAGE_IS_PRESENT;
2525− if (!pmd_uffd(pmd))
2526− categories |= PAGE_IS_WRITTEN;
2535+ if (!pmd_uffd(pmd)) {
2536+ if (userfaultfd_wp(vma))
2537+ categories |= PAGE_IS_WRITTEN;
2538+ if (userfaultfd_rwp(vma))
2539+ categories |= PAGE_IS_ACCESSED;
2540+ }
25272541
25282542 if (p->masks_of_interest & PAGE_IS_FILE) {
25292543 page = vm_normal_page_pmd(vma, addr, pmd);
@@ -2537,8 +2551,12 @@ static unsigned long pagemap_thp_category(struct pagemap_scan_private *p, · 7 unmodified lines
25372551 categories |= PAGE_IS_SOFT_DIRTY;
25382552 } else {
25392553 categories |= PAGE_IS_SWAPPED;
2540− if (!pmd_swp_uffd(pmd))
2541− categories |= PAGE_IS_WRITTEN;
2554+ if (!pmd_swp_uffd(pmd)) {
2555+ if (userfaultfd_wp(vma))
2556+ categories |= PAGE_IS_WRITTEN;
2557+ if (userfaultfd_rwp(vma))
2558+ categories |= PAGE_IS_ACCESSED;
2559+ }
25422560 if (pmd_swp_soft_dirty(pmd))
25432561 categories |= PAGE_IS_SOFT_DIRTY;
25442562
@@ -2571,7 +2589,8 @@ static void make_uffd_wp_pmd(struct vm_area_struct *vma, · 26 unmodified lines
25712589 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
25722590
25732591 #ifdef CONFIG_HUGETLB_PAGE
2574−static unsigned long pagemap_hugetlb_category(pte_t pte)
2592+static unsigned long pagemap_hugetlb_category(struct vm_area_struct *vma,
2593+ pte_t pte)
25752594 {
25762595 unsigned long categories = PAGE_IS_HUGE;
25772596
@@ -2586,8 +2605,12 @@ static unsigned long pagemap_hugetlb_category(pte_t pte) · 8 unmodified lines
25862605 if (pte_present(pte)) {
25872606 categories |= PAGE_IS_PRESENT;
25882607
2589− if (!huge_pte_uffd(pte))
2590− categories |= PAGE_IS_WRITTEN;
2608+ if (!huge_pte_uffd(pte)) {
2609+ if (userfaultfd_wp(vma))
2610+ categories |= PAGE_IS_WRITTEN;
2611+ if (userfaultfd_rwp(vma))
2612+ categories |= PAGE_IS_ACCESSED;
2613+ }
25912614 if (!PageAnon(pte_page(pte)))
25922615 categories |= PAGE_IS_FILE;
25932616 if (is_zero_pfn(pte_pfn(pte)))
@@ -2597,8 +2620,12 @@ static unsigned long pagemap_hugetlb_category(pte_t pte) · 3 unmodified lines
25972620 } else {
25982621 categories |= PAGE_IS_SWAPPED;
25992622
2600− if (!pte_swp_uffd_any(pte))
2601− categories |= PAGE_IS_WRITTEN;
2623+ if (!pte_swp_uffd_any(pte)) {
2624+ if (userfaultfd_wp(vma))
2625+ categories |= PAGE_IS_WRITTEN;
2626+ if (userfaultfd_rwp(vma))
2627+ categories |= PAGE_IS_ACCESSED;
2628+ }
26022629 if (pte_swp_soft_dirty(pte))
26032630 categories |= PAGE_IS_SOFT_DIRTY;
26042631 }
@@ -2874,7 +2901,8 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, · 269 unmodified lines
28742901 goto flush_and_return;
28752902 }
28762903
2877− if (!p->arg.category_anyof_mask && !p->arg.category_inverted &&
2904+ if (userfaultfd_wp(vma) && !p->arg.category_anyof_mask &&
2905+ !p->arg.category_inverted &&
28782906 p->arg.category_mask == PAGE_IS_WRITTEN &&
28792907 p->arg.return_mask == PAGE_IS_WRITTEN) {
28802908 for (addr = start; addr < end; pte++, addr += PAGE_SIZE) {
@@ -2949,7 +2977,8 @@ static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask, · 68 unmodified lines
29492977 /* Go the short route when not write-protecting pages. */
29502978
29512979 pte = huge_ptep_get(walk->mm, start, ptep);
2952− categories = p->cur_vma_category | pagemap_hugetlb_category(pte);
2980+ categories = p->cur_vma_category |
2981+ pagemap_hugetlb_category(vma, pte);
29532982
29542983 if (!pagemap_scan_is_interesting_page(categories, p))
29552984 return 0;
@@ -2961,7 +2990,7 @@ static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask, · 5 unmodified lines
29612990 ptl = huge_pte_lock(hstate_vma(vma), vma->vm_mm, ptep);
29622991
29632992 pte = huge_ptep_get(walk->mm, start, ptep);
2964− categories = p->cur_vma_category | pagemap_hugetlb_category(pte);
2993+ categories = p->cur_vma_category | pagemap_hugetlb_category(vma, pte);
29652994
29662995 if (!pagemap_scan_is_interesting_page(categories, p))
29672996 goto out_unlock;