History

mm: add VM_UFFD_RWP VMA flag

Preparatory patch for userfaultfd read-write protection (RWP). RWP extends userfaultfd protection from plain write-protection (WP) to full read-write protection: accesses to an RWP-protected range -- reads as well as writes -- trap through userfaultfd. Reserve VM_UFFD_RWP, add the userfaultfd_rwp() and userfaultfd_protected() helpers, and wire up the smaps "ur" entry and the trace-flag table the rest of the series will use. The flag is gated on CONFIG_USERFAULTFD_RWP, which is introduced together with the UAPI in a later patch; until then VM_UFFD_RWP aliases VM_NONE and every downstream check folds to dead code. Nothing sets or queries the flag yet.

Kiryl Shutsemau (Meta) committed 4 months ago

Commit 134c1d01 parent 2730ee25 files · +75 −24Browse files

Unified · Split

5 files
  1. Documentation/filesystems/proc.rst +1 −0
  2. fs/proc/task_mmu.c +3 −0
  3. include/linux/mm.h +28 −15
  4. include/linux/userfaultfd_k.h +36 −9
  5. include/trace/events/mmflags.h +7 −0
Documentation/filesystems/proc.rst+1−0
@@ -608,6 +608,7 @@ encoded manner. The codes are the following: · 607 unmodified lines
608608 um userfaultfd missing tracking
609609 uw userfaultfd wr-protect tracking
610610 ui userfaultfd minor fault
611+ ur userfaultfd read-write-protect tracking
611612 ss shadow/guarded control stack page
612613 sl sealed
613614 lf lock on fault pages
fs/proc/task_mmu.c+3−0
@@ -1237,6 +1237,9 @@ static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma) · 1236 unmodified lines
12371237 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
12381238 [ilog2(VM_UFFD_MINOR)] = "ui",
12391239 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */
1240+#ifdef CONFIG_USERFAULTFD_RWP
1241+ [ilog2(VM_UFFD_RWP)] = "ur",
1242+#endif
12401243 #ifdef CONFIG_ARCH_HAS_USER_SHADOW_STACK
12411244 [ilog2(VM_SHADOW_STACK)] = "ss",
12421245 #endif
include/linux/mm.h+28−15
@@ -354,6 +354,7 @@ enum { · 353 unmodified lines
354354 #endif
355355 DECLARE_VMA_BIT(UFFD_MINOR, 41),
356356 DECLARE_VMA_BIT(SEALED, 42),
357+ DECLARE_VMA_BIT(UFFD_RWP, 43),
357358 /* Flags that reuse flags above. */
358359 DECLARE_VMA_BIT_ALIAS(PKEY_BIT0, HIGH_ARCH_0),
359360 DECLARE_VMA_BIT_ALIAS(PKEY_BIT1, HIGH_ARCH_1),
@@ -497,12 +498,17 @@ enum { · 137 unmodified lines
497498 #else
498499 #define VM_UFFD_MINOR VM_NONE
499500 #endif
501+#ifdef CONFIG_USERFAULTFD_RWP
502+#define VM_UFFD_RWP INIT_VM_FLAG(UFFD_RWP)
503+#else
504+#define VM_UFFD_RWP VM_NONE
505+#endif
500506
501507 /*
502− * vma_flags_t masks for the userfaultfd VMA flags. VMA_UFFD_MINOR is gated on
503− * the same config as VM_UFFD_MINOR -- which implies 64BIT, where the bit fits
504− * -- so an out-of-range bit is never fed to mk_vma_flags() on a build whose
505− * bitmap cannot hold it.
508+ * vma_flags_t masks for the userfaultfd VMA flags. The two high-bit modes are
509+ * gated on the same configs as their VM_* flags above -- both of which imply
510+ * 64BIT -- so an out-of-range bit is never fed to mk_vma_flags() on a build
511+ * whose bitmap cannot hold it.
506512 */
507513 #define VMA_UFFD_MISSING mk_vma_flags(VMA_UFFD_MISSING_BIT)
508514 #define VMA_UFFD_WP mk_vma_flags(VMA_UFFD_WP_BIT)
@@ -511,6 +517,11 @@ enum { · 2 unmodified lines
511517 #else
512518 #define VMA_UFFD_MINOR EMPTY_VMA_FLAGS
513519 #endif
520+#ifdef CONFIG_USERFAULTFD_RWP
521+#define VMA_UFFD_RWP mk_vma_flags(VMA_UFFD_RWP_BIT)
522+#else
523+#define VMA_UFFD_RWP EMPTY_VMA_FLAGS
524+#endif
514525
515526 #ifdef CONFIG_64BIT
516527 #define VM_ALLOW_ANY_UNCACHED INIT_VM_FLAG(ALLOW_ANY_UNCACHED)
@@ -649,22 +660,24 @@ enum { · 132 unmodified lines
649660 * reconsistuted upon page fault, so necessitate page table copying upon fork.
650661 *
651662 * Note that these flags should be compared with the DESTINATION VMA not the
652− * source, as VM_UFFD_WP may not be propagated to destination, while all other
653− * flags will be.
663+ * source: VM_UFFD_WP and VM_UFFD_RWP may be cleared on the destination
664+ * (dup_userfaultfd() -> userfaultfd_reset_ctx() when the parent context did
665+ * not negotiate UFFD_FEATURE_EVENT_FORK), while all other flags propagate.
654666 *
655667 * VM_PFNMAP / VM_MIXEDMAP - These contain kernel-mapped data which cannot be
656668 * reasonably reconstructed on page fault.
657669 *
658670 * VM_UFFD_WP - Encodes metadata about an installed uffd
659− * write protect handler, which cannot be
660− * reconstructed on page fault.
661− *
662− * We always copy pgtables when dst_vma has uffd-wp
663− * enabled even if it's file-backed
664− * (e.g. shmem). Because when uffd-wp is enabled,
665− * pgtable contains uffd-wp protection information,
666− * that's something we can't retrieve from page cache,
667− * and skip copying will lose those info.
671+ * VM_UFFD_RWP write- or read-write-protect handler, which
672+ * cannot be reconstructed on page fault.
673+ *
674+ * We always copy pgtables when dst_vma has the
675+ * uffd PTE bit in use even if it's file-backed
676+ * (e.g. shmem). Because when the uffd bit is
677+ * in use, the pgtable contains the protection
678+ * information, that's something we can't
679+ * retrieve from page cache, and skip copying
680+ * will lose those info.
668681 *
669682 * VM_MAYBE_GUARD - Could contain page guard region markers which
670683 * by design are a property of the page tables
include/linux/userfaultfd_k.h+36−9
@@ -21,10 +21,11 @@ · 20 unmodified lines
2121 #include <linux/hugetlb_inline.h>
2222
2323 /* The set of all possible UFFD-related VM flags. */
24−#define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_WP | VM_UFFD_MINOR)
24+#define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_MINOR | \
25+ VM_UFFD_WP | VM_UFFD_RWP)
2526
2627 #define __VMA_UFFD_FLAGS mk_vma_flags_from_masks(VMA_UFFD_MISSING, VMA_UFFD_WP, \
27− VMA_UFFD_MINOR)
28+ VMA_UFFD_MINOR, VMA_UFFD_RWP)
2829
2930 /*
3031 * CAREFUL: Check include/uapi/asm-generic/fcntl.h when defining
@@ -168,7 +169,8 @@ static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, · 137 unmodified lines
168169 /*
169170 * Never enable huge pmd sharing on some uffd registered vmas:
170171 *
171− * - VM_UFFD_WP VMAs, because write protect information is per pgtable entry.
172+ * - VM_UFFD_WP and VM_UFFD_RWP VMAs, because the write protect / access
173+ * tracking information is per pgtable entry.
172174 *
173175 * - VM_UFFD_MINOR VMAs, because otherwise we would never get minor faults for
174176 * VMAs which share huge pmds. (If you have two mappings to the same
@@ -179,20 +181,25 @@ static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, · 4 unmodified lines
179181 static inline bool uffd_disable_huge_pmd_share(struct vm_area_struct *vma)
180182 {
181183 return vma_test_any_mask(vma,
182− mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_MINOR));
184+ mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_RWP,
185+ VMA_UFFD_MINOR));
183186 }
184187
185188 /*
186− * Don't do fault around for either WP or MINOR registered uffd range. For
189+ * Don't do fault around for WP, RWP or MINOR registered uffd range. For
187190 * MINOR registered range, fault around will be a total disaster and ptes can
188191 * be installed without notifications; for WP it should mostly be fine as long
189192 * as the fault around checks for pte_none() before the installation, however
190− * to be super safe we just forbid it.
193+ * to be super safe we just forbid it; for RWP, pre-faulted neighbours would
194+ * be indistinguishable from accessed pages in PAGEMAP_SCAN (PAGE_IS_ACCESSED)
195+ * and pollute the tracked working set, so each page must be populated by its
196+ * own fault.
191197 */
192198 static inline bool uffd_disable_fault_around(struct vm_area_struct *vma)
193199 {
194200 return vma_test_any_mask(vma,
195− mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_MINOR));
201+ mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_RWP,
202+ VMA_UFFD_MINOR));
196203 }
197204
198205 static inline bool userfaultfd_missing(struct vm_area_struct *vma)
@@ -210,6 +217,16 @@ static inline bool userfaultfd_minor(struct vm_area_struct *vma) · 11 unmodified lines
210217 return vma_test_any_mask(vma, VMA_UFFD_MINOR);
211218 }
212219
220+static inline bool userfaultfd_rwp(struct vm_area_struct *vma)
221+{
222+ return vma_test_single_mask(vma, VMA_UFFD_RWP);
223+}
224+
225+static inline bool userfaultfd_protected(struct vm_area_struct *vma)
226+{
227+ return userfaultfd_wp(vma) || userfaultfd_rwp(vma);
228+}
229+
213230 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma,
214231 pte_t pte)
215232 {
@@ -330,6 +347,16 @@ static inline bool userfaultfd_minor(struct vm_area_struct *vma) · 114 unmodified lines
330347 return false;
331348 }
332349
350+static inline bool userfaultfd_rwp(struct vm_area_struct *vma)
351+{
352+ return false;
353+}
354+
355+static inline bool userfaultfd_protected(struct vm_area_struct *vma)
356+{
357+ return false;
358+}
359+
333360 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma,
334361 pte_t pte)
335362 {
@@ -423,8 +450,8 @@ static inline bool userfaultfd_wp_use_markers(struct vm_area_struct *vma) · 87 unmodified lines
423450 }
424451
425452 /*
426− * Returns true if this is a swap pte and was uffd-wp wr-protected in either
427− * forms (pte marker or a normal swap pte), false otherwise.
453+ * Returns true if this swap pte carries uffd-tracked state in either
454+ * form (pte marker or a normal swap pte), false otherwise.
428455 */
429456 static inline bool pte_swp_uffd_any(pte_t pte)
430457 {
include/trace/events/mmflags.h+7−0
@@ -186,6 +186,12 @@ IF_HAVE_PG_ARCH_3(arch_3) · 185 unmodified lines
186186 # define IF_HAVE_UFFD_MINOR(flag, name)
187187 #endif
188188
189+#ifdef CONFIG_USERFAULTFD_RWP
190+# define IF_HAVE_UFFD_RWP(flag, name) {flag, name},
191+#else
192+# define IF_HAVE_UFFD_RWP(flag, name)
193+#endif
194+
189195 #if defined(CONFIG_64BIT) || defined(CONFIG_PPC32)
190196 # define IF_HAVE_VM_DROPPABLE(flag, name) {flag, name},
191197 #else
@@ -207,6 +213,7 @@ IF_HAVE_UFFD_MINOR(VM_UFFD_MINOR, "uffd_minor" ) \ · 15 unmodified lines
207213 {VM_PFNMAP, "pfnmap" }, \
208214 {VM_MAYBE_GUARD, "maybe_guard" }, \
209215 {VM_UFFD_WP, "uffd_wp" }, \
216+IF_HAVE_UFFD_RWP(VM_UFFD_RWP, "uffd_rwp" ) \
210217 {VM_LOCKED, "locked" }, \
211218 {VM_IO, "io" }, \
212219 {VM_SEQ_READ, "seqread" }, \