mm: add VM_UFFD_RWP VMA flag

Preparatory patch for userfaultfd read-write protection (RWP). RWP extends userfaultfd protection from plain write-protection (WP) to full read-write protection: accesses to an RWP-protected range -- reads as well as writes -- trap through userfaultfd. Reserve VM_UFFD_RWP, add the userfaultfd_rwp() and userfaultfd_protected() helpers, and wire up the smaps "ur" entry and the trace-flag table the rest of the series will use. The flag is gated on CONFIG_USERFAULTFD_RWP, which is introduced together with the UAPI in a later patch; until then VM_UFFD_RWP aliases VM_NONE and every downstream check folds to dead code. Nothing sets or queries the flag yet.

Kiryl Shutsemau (Meta) committed 4 months ago

Commit 134c1d01 parent 2730ee25 files · +75 −24Browse files

Unified · Split

5 files

Documentation/filesystems/proc.rst +1 −0
fs/proc/task_mmu.c +3 −0
include/linux/mm.h +28 −15
include/linux/userfaultfd_k.h +36 −9
include/trace/events/mmflags.h +7 −0

Documentation/filesystems/proc.rst+1−0

@@ -608,6 +608,7 @@ encoded manner. The codes are the following: · 607 unmodified lines
608	um userfaultfd missing tracking	608	um userfaultfd missing tracking
609	uw userfaultfd wr-protect tracking	609	uw userfaultfd wr-protect tracking
610	ui userfaultfd minor fault	610	ui userfaultfd minor fault
		611	ur userfaultfd read-write-protect tracking
611	ss shadow/guarded control stack page	612	ss shadow/guarded control stack page
612	sl sealed	613	sl sealed
613	lf lock on fault pages	614	lf lock on fault pages

fs/proc/task_mmu.c+3−0

 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
 		[ilog2(VM_UFFD_MINOR)]	= "ui",
 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */
+#ifdef CONFIG_USERFAULTFD_RWP
+		[ilog2(VM_UFFD_RWP)]	= "ur",
+#endif
 #ifdef CONFIG_ARCH_HAS_USER_SHADOW_STACK
 		[ilog2(VM_SHADOW_STACK)] = "ss",
 #endif

include/linux/mm.h+28−15

 #endif
 	DECLARE_VMA_BIT(UFFD_MINOR, 41),
 	DECLARE_VMA_BIT(SEALED, 42),
+	DECLARE_VMA_BIT(UFFD_RWP, 43),
 	/* Flags that reuse flags above. */
 	DECLARE_VMA_BIT_ALIAS(PKEY_BIT0, HIGH_ARCH_0),
 	DECLARE_VMA_BIT_ALIAS(PKEY_BIT1, HIGH_ARCH_1),
 #else
 #define VM_UFFD_MINOR	VM_NONE
 #endif
+#ifdef CONFIG_USERFAULTFD_RWP
+#define VM_UFFD_RWP		INIT_VM_FLAG(UFFD_RWP)
+#else
+#define VM_UFFD_RWP		VM_NONE
+#endif
 /*
- * vma_flags_t masks for the userfaultfd VMA flags. VMA_UFFD_MINOR is gated on
+ * vma_flags_t masks for the userfaultfd VMA flags. The two high-bit modes are
- * the same config as VM_UFFD_MINOR -- which implies 64BIT, where the bit fits
+ * gated on the same configs as their VM_* flags above -- both of which imply
- * -- so an out-of-range bit is never fed to mk_vma_flags() on a build whose
+ * 64BIT -- so an out-of-range bit is never fed to mk_vma_flags() on a build
- * bitmap cannot hold it.
+ * whose bitmap cannot hold it.
  */
 #define VMA_UFFD_MISSING	mk_vma_flags(VMA_UFFD_MISSING_BIT)
 #define VMA_UFFD_WP		mk_vma_flags(VMA_UFFD_WP_BIT)
 #else
 #define VMA_UFFD_MINOR		EMPTY_VMA_FLAGS
 #endif
+#ifdef CONFIG_USERFAULTFD_RWP
+#define VMA_UFFD_RWP		mk_vma_flags(VMA_UFFD_RWP_BIT)
+#else
+#define VMA_UFFD_RWP		EMPTY_VMA_FLAGS
+#endif
 #ifdef CONFIG_64BIT
 #define VM_ALLOW_ANY_UNCACHED	INIT_VM_FLAG(ALLOW_ANY_UNCACHED)
  * reconsistuted upon page fault, so necessitate page table copying upon fork.
  *
  * Note that these flags should be compared with the DESTINATION VMA not the
- * source, as VM_UFFD_WP may not be propagated to destination, while all other
+ * source: VM_UFFD_WP and VM_UFFD_RWP may be cleared on the destination
- * flags will be.
+ * (dup_userfaultfd() -> userfaultfd_reset_ctx() when the parent context did
+ * not negotiate UFFD_FEATURE_EVENT_FORK), while all other flags propagate.
  *
  * VM_PFNMAP / VM_MIXEDMAP - These contain kernel-mapped data which cannot be
  *                           reasonably reconstructed on page fault.
  *
  *              VM_UFFD_WP - Encodes metadata about an installed uffd
- *                           write protect handler, which cannot be
+ *              VM_UFFD_RWP  write- or read-write-protect handler, which
- *                           reconstructed on page fault.
+ *                           cannot be reconstructed on page fault.
  *
- *                           We always copy pgtables when dst_vma has uffd-wp
+ *                           We always copy pgtables when dst_vma has the
- *                           enabled even if it's file-backed
+ *                           uffd PTE bit in use even if it's file-backed
- *                           (e.g. shmem). Because when uffd-wp is enabled,
+ *                           (e.g. shmem). Because when the uffd bit is
- *                           pgtable contains uffd-wp protection information,
+ *                           in use, the pgtable contains the protection
- *                           that's something we can't retrieve from page cache,
+ *                           information, that's something we can't
- *                           and skip copying will lose those info.
+ *                           retrieve from page cache, and skip copying
+ *                           will lose those info.
  *
  *          VM_MAYBE_GUARD - Could contain page guard region markers which
  *                           by design are a property of the page tables

include/linux/userfaultfd_k.h+36−9

 #include <linux/hugetlb_inline.h>
 /* The set of all possible UFFD-related VM flags. */
-#define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_WP | VM_UFFD_MINOR)
+#define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_MINOR | \
+			 VM_UFFD_WP | VM_UFFD_RWP)
 #define __VMA_UFFD_FLAGS mk_vma_flags_from_masks(VMA_UFFD_MISSING, VMA_UFFD_WP, \
-						 VMA_UFFD_MINOR)
+						 VMA_UFFD_MINOR, VMA_UFFD_RWP)
 /*
  * CAREFUL: Check include/uapi/asm-generic/fcntl.h when defining
 /*
  * Never enable huge pmd sharing on some uffd registered vmas:
  *
- * - VM_UFFD_WP VMAs, because write protect information is per pgtable entry.
+ * - VM_UFFD_WP and VM_UFFD_RWP VMAs, because the write protect / access
+ *   tracking information is per pgtable entry.
  *
  * - VM_UFFD_MINOR VMAs, because otherwise we would never get minor faults for
  *   VMAs which share huge pmds. (If you have two mappings to the same
 static inline bool uffd_disable_huge_pmd_share(struct vm_area_struct *vma)
 {
 	return vma_test_any_mask(vma,
-		mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_MINOR));
+		mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_RWP,
+					VMA_UFFD_MINOR));
 }
 /*
- * Don't do fault around for either WP or MINOR registered uffd range.  For
+ * Don't do fault around for WP, RWP or MINOR registered uffd range.  For
  * MINOR registered range, fault around will be a total disaster and ptes can
  * be installed without notifications; for WP it should mostly be fine as long
  * as the fault around checks for pte_none() before the installation, however
- * to be super safe we just forbid it.
+ * to be super safe we just forbid it; for RWP, pre-faulted neighbours would
+ * be indistinguishable from accessed pages in PAGEMAP_SCAN (PAGE_IS_ACCESSED)
+ * and pollute the tracked working set, so each page must be populated by its
+ * own fault.
  */
 static inline bool uffd_disable_fault_around(struct vm_area_struct *vma)
 {
 	return vma_test_any_mask(vma,
-		mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_MINOR));
+		mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_RWP,
+					VMA_UFFD_MINOR));
 }
 static inline bool userfaultfd_missing(struct vm_area_struct *vma)
 	return vma_test_any_mask(vma, VMA_UFFD_MINOR);
 }
+static inline bool userfaultfd_rwp(struct vm_area_struct *vma)
+{
+	return vma_test_single_mask(vma, VMA_UFFD_RWP);
+}
+static inline bool userfaultfd_protected(struct vm_area_struct *vma)
+{
+	return userfaultfd_wp(vma) || userfaultfd_rwp(vma);
+}
 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma,
 				      pte_t pte)
 {
 	return false;
 }
+static inline bool userfaultfd_rwp(struct vm_area_struct *vma)
+{
+	return false;
+}
+static inline bool userfaultfd_protected(struct vm_area_struct *vma)
+{
+	return false;
+}
 static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma,
 				      pte_t pte)
 {
 }
 /*
- * Returns true if this is a swap pte and was uffd-wp wr-protected in either
+ * Returns true if this swap pte carries uffd-tracked state in either
- * forms (pte marker or a normal swap pte), false otherwise.
+ * form (pte marker or a normal swap pte), false otherwise.
  */
 static inline bool pte_swp_uffd_any(pte_t pte)
 {

include/trace/events/mmflags.h+7−0

 # define IF_HAVE_UFFD_MINOR(flag, name)
 #endif
+#ifdef CONFIG_USERFAULTFD_RWP
+# define IF_HAVE_UFFD_RWP(flag, name) {flag, name},
+#else
+# define IF_HAVE_UFFD_RWP(flag, name)
+#endif
 #if defined(CONFIG_64BIT) || defined(CONFIG_PPC32)
 # define IF_HAVE_VM_DROPPABLE(flag, name) {flag, name},
 #else
 	{VM_PFNMAP,			"pfnmap"	},		\
 	{VM_MAYBE_GUARD,		"maybe_guard"	},		\
 	{VM_UFFD_WP,			"uffd_wp"	},		\
+IF_HAVE_UFFD_RWP(VM_UFFD_RWP,		"uffd_rwp"	)		\
 	{VM_LOCKED,			"locked"	},		\
 	{VM_IO,				"io"		},		\
 	{VM_SEQ_READ,			"seqread"	},		\