History

Merge branch 'akpm' (patches from Andrew)

Merge misc fixes from Andrew Morton: "8 patches. Subsystems affected by this patch series: mm (hugetlb, pagemap, and userfaultfd), memfd, selftests, and kconfig" * emailed patches from Andrew Morton <akpm@linux-foundation.org>: configs/debug: set CONFIG_DEBUG_INFO=y properly proc: fix documentation and description of pagemap kselftest/vm: fix tests build with old libc memfd: fix F_SEAL_WRITE after shmem huge page allocated mm: fix use-after-free when anon vma name is used after vma is freed mm: prevent vm_area_struct::anon_name refcount saturation mm: refactor vm_area_struct::anon_vma_name usage code selftests/vm: cleanup hugetlb file after mremap test

Linus Torvalds committed 4 years ago

Commit 00144042 parents f9026e1 d1eff1618 files · +194 −137Browse files

Unified · Split

18 files
  1. Documentation/admin-guide/mm/pagemap.rst +1 −1
  2. fs/proc/task_mmu.c +5 −4
  3. fs/userfaultfd.c +3 −3
  4. include/linux/mm.h +4 −3
  5. include/linux/mm_inline.h +69 −28
  6. include/linux/mm_types.h +4 −1
  7. kernel/configs/debug.config +1 −1
  8. kernel/fork.c +2 −2
  9. kernel/sys.c +12 −7
  10. mm/madvise.c +34 −58
  11. mm/memfd.c +28 −12
  12. mm/mempolicy.c +1 −1
  13. mm/mlock.c +1 −1
  14. mm/mmap.c +6 −6
  15. mm/mprotect.c +1 −1
  16. tools/testing/selftests/vm/hugepage-mremap.c +19 −7
  17. tools/testing/selftests/vm/run_vmtests.sh +2 −1
  18. tools/testing/selftests/vm/userfaultfd.c +1 −0
Documentation/admin-guide/mm/pagemap.rst+1−1
@@ -23,7 +23,7 @@ There are four components to pagemap: · 22 unmodified lines
23 * Bit 56 page exclusively mapped (since 4.2)23 * Bit 56 page exclusively mapped (since 4.2)
24 * Bit 57 pte is uffd-wp write-protected (since 5.13) (see24 * Bit 57 pte is uffd-wp write-protected (since 5.13) (see
25 :ref:`Documentation/admin-guide/mm/userfaultfd.rst <userfaultfd>`)25 :ref:`Documentation/admin-guide/mm/userfaultfd.rst <userfaultfd>`)
26 * Bits 57-60 zero26 * Bits 58-60 zero
27 * Bit 61 page is file-page or shared-anon (since 3.5)27 * Bit 61 page is file-page or shared-anon (since 3.5)
28 * Bit 62 page swapped28 * Bit 62 page swapped
29 * Bit 63 page present29 * Bit 63 page present
fs/proc/task_mmu.c+5−4
@@ -309,7 +309,7 @@ show_map_vma(struct seq_file *m, struct vm_area_struct *vma) · 308 unmodified lines
309 309
310 name = arch_vma_name(vma);310 name = arch_vma_name(vma);
311 if (!name) {311 if (!name) {
312 const char *anon_name;312 struct anon_vma_name *anon_name;
313 313
314 if (!mm) {314 if (!mm) {
315 name = "[vdso]";315 name = "[vdso]";
@@ -327,10 +327,10 @@ show_map_vma(struct seq_file *m, struct vm_area_struct *vma) · 11 unmodified lines
327 goto done;327 goto done;
328 }328 }
329 329
330 anon_name = vma_anon_name(vma);330 anon_name = anon_vma_name(vma);
331 if (anon_name) {331 if (anon_name) {
332 seq_pad(m, ' ');332 seq_pad(m, ' ');
333 seq_printf(m, "[anon:%s]", anon_name);333 seq_printf(m, "[anon:%s]", anon_name->name);
334 }334 }
335 }335 }
336 336
@@ -1597,7 +1597,8 @@ static const struct mm_walk_ops pagemap_ops = { · 1260 unmodified lines
1597 * Bits 5-54 swap offset if swapped1597 * Bits 5-54 swap offset if swapped
1598 * Bit 55 pte is soft-dirty (see Documentation/admin-guide/mm/soft-dirty.rst)1598 * Bit 55 pte is soft-dirty (see Documentation/admin-guide/mm/soft-dirty.rst)
1599 * Bit 56 page exclusively mapped1599 * Bit 56 page exclusively mapped
1600 * Bits 57-60 zero1600 * Bit 57 pte is uffd-wp write-protected
1601 * Bits 58-60 zero
1601 * Bit 61 page is file-page or shared-anon1602 * Bit 61 page is file-page or shared-anon
1602 * Bit 62 page swapped1603 * Bit 62 page swapped
1603 * Bit 63 page present1604 * Bit 63 page present
fs/userfaultfd.c+3−3
@@ -878,7 +878,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file) · 877 unmodified lines
878 new_flags, vma->anon_vma,878 new_flags, vma->anon_vma,
879 vma->vm_file, vma->vm_pgoff,879 vma->vm_file, vma->vm_pgoff,
880 vma_policy(vma),880 vma_policy(vma),
881 NULL_VM_UFFD_CTX, vma_anon_name(vma));881 NULL_VM_UFFD_CTX, anon_vma_name(vma));
882 if (prev)882 if (prev)
883 vma = prev;883 vma = prev;
884 else884 else
@@ -1438,7 +1438,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, · 553 unmodified lines
1438 vma->anon_vma, vma->vm_file, vma->vm_pgoff,1438 vma->anon_vma, vma->vm_file, vma->vm_pgoff,
1439 vma_policy(vma),1439 vma_policy(vma),
1440 ((struct vm_userfaultfd_ctx){ ctx }),1440 ((struct vm_userfaultfd_ctx){ ctx }),
1441 vma_anon_name(vma));1441 anon_vma_name(vma));
1442 if (prev) {1442 if (prev) {
1443 vma = prev;1443 vma = prev;
1444 goto next;1444 goto next;
@@ -1615,7 +1615,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, · 170 unmodified lines
1615 prev = vma_merge(mm, prev, start, vma_end, new_flags,1615 prev = vma_merge(mm, prev, start, vma_end, new_flags,
1616 vma->anon_vma, vma->vm_file, vma->vm_pgoff,1616 vma->anon_vma, vma->vm_file, vma->vm_pgoff,
1617 vma_policy(vma),1617 vma_policy(vma),
1618 NULL_VM_UFFD_CTX, vma_anon_name(vma));1618 NULL_VM_UFFD_CTX, anon_vma_name(vma));
1619 if (prev) {1619 if (prev) {
1620 vma = prev;1620 vma = prev;
1621 goto next;1621 goto next;
include/linux/mm.h+4−3
@@ -2626,7 +2626,7 @@ static inline int vma_adjust(struct vm_area_struct *vma, unsigned long start, · 2625 unmodified lines
2626extern struct vm_area_struct *vma_merge(struct mm_struct *,2626extern struct vm_area_struct *vma_merge(struct mm_struct *,
2627 struct vm_area_struct *prev, unsigned long addr, unsigned long end,2627 struct vm_area_struct *prev, unsigned long addr, unsigned long end,
2628 unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,2628 unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
2629 struct mempolicy *, struct vm_userfaultfd_ctx, const char *);2629 struct mempolicy *, struct vm_userfaultfd_ctx, struct anon_vma_name *);
2630extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);2630extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
2631extern int __split_vma(struct mm_struct *, struct vm_area_struct *,2631extern int __split_vma(struct mm_struct *, struct vm_area_struct *,
2632 unsigned long addr, int new_below);2632 unsigned long addr, int new_below);
@@ -3372,11 +3372,12 @@ static inline int seal_check_future_write(int seals, struct vm_area_struct *vma) · 739 unmodified lines
3372 3372
3373#ifdef CONFIG_ANON_VMA_NAME3373#ifdef CONFIG_ANON_VMA_NAME
3374int madvise_set_anon_name(struct mm_struct *mm, unsigned long start,3374int madvise_set_anon_name(struct mm_struct *mm, unsigned long start,
3375 unsigned long len_in, const char *name);3375 unsigned long len_in,
3376 struct anon_vma_name *anon_name);
3376#else3377#else
3377static inline int3378static inline int
3378madvise_set_anon_name(struct mm_struct *mm, unsigned long start,3379madvise_set_anon_name(struct mm_struct *mm, unsigned long start,
3379 unsigned long len_in, const char *name) {3380 unsigned long len_in, struct anon_vma_name *anon_name) {
3380 return 0;3381 return 0;
3381}3382}
3382#endif3383#endif
include/linux/mm_inline.h+69−28
@@ -140,50 +140,91 @@ static __always_inline void del_page_from_lru_list(struct page *page, · 139 unmodified lines
140 140
141#ifdef CONFIG_ANON_VMA_NAME141#ifdef CONFIG_ANON_VMA_NAME
142/*142/*
143 * mmap_lock should be read-locked when calling vma_anon_name() and while using143 * mmap_lock should be read-locked when calling anon_vma_name(). Caller should
144 * the returned pointer.144 * either keep holding the lock while using the returned pointer or it should
145 * raise anon_vma_name refcount before releasing the lock.
145 */146 */
146extern const char *vma_anon_name(struct vm_area_struct *vma);147extern struct anon_vma_name *anon_vma_name(struct vm_area_struct *vma);
148extern struct anon_vma_name *anon_vma_name_alloc(const char *name);
149extern void anon_vma_name_free(struct kref *kref);
147 150
148/*151/* mmap_lock should be read-locked */
149 * mmap_lock should be read-locked for orig_vma->vm_mm.152static inline void anon_vma_name_get(struct anon_vma_name *anon_name)
150 * mmap_lock should be write-locked for new_vma->vm_mm or new_vma should be153{
151 * isolated.154 if (anon_name)
152 */155 kref_get(&anon_name->kref);
153extern void dup_vma_anon_name(struct vm_area_struct *orig_vma,156}
154 struct vm_area_struct *new_vma);
155 157
156/*158static inline void anon_vma_name_put(struct anon_vma_name *anon_name)
157 * mmap_lock should be write-locked or vma should have been isolated under159{
158 * write-locked mmap_lock protection.160 if (anon_name)
159 */161 kref_put(&anon_name->kref, anon_vma_name_free);
160extern void free_vma_anon_name(struct vm_area_struct *vma);162}
161 163
162/* mmap_lock should be read-locked */164static inline
163static inline bool is_same_vma_anon_name(struct vm_area_struct *vma,165struct anon_vma_name *anon_vma_name_reuse(struct anon_vma_name *anon_name)
164 const char *name)166{
167 /* Prevent anon_name refcount saturation early on */
168 if (kref_read(&anon_name->kref) < REFCOUNT_MAX) {
169 anon_vma_name_get(anon_name);
170 return anon_name;
171
172 }
173 return anon_vma_name_alloc(anon_name->name);
174}
175
176static inline void dup_anon_vma_name(struct vm_area_struct *orig_vma,
177 struct vm_area_struct *new_vma)
178{
179 struct anon_vma_name *anon_name = anon_vma_name(orig_vma);
180
181 if (anon_name)
182 new_vma->anon_name = anon_vma_name_reuse(anon_name);
183}
184
185static inline void free_anon_vma_name(struct vm_area_struct *vma)
165{186{
166 const char *vma_name = vma_anon_name(vma);187 /*
188 * Not using anon_vma_name because it generates a warning if mmap_lock
189 * is not held, which might be the case here.
190 */
191 if (!vma->vm_file)
192 anon_vma_name_put(vma->anon_name);
193}
167 194
168 /* either both NULL, or pointers to same string */195static inline bool anon_vma_name_eq(struct anon_vma_name *anon_name1,
169 if (vma_name == name)196 struct anon_vma_name *anon_name2)
197{
198 if (anon_name1 == anon_name2)
170 return true;199 return true;
171 200
172 return name && vma_name && !strcmp(name, vma_name);201 return anon_name1 && anon_name2 &&
202 !strcmp(anon_name1->name, anon_name2->name);
173}203}
204
174#else /* CONFIG_ANON_VMA_NAME */205#else /* CONFIG_ANON_VMA_NAME */
175static inline const char *vma_anon_name(struct vm_area_struct *vma)206static inline struct anon_vma_name *anon_vma_name(struct vm_area_struct *vma)
176{207{
177 return NULL;208 return NULL;
178}209}
179static inline void dup_vma_anon_name(struct vm_area_struct *orig_vma,210
180 struct vm_area_struct *new_vma) {}211static inline struct anon_vma_name *anon_vma_name_alloc(const char *name)
181static inline void free_vma_anon_name(struct vm_area_struct *vma) {}212{
182static inline bool is_same_vma_anon_name(struct vm_area_struct *vma,213 return NULL;
183 const char *name)214}
215
216static inline void anon_vma_name_get(struct anon_vma_name *anon_name) {}
217static inline void anon_vma_name_put(struct anon_vma_name *anon_name) {}
218static inline void dup_anon_vma_name(struct vm_area_struct *orig_vma,
219 struct vm_area_struct *new_vma) {}
220static inline void free_anon_vma_name(struct vm_area_struct *vma) {}
221
222static inline bool anon_vma_name_eq(struct anon_vma_name *anon_name1,
223 struct anon_vma_name *anon_name2)
184{224{
185 return true;225 return true;
186}226}
227
187#endif /* CONFIG_ANON_VMA_NAME */228#endif /* CONFIG_ANON_VMA_NAME */
188 229
189static inline void init_tlb_flush_pending(struct mm_struct *mm)230static inline void init_tlb_flush_pending(struct mm_struct *mm)
include/linux/mm_types.h+4−1
@@ -416,7 +416,10 @@ struct vm_area_struct { · 415 unmodified lines
416 struct rb_node rb;416 struct rb_node rb;
417 unsigned long rb_subtree_last;417 unsigned long rb_subtree_last;
418 } shared;418 } shared;
419 /* Serialized by mmap_sem. */419 /*
420 * Serialized by mmap_sem. Never use directly because it is
421 * valid only when vm_file is NULL. Use anon_vma_name instead.
422 */
420 struct anon_vma_name *anon_name;423 struct anon_vma_name *anon_name;
421 };424 };
422 425
kernel/configs/debug.config+1−1
@@ -16,7 +16,7 @@ CONFIG_SYMBOLIC_ERRNAME=y · 15 unmodified lines
16#16#
17# Compile-time checks and compiler options17# Compile-time checks and compiler options
18#18#
19CONFIG_DEBUG_INFO=y19CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
20CONFIG_DEBUG_SECTION_MISMATCH=y20CONFIG_DEBUG_SECTION_MISMATCH=y
21CONFIG_FRAME_WARN=204821CONFIG_FRAME_WARN=2048
22CONFIG_SECTION_MISMATCH_WARN_ONLY=y22CONFIG_SECTION_MISMATCH_WARN_ONLY=y
kernel/fork.c+2−2
@@ -366,14 +366,14 @@ struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig) · 365 unmodified lines
366 *new = data_race(*orig);366 *new = data_race(*orig);
367 INIT_LIST_HEAD(&new->anon_vma_chain);367 INIT_LIST_HEAD(&new->anon_vma_chain);
368 new->vm_next = new->vm_prev = NULL;368 new->vm_next = new->vm_prev = NULL;
369 dup_vma_anon_name(orig, new);369 dup_anon_vma_name(orig, new);
370 }370 }
371 return new;371 return new;
372}372}
373 373
374void vm_area_free(struct vm_area_struct *vma)374void vm_area_free(struct vm_area_struct *vma)
375{375{
376 free_vma_anon_name(vma);376 free_anon_vma_name(vma);
377 kmem_cache_free(vm_area_cachep, vma);377 kmem_cache_free(vm_area_cachep, vma);
378}378}
379 379
kernel/sys.c+12−7
@@ -7,6 +7,7 @@ · 6 unmodified lines
7 7
8#include <linux/export.h>8#include <linux/export.h>
9#include <linux/mm.h>9#include <linux/mm.h>
10#include <linux/mm_inline.h>
10#include <linux/utsname.h>11#include <linux/utsname.h>
11#include <linux/mman.h>12#include <linux/mman.h>
12#include <linux/reboot.h>13#include <linux/reboot.h>
@@ -2286,15 +2287,16 @@ static int prctl_set_vma(unsigned long opt, unsigned long addr, · 2273 unmodified lines
2286{2287{
2287 struct mm_struct *mm = current->mm;2288 struct mm_struct *mm = current->mm;
2288 const char __user *uname;2289 const char __user *uname;
2289 char *name, *pch;2290 struct anon_vma_name *anon_name = NULL;
2290 int error;2291 int error;
2291 2292
2292 switch (opt) {2293 switch (opt) {
2293 case PR_SET_VMA_ANON_NAME:2294 case PR_SET_VMA_ANON_NAME:
2294 uname = (const char __user *)arg;2295 uname = (const char __user *)arg;
2295 if (uname) {2296 if (uname) {
2296 name = strndup_user(uname, ANON_VMA_NAME_MAX_LEN);2297 char *name, *pch;
2297 2298
2299 name = strndup_user(uname, ANON_VMA_NAME_MAX_LEN);
2298 if (IS_ERR(name))2300 if (IS_ERR(name))
2299 return PTR_ERR(name);2301 return PTR_ERR(name);
2300 2302
@@ -2304,15 +2306,18 @@ static int prctl_set_vma(unsigned long opt, unsigned long addr, · 3 unmodified lines
2304 return -EINVAL;2306 return -EINVAL;
2305 }2307 }
2306 }2308 }
2307 } else {2309 /* anon_vma has its own copy */
2308 /* Reset the name */2310 anon_name = anon_vma_name_alloc(name);
2309 name = NULL;2311 kfree(name);
2312 if (!anon_name)
2313 return -ENOMEM;
2314
2310 }2315 }
2311 2316
2312 mmap_write_lock(mm);2317 mmap_write_lock(mm);
2313 error = madvise_set_anon_name(mm, addr, size, name);2318 error = madvise_set_anon_name(mm, addr, size, anon_name);
2314 mmap_write_unlock(mm);2319 mmap_write_unlock(mm);
2315 kfree(name);2320 anon_vma_name_put(anon_name);
2316 break;2321 break;
2317 default:2322 default:
2318 error = -EINVAL;2323 error = -EINVAL;
mm/madvise.c+34−58
@@ -65,7 +65,7 @@ static int madvise_need_mmap_write(int behavior) · 64 unmodified lines
65}65}
66 66
67#ifdef CONFIG_ANON_VMA_NAME67#ifdef CONFIG_ANON_VMA_NAME
68static struct anon_vma_name *anon_vma_name_alloc(const char *name)68struct anon_vma_name *anon_vma_name_alloc(const char *name)
69{69{
70 struct anon_vma_name *anon_name;70 struct anon_vma_name *anon_name;
71 size_t count;71 size_t count;
@@ -81,78 +81,48 @@ static struct anon_vma_name *anon_vma_name_alloc(const char *name) · 9 unmodified lines
81 return anon_name;81 return anon_name;
82}82}
83 83
84static void vma_anon_name_free(struct kref *kref)84void anon_vma_name_free(struct kref *kref)
85{85{
86 struct anon_vma_name *anon_name =86 struct anon_vma_name *anon_name =
87 container_of(kref, struct anon_vma_name, kref);87 container_of(kref, struct anon_vma_name, kref);
88 kfree(anon_name);88 kfree(anon_name);
89}89}
90 90
91static inline bool has_vma_anon_name(struct vm_area_struct *vma)91struct anon_vma_name *anon_vma_name(struct vm_area_struct *vma)
92{92{
93 return !vma->vm_file && vma->anon_name;
94}
95
96const char *vma_anon_name(struct vm_area_struct *vma)
97{
98 if (!has_vma_anon_name(vma))
99 return NULL;
100
101 mmap_assert_locked(vma->vm_mm);93 mmap_assert_locked(vma->vm_mm);
102 94
103 return vma->anon_name->name;95 if (vma->vm_file)
104}96 return NULL;
105
106void dup_vma_anon_name(struct vm_area_struct *orig_vma,
107 struct vm_area_struct *new_vma)
108{
109 if (!has_vma_anon_name(orig_vma))
110 return;
111
112 kref_get(&orig_vma->anon_name->kref);
113 new_vma->anon_name = orig_vma->anon_name;
114}
115
116void free_vma_anon_name(struct vm_area_struct *vma)
117{
118 struct anon_vma_name *anon_name;
119
120 if (!has_vma_anon_name(vma))
121 return;
122 97
123 anon_name = vma->anon_name;98 return vma->anon_name;
124 vma->anon_name = NULL;
125 kref_put(&anon_name->kref, vma_anon_name_free);
126}99}
127 100
128/* mmap_lock should be write-locked */101/* mmap_lock should be write-locked */
129static int replace_vma_anon_name(struct vm_area_struct *vma, const char *name)102static int replace_anon_vma_name(struct vm_area_struct *vma,
103 struct anon_vma_name *anon_name)
130{104{
131 const char *anon_name;105 struct anon_vma_name *orig_name = anon_vma_name(vma);
132 106
133 if (!name) {107 if (!anon_name) {
134 free_vma_anon_name(vma);108 vma->anon_name = NULL;
109 anon_vma_name_put(orig_name);
135 return 0;110 return 0;
136 }111 }
137 112
138 anon_name = vma_anon_name(vma);113 if (anon_vma_name_eq(orig_name, anon_name))
139 if (anon_name) {114 return 0;
140 /* Same name, nothing to do here */
141 if (!strcmp(name, anon_name))
142 return 0;
143 115
144 free_vma_anon_name(vma);116 vma->anon_name = anon_vma_name_reuse(anon_name);
145 }117 anon_vma_name_put(orig_name);
146 vma->anon_name = anon_vma_name_alloc(name);
147 if (!vma->anon_name)
148 return -ENOMEM;
149 118
150 return 0;119 return 0;
151}120}
152#else /* CONFIG_ANON_VMA_NAME */121#else /* CONFIG_ANON_VMA_NAME */
153static int replace_vma_anon_name(struct vm_area_struct *vma, const char *name)122static int replace_anon_vma_name(struct vm_area_struct *vma,
123 struct anon_vma_name *anon_name)
154{124{
155 if (name)125 if (anon_name)
156 return -EINVAL;126 return -EINVAL;
157 127
158 return 0;128 return 0;
@@ -161,17 +131,19 @@ static int replace_vma_anon_name(struct vm_area_struct *vma, const char *name) · 2 unmodified lines
161/*131/*
162 * Update the vm_flags on region of a vma, splitting it or merging it as132 * Update the vm_flags on region of a vma, splitting it or merging it as
163 * necessary. Must be called with mmap_sem held for writing;133 * necessary. Must be called with mmap_sem held for writing;
134 * Caller should ensure anon_name stability by raising its refcount even when
135 * anon_name belongs to a valid vma because this function might free that vma.
164 */136 */
165static int madvise_update_vma(struct vm_area_struct *vma,137static int madvise_update_vma(struct vm_area_struct *vma,
166 struct vm_area_struct **prev, unsigned long start,138 struct vm_area_struct **prev, unsigned long start,
167 unsigned long end, unsigned long new_flags,139 unsigned long end, unsigned long new_flags,
168 const char *name)140 struct anon_vma_name *anon_name)
169{141{
170 struct mm_struct *mm = vma->vm_mm;142 struct mm_struct *mm = vma->vm_mm;
171 int error;143 int error;
172 pgoff_t pgoff;144 pgoff_t pgoff;
173 145
174 if (new_flags == vma->vm_flags && is_same_vma_anon_name(vma, name)) {146 if (new_flags == vma->vm_flags && anon_vma_name_eq(anon_vma_name(vma), anon_name)) {
175 *prev = vma;147 *prev = vma;
176 return 0;148 return 0;
177 }149 }
@@ -179,7 +151,7 @@ static int madvise_update_vma(struct vm_area_struct *vma, · 1 unmodified line
179 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);151 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
180 *prev = vma_merge(mm, *prev, start, end, new_flags, vma->anon_vma,152 *prev = vma_merge(mm, *prev, start, end, new_flags, vma->anon_vma,
181 vma->vm_file, pgoff, vma_policy(vma),153 vma->vm_file, pgoff, vma_policy(vma),
182 vma->vm_userfaultfd_ctx, name);154 vma->vm_userfaultfd_ctx, anon_name);
183 if (*prev) {155 if (*prev) {
184 vma = *prev;156 vma = *prev;
185 goto success;157 goto success;
@@ -209,7 +181,7 @@ success: · 23 unmodified lines
209 */181 */
210 vma->vm_flags = new_flags;182 vma->vm_flags = new_flags;
211 if (!vma->vm_file) {183 if (!vma->vm_file) {
212 error = replace_vma_anon_name(vma, name);184 error = replace_anon_vma_name(vma, anon_name);
213 if (error)185 if (error)
214 return error;186 return error;
215 }187 }
@@ -975,6 +947,7 @@ static int madvise_vma_behavior(struct vm_area_struct *vma, · 759 unmodified lines
975 unsigned long behavior)947 unsigned long behavior)
976{948{
977 int error;949 int error;
950 struct anon_vma_name *anon_name;
978 unsigned long new_flags = vma->vm_flags;951 unsigned long new_flags = vma->vm_flags;
979 952
980 switch (behavior) {953 switch (behavior) {
@@ -1040,8 +1013,11 @@ static int madvise_vma_behavior(struct vm_area_struct *vma, · 59 unmodified lines
1040 break;1013 break;
1041 }1014 }
1042 1015
1016 anon_name = anon_vma_name(vma);
1017 anon_vma_name_get(anon_name);
1043 error = madvise_update_vma(vma, prev, start, end, new_flags,1018 error = madvise_update_vma(vma, prev, start, end, new_flags,
1044 vma_anon_name(vma));1019 anon_name);
1020 anon_vma_name_put(anon_name);
1045 1021
1046out:1022out:
1047 /*1023 /*
@@ -1225,7 +1201,7 @@ int madvise_walk_vmas(struct mm_struct *mm, unsigned long start, · 177 unmodified lines
1225static int madvise_vma_anon_name(struct vm_area_struct *vma,1201static int madvise_vma_anon_name(struct vm_area_struct *vma,
1226 struct vm_area_struct **prev,1202 struct vm_area_struct **prev,
1227 unsigned long start, unsigned long end,1203 unsigned long start, unsigned long end,
1228 unsigned long name)1204 unsigned long anon_name)
1229{1205{
1230 int error;1206 int error;
1231 1207
@@ -1234,7 +1210,7 @@ static int madvise_vma_anon_name(struct vm_area_struct *vma, · 2 unmodified lines
1234 return -EBADF;1210 return -EBADF;
1235 1211
1236 error = madvise_update_vma(vma, prev, start, end, vma->vm_flags,1212 error = madvise_update_vma(vma, prev, start, end, vma->vm_flags,
1237 (const char *)name);1213 (struct anon_vma_name *)anon_name);
1238 1214
1239 /*1215 /*
1240 * madvise() returns EAGAIN if kernel resources, such as1216 * madvise() returns EAGAIN if kernel resources, such as
@@ -1246,7 +1222,7 @@ static int madvise_vma_anon_name(struct vm_area_struct *vma, · 5 unmodified lines
1246}1222}
1247 1223
1248int madvise_set_anon_name(struct mm_struct *mm, unsigned long start,1224int madvise_set_anon_name(struct mm_struct *mm, unsigned long start,
1249 unsigned long len_in, const char *name)1225 unsigned long len_in, struct anon_vma_name *anon_name)
1250{1226{
1251 unsigned long end;1227 unsigned long end;
1252 unsigned long len;1228 unsigned long len;
@@ -1266,7 +1242,7 @@ int madvise_set_anon_name(struct mm_struct *mm, unsigned long start, · 13 unmodified lines
1266 if (end == start)1242 if (end == start)
1267 return 0;1243 return 0;
1268 1244
1269 return madvise_walk_vmas(mm, start, end, (unsigned long)name,1245 return madvise_walk_vmas(mm, start, end, (unsigned long)anon_name,
1270 madvise_vma_anon_name);1246 madvise_vma_anon_name);
1271}1247}
1272#endif /* CONFIG_ANON_VMA_NAME */1248#endif /* CONFIG_ANON_VMA_NAME */
mm/memfd.c+28−12
@@ -31,20 +31,28 @@ · 30 unmodified lines
31static void memfd_tag_pins(struct xa_state *xas)31static void memfd_tag_pins(struct xa_state *xas)
32{32{
33 struct page *page;33 struct page *page;
34 unsigned int tagged = 0;34 int latency = 0;
35 int cache_count;
35 36
36 lru_add_drain();37 lru_add_drain();
37 38
38 xas_lock_irq(xas);39 xas_lock_irq(xas);
39 xas_for_each(xas, page, ULONG_MAX) {40 xas_for_each(xas, page, ULONG_MAX) {
40 if (xa_is_value(page))41 cache_count = 1;
41 continue;42 if (!xa_is_value(page) &&
42 page = find_subpage(page, xas->xa_index);43 PageTransHuge(page) && !PageHuge(page))
43 if (page_count(page) - page_mapcount(page) > 1)44 cache_count = HPAGE_PMD_NR;
45
46 if (!xa_is_value(page) &&
47 page_count(page) - total_mapcount(page) != cache_count)
44 xas_set_mark(xas, MEMFD_TAG_PINNED);48 xas_set_mark(xas, MEMFD_TAG_PINNED);
49 if (cache_count != 1)
50 xas_set(xas, page->index + cache_count);
45 51
46 if (++tagged % XA_CHECK_SCHED)52 latency += cache_count;
53 if (latency < XA_CHECK_SCHED)
47 continue;54 continue;
55 latency = 0;
48 56
49 xas_pause(xas);57 xas_pause(xas);
50 xas_unlock_irq(xas);58 xas_unlock_irq(xas);
@@ -73,7 +81,8 @@ static int memfd_wait_for_pins(struct address_space *mapping) · 22 unmodified lines
73 81
74 error = 0;82 error = 0;
75 for (scan = 0; scan <= LAST_SCAN; scan++) {83 for (scan = 0; scan <= LAST_SCAN; scan++) {
76 unsigned int tagged = 0;84 int latency = 0;
85 int cache_count;
77 86
78 if (!xas_marked(&xas, MEMFD_TAG_PINNED))87 if (!xas_marked(&xas, MEMFD_TAG_PINNED))
79 break;88 break;
@@ -87,10 +96,14 @@ static int memfd_wait_for_pins(struct address_space *mapping) · 7 unmodified lines
87 xas_lock_irq(&xas);96 xas_lock_irq(&xas);
88 xas_for_each_marked(&xas, page, ULONG_MAX, MEMFD_TAG_PINNED) {97 xas_for_each_marked(&xas, page, ULONG_MAX, MEMFD_TAG_PINNED) {
89 bool clear = true;98 bool clear = true;
90 if (xa_is_value(page))99
91 continue;100 cache_count = 1;
92 page = find_subpage(page, xas.xa_index);101 if (!xa_is_value(page) &&
93 if (page_count(page) - page_mapcount(page) != 1) {102 PageTransHuge(page) && !PageHuge(page))
103 cache_count = HPAGE_PMD_NR;
104
105 if (!xa_is_value(page) && cache_count !=
106 page_count(page) - total_mapcount(page)) {
94 /*107 /*
95 * On the last scan, we clean up all those tags108 * On the last scan, we clean up all those tags
96 * we inserted; but make a note that we still109 * we inserted; but make a note that we still
@@ -103,8 +116,11 @@ static int memfd_wait_for_pins(struct address_space *mapping) · 6 unmodified lines
103 }116 }
104 if (clear)117 if (clear)
105 xas_clear_mark(&xas, MEMFD_TAG_PINNED);118 xas_clear_mark(&xas, MEMFD_TAG_PINNED);
106 if (++tagged % XA_CHECK_SCHED)119
120 latency += cache_count;
121 if (latency < XA_CHECK_SCHED)
107 continue;122 continue;
123 latency = 0;
108 124
109 xas_pause(&xas);125 xas_pause(&xas);
110 xas_unlock_irq(&xas);126 xas_unlock_irq(&xas);
mm/mempolicy.c+1−1
@@ -814,7 +814,7 @@ static int mbind_range(struct mm_struct *mm, unsigned long start, · 813 unmodified lines
814 prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags,814 prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags,
815 vma->anon_vma, vma->vm_file, pgoff,815 vma->anon_vma, vma->vm_file, pgoff,
816 new_pol, vma->vm_userfaultfd_ctx,816 new_pol, vma->vm_userfaultfd_ctx,
817 vma_anon_name(vma));817 anon_vma_name(vma));
818 if (prev) {818 if (prev) {
819 vma = prev;819 vma = prev;
820 next = vma->vm_next;820 next = vma->vm_next;
mm/mlock.c+1−1
@@ -512,7 +512,7 @@ static int mlock_fixup(struct vm_area_struct *vma, struct vm_area_struct **prev, · 511 unmodified lines
512 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);512 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
513 *prev = vma_merge(mm, *prev, start, end, newflags, vma->anon_vma,513 *prev = vma_merge(mm, *prev, start, end, newflags, vma->anon_vma,
514 vma->vm_file, pgoff, vma_policy(vma),514 vma->vm_file, pgoff, vma_policy(vma),
515 vma->vm_userfaultfd_ctx, vma_anon_name(vma));515 vma->vm_userfaultfd_ctx, anon_vma_name(vma));
516 if (*prev) {516 if (*prev) {
517 vma = *prev;517 vma = *prev;
518 goto success;518 goto success;
mm/mmap.c+6−6
@@ -1031,7 +1031,7 @@ again: · 1030 unmodified lines
1031static inline int is_mergeable_vma(struct vm_area_struct *vma,1031static inline int is_mergeable_vma(struct vm_area_struct *vma,
1032 struct file *file, unsigned long vm_flags,1032 struct file *file, unsigned long vm_flags,
1033 struct vm_userfaultfd_ctx vm_userfaultfd_ctx,1033 struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
1034 const char *anon_name)1034 struct anon_vma_name *anon_name)
1035{1035{
1036 /*1036 /*
1037 * VM_SOFTDIRTY should not prevent from VMA merging, if we1037 * VM_SOFTDIRTY should not prevent from VMA merging, if we
@@ -1049,7 +1049,7 @@ static inline int is_mergeable_vma(struct vm_area_struct *vma, · 11 unmodified lines
1049 return 0;1049 return 0;
1050 if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx))1050 if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx))
1051 return 0;1051 return 0;
1052 if (!is_same_vma_anon_name(vma, anon_name))1052 if (!anon_vma_name_eq(anon_vma_name(vma), anon_name))
1053 return 0;1053 return 0;
1054 return 1;1054 return 1;
1055}1055}
@@ -1084,7 +1084,7 @@ can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags, · 28 unmodified lines
1084 struct anon_vma *anon_vma, struct file *file,1084 struct anon_vma *anon_vma, struct file *file,
1085 pgoff_t vm_pgoff,1085 pgoff_t vm_pgoff,
1086 struct vm_userfaultfd_ctx vm_userfaultfd_ctx,1086 struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
1087 const char *anon_name)1087 struct anon_vma_name *anon_name)
1088{1088{
1089 if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name) &&1089 if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name) &&
1090 is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {1090 is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
@@ -1106,7 +1106,7 @@ can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags, · 15 unmodified lines
1106 struct anon_vma *anon_vma, struct file *file,1106 struct anon_vma *anon_vma, struct file *file,
1107 pgoff_t vm_pgoff,1107 pgoff_t vm_pgoff,
1108 struct vm_userfaultfd_ctx vm_userfaultfd_ctx,1108 struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
1109 const char *anon_name)1109 struct anon_vma_name *anon_name)
1110{1110{
1111 if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name) &&1111 if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name) &&
1112 is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {1112 is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
@@ -1167,7 +1167,7 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm, · 54 unmodified lines
1167 struct anon_vma *anon_vma, struct file *file,1167 struct anon_vma *anon_vma, struct file *file,
1168 pgoff_t pgoff, struct mempolicy *policy,1168 pgoff_t pgoff, struct mempolicy *policy,
1169 struct vm_userfaultfd_ctx vm_userfaultfd_ctx,1169 struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
1170 const char *anon_name)1170 struct anon_vma_name *anon_name)
1171{1171{
1172 pgoff_t pglen = (end - addr) >> PAGE_SHIFT;1172 pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
1173 struct vm_area_struct *area, *next;1173 struct vm_area_struct *area, *next;
@@ -3256,7 +3256,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, · 2082 unmodified lines
3256 return NULL; /* should never get here */3256 return NULL; /* should never get here */
3257 new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags,3257 new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags,
3258 vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),3258 vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
3259 vma->vm_userfaultfd_ctx, vma_anon_name(vma));3259 vma->vm_userfaultfd_ctx, anon_vma_name(vma));
3260 if (new_vma) {3260 if (new_vma) {
3261 /*3261 /*
3262 * Source vma may have been merged into new_vma3262 * Source vma may have been merged into new_vma
mm/mprotect.c+1−1
@@ -464,7 +464,7 @@ mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev, · 463 unmodified lines
464 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);464 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
465 *pprev = vma_merge(mm, *pprev, start, end, newflags,465 *pprev = vma_merge(mm, *pprev, start, end, newflags,
466 vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),466 vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
467 vma->vm_userfaultfd_ctx, vma_anon_name(vma));467 vma->vm_userfaultfd_ctx, anon_vma_name(vma));
468 if (*pprev) {468 if (*pprev) {
469 vma = *pprev;469 vma = *pprev;
470 VM_WARN_ON((vma->vm_flags ^ newflags) & ~VM_SOFTDIRTY);470 VM_WARN_ON((vma->vm_flags ^ newflags) & ~VM_SOFTDIRTY);
tools/testing/selftests/vm/hugepage-mremap.c+19−7
@@ -3,9 +3,10 @@ · 2 unmodified lines
3 * hugepage-mremap:3 * hugepage-mremap:
4 *4 *
5 * Example of remapping huge page memory in a user application using the5 * Example of remapping huge page memory in a user application using the
6 * mremap system call. Code assumes a hugetlbfs filesystem is mounted6 * mremap system call. The path to a file in a hugetlbfs filesystem must
7 * at './huge'. The amount of memory used by this test is decided by a command7 * be passed as the last argument to this test. The amount of memory used
8 * line argument in MBs. If missing, the default amount is 10MB.8 * by this test in MBs can optionally be passed as an argument. If no memory
9 * amount is passed, the default amount is 10MB.
9 *10 *
10 * To make sure the test triggers pmd sharing and goes through the 'unshare'11 * To make sure the test triggers pmd sharing and goes through the 'unshare'
11 * path in the mremap code use 1GB (1024) or more.12 * path in the mremap code use 1GB (1024) or more.
@@ -25,7 +26,6 @@ · 13 unmodified lines
25#define DEFAULT_LENGTH_MB 10UL26#define DEFAULT_LENGTH_MB 10UL
26#define MB_TO_BYTES(x) (x * 1024 * 1024)27#define MB_TO_BYTES(x) (x * 1024 * 1024)
27 28
28#define FILE_NAME "huge/hugepagefile"
29#define PROTECTION (PROT_READ | PROT_WRITE | PROT_EXEC)29#define PROTECTION (PROT_READ | PROT_WRITE | PROT_EXEC)
30#define FLAGS (MAP_SHARED | MAP_ANONYMOUS)30#define FLAGS (MAP_SHARED | MAP_ANONYMOUS)
31 31
@@ -107,17 +107,26 @@ static void register_region_with_uffd(char *addr, size_t len) · 75 unmodified lines
107 107
108int main(int argc, char *argv[])108int main(int argc, char *argv[])
109{109{
110 size_t length;
111
112 if (argc != 2 && argc != 3) {
113 printf("Usage: %s [length_in_MB] <hugetlb_file>\n", argv[0]);
114 exit(1);
115 }
116
110 /* Read memory length as the first arg if valid, otherwise fallback to117 /* Read memory length as the first arg if valid, otherwise fallback to
111 * the default length. Any additional args are ignored.118 * the default length.
112 */119 */
113 size_t length = argc > 1 ? (size_t)atoi(argv[1]) : 0UL;120 if (argc == 3)
121 length = argc > 2 ? (size_t)atoi(argv[1]) : 0UL;
114 122
115 length = length > 0 ? length : DEFAULT_LENGTH_MB;123 length = length > 0 ? length : DEFAULT_LENGTH_MB;
116 length = MB_TO_BYTES(length);124 length = MB_TO_BYTES(length);
117 125
118 int ret = 0;126 int ret = 0;
119 127
120 int fd = open(FILE_NAME, O_CREAT | O_RDWR, 0755);128 /* last arg is the hugetlb file name */
129 int fd = open(argv[argc-1], O_CREAT | O_RDWR, 0755);
121 130
122 if (fd < 0) {131 if (fd < 0) {
123 perror("Open failed");132 perror("Open failed");
@@ -169,5 +178,8 @@ int main(int argc, char *argv[]) · 45 unmodified lines
169 178
170 munmap(addr, length);179 munmap(addr, length);
171 180
181 close(fd);
182 unlink(argv[argc-1]);
183
172 return ret;184 return ret;
173}185}
tools/testing/selftests/vm/run_vmtests.sh+2−1
@@ -111,13 +111,14 @@ fi · 110 unmodified lines
111echo "-----------------------"111echo "-----------------------"
112echo "running hugepage-mremap"112echo "running hugepage-mremap"
113echo "-----------------------"113echo "-----------------------"
114./hugepage-mremap 256114./hugepage-mremap $mnt/huge_mremap
115if [ $? -ne 0 ]; then115if [ $? -ne 0 ]; then
116 echo "[FAIL]"116 echo "[FAIL]"
117 exitcode=1117 exitcode=1
118else118else
119 echo "[PASS]"119 echo "[PASS]"
120fi120fi
121rm -f $mnt/huge_mremap
121 122
122echo "NOTE: The above hugetlb tests provide minimal coverage. Use"123echo "NOTE: The above hugetlb tests provide minimal coverage. Use"
123echo " https://github.com/libhugetlbfs/libhugetlbfs.git for"124echo " https://github.com/libhugetlbfs/libhugetlbfs.git for"
tools/testing/selftests/vm/userfaultfd.c+1−0
@@ -46,6 +46,7 @@ · 45 unmodified lines
46#include <signal.h>46#include <signal.h>
47#include <poll.h>47#include <poll.h>
48#include <string.h>48#include <string.h>
49#include <linux/mman.h>
49#include <sys/mman.h>50#include <sys/mman.h>
50#include <sys/syscall.h>51#include <sys/syscall.h>
51#include <sys/ioctl.h>52#include <sys/ioctl.h>