History

Merge tag 'mm-hotfixes-stable-2026-01-29-09-41' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm

Pull misc fixes from Andrew Morton: "16 hotfixes. 9 are cc:stable, 12 are for MM. There's a patch series from Pratyush Yadav which fixes a few things in the new-in-6.19 LUO memfd code. Plus the usual shower of singletons - please see the changelogs for details" * tag 'mm-hotfixes-stable-2026-01-29-09-41' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm: vmcoreinfo: make hwerr_data visible for debugging mm/zone_device: reinitialize large zone device private folios mm/mm_init: don't cond_resched() in deferred_init_memmap_chunk() if called from deferred_grow_zone() mm/kfence: randomize the freelist on initialization kho: kho_preserve_vmalloc(): don't return 0 when ENOMEM kho: init alloc tags when restoring pages from reserved memory mm: memfd_luo: restore and free memfd_luo_ser on failure mm: memfd_luo: use memfd_alloc_file() instead of shmem_file_setup() memfd: export alloc_file() flex_proportions: make fprop_new_period() hardirq safe mailmap: add entry for Viacheslav Bocharov mm/memory-failure: teach kill_accessing_process to accept hugetlb tail page pfn mm/memory-failure: fix missing ->mf_stats count in hugetlb poison mm, swap: restore swap_space attr aviod kernel panic mm/kasan: fix KASAN poisoning in vrealloc() mm/shmem, swap: fix race of truncate and swap entry split

Linus Torvalds committed 9 months ago

Commit bcb60582 parents 1cac389 bd5878223 files · +239 −87Browse files

Unified · Split

23 files
  1. .mailmap +1 −0
  2. arch/powerpc/kvm/book3s_hv_uvmem.c +1 −1
  3. drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +1 −1
  4. drivers/gpu/drm/drm_pagemap.c +1 −1
  5. drivers/gpu/drm/nouveau/nouveau_dmem.c +1 −1
  6. include/linux/kasan.h +14 −0
  7. include/linux/memfd.h +6 −0
  8. include/linux/memremap.h +6 −3
  9. kernel/liveupdate/kexec_handover.c +11 −1
  10. kernel/vmcore_info.c +5 −1
  11. lib/flex_proportions.c +3 −2
  12. lib/test_hmm.c +3 −1
  13. mm/kasan/common.c +21 −0
  14. mm/kfence/core.c +19 −4
  15. mm/memfd.c +2 −2
  16. mm/memfd_luo.c +6 −4
  17. mm/memory-failure.c +60 −39
  18. mm/memremap.c +34 −1
  19. mm/mm_init.c +6 −6
  20. mm/shmem.c +34 −11
  21. mm/swap.h +1 −1
  22. mm/swap_state.c +1 −2
  23. mm/vmalloc.c +2 −5
.mailmap+1−0
@@ -851,6 +851,7 @@ Valentin Schneider <vschneid@redhat.com> <valentin.schneider@arm.com> · 850 unmodified lines
851Veera Sundaram Sankaran <quic_veeras@quicinc.com> <veeras@codeaurora.org>851Veera Sundaram Sankaran <quic_veeras@quicinc.com> <veeras@codeaurora.org>
852Veerabhadrarao Badiganti <quic_vbadigan@quicinc.com> <vbadigan@codeaurora.org>852Veerabhadrarao Badiganti <quic_vbadigan@quicinc.com> <vbadigan@codeaurora.org>
853Venkateswara Naralasetty <quic_vnaralas@quicinc.com> <vnaralas@codeaurora.org>853Venkateswara Naralasetty <quic_vnaralas@quicinc.com> <vnaralas@codeaurora.org>
854Viacheslav Bocharov <v@baodeep.com> <adeep@lexina.in>
854Vikash Garodia <vikash.garodia@oss.qualcomm.com> <vgarodia@codeaurora.org>855Vikash Garodia <vikash.garodia@oss.qualcomm.com> <vgarodia@codeaurora.org>
855Vikash Garodia <vikash.garodia@oss.qualcomm.com> <quic_vgarodia@quicinc.com>856Vikash Garodia <vikash.garodia@oss.qualcomm.com> <quic_vgarodia@quicinc.com>
856Vincent Mailhol <mailhol@kernel.org> <mailhol.vincent@wanadoo.fr>857Vincent Mailhol <mailhol@kernel.org> <mailhol.vincent@wanadoo.fr>
arch/powerpc/kvm/book3s_hv_uvmem.c+1−1
@@ -723,7 +723,7 @@ static struct page *kvmppc_uvmem_get_page(unsigned long gpa, struct kvm *kvm) · 722 unmodified lines
723 723
724 dpage = pfn_to_page(uvmem_pfn);724 dpage = pfn_to_page(uvmem_pfn);
725 dpage->zone_device_data = pvt;725 dpage->zone_device_data = pvt;
726 zone_device_page_init(dpage, 0);726 zone_device_page_init(dpage, &kvmppc_uvmem_pgmap, 0);
727 return dpage;727 return dpage;
728out_clear:728out_clear:
729 spin_lock(&kvmppc_uvmem_bitmap_lock);729 spin_lock(&kvmppc_uvmem_bitmap_lock);
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c+1−1
@@ -217,7 +217,7 @@ svm_migrate_get_vram_page(struct svm_range *prange, unsigned long pfn) · 216 unmodified lines
217 page = pfn_to_page(pfn);217 page = pfn_to_page(pfn);
218 svm_range_bo_ref(prange->svm_bo);218 svm_range_bo_ref(prange->svm_bo);
219 page->zone_device_data = prange->svm_bo;219 page->zone_device_data = prange->svm_bo;
220 zone_device_page_init(page, 0);220 zone_device_page_init(page, page_pgmap(page), 0);
221}221}
222 222
223static void223static void
drivers/gpu/drm/drm_pagemap.c+1−1
@@ -197,7 +197,7 @@ static void drm_pagemap_get_devmem_page(struct page *page, · 196 unmodified lines
197 struct drm_pagemap_zdd *zdd)197 struct drm_pagemap_zdd *zdd)
198{198{
199 page->zone_device_data = drm_pagemap_zdd_get(zdd);199 page->zone_device_data = drm_pagemap_zdd_get(zdd);
200 zone_device_page_init(page, 0);200 zone_device_page_init(page, page_pgmap(page), 0);
201}201}
202 202
203/**203/**
drivers/gpu/drm/nouveau/nouveau_dmem.c+1−1
@@ -425,7 +425,7 @@ nouveau_dmem_page_alloc_locked(struct nouveau_drm *drm, bool is_large) · 424 unmodified lines
425 order = ilog2(DMEM_CHUNK_NPAGES);425 order = ilog2(DMEM_CHUNK_NPAGES);
426 }426 }
427 427
428 zone_device_folio_init(folio, order);428 zone_device_folio_init(folio, page_pgmap(folio_page(folio, 0)), order);
429 return page;429 return page;
430}430}
431 431
include/linux/kasan.h+14−0
@@ -641,6 +641,17 @@ kasan_unpoison_vmap_areas(struct vm_struct **vms, int nr_vms, · 640 unmodified lines
641 __kasan_unpoison_vmap_areas(vms, nr_vms, flags);641 __kasan_unpoison_vmap_areas(vms, nr_vms, flags);
642}642}
643 643
644void __kasan_vrealloc(const void *start, unsigned long old_size,
645 unsigned long new_size);
646
647static __always_inline void kasan_vrealloc(const void *start,
648 unsigned long old_size,
649 unsigned long new_size)
650{
651 if (kasan_enabled())
652 __kasan_vrealloc(start, old_size, new_size);
653}
654
644#else /* CONFIG_KASAN_VMALLOC */655#else /* CONFIG_KASAN_VMALLOC */
645 656
646static inline void kasan_populate_early_vm_area_shadow(void *start,657static inline void kasan_populate_early_vm_area_shadow(void *start,
@@ -670,6 +681,9 @@ kasan_unpoison_vmap_areas(struct vm_struct **vms, int nr_vms, · 23 unmodified lines
670 kasan_vmalloc_flags_t flags)681 kasan_vmalloc_flags_t flags)
671{ }682{ }
672 683
684static inline void kasan_vrealloc(const void *start, unsigned long old_size,
685 unsigned long new_size) { }
686
673#endif /* CONFIG_KASAN_VMALLOC */687#endif /* CONFIG_KASAN_VMALLOC */
674 688
675#if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \689#if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \
include/linux/memfd.h+6−0
@@ -17,6 +17,7 @@ struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t idx); · 16 unmodified lines
17 * to by vm_flags_ptr.17 * to by vm_flags_ptr.
18 */18 */
19int memfd_check_seals_mmap(struct file *file, vm_flags_t *vm_flags_ptr);19int memfd_check_seals_mmap(struct file *file, vm_flags_t *vm_flags_ptr);
20struct file *memfd_alloc_file(const char *name, unsigned int flags);
20#else21#else
21static inline long memfd_fcntl(struct file *f, unsigned int c, unsigned int a)22static inline long memfd_fcntl(struct file *f, unsigned int c, unsigned int a)
22{23{
@@ -31,6 +32,11 @@ static inline int memfd_check_seals_mmap(struct file *file, · 8 unmodified lines
31{32{
32 return 0;33 return 0;
33}34}
35
36static inline struct file *memfd_alloc_file(const char *name, unsigned int flags)
37{
38 return ERR_PTR(-EINVAL);
39}
34#endif40#endif
35 41
36#endif /* __LINUX_MEMFD_H */42#endif /* __LINUX_MEMFD_H */
include/linux/memremap.h+6−3
@@ -224,7 +224,8 @@ static inline bool is_fsdax_page(const struct page *page) · 223 unmodified lines
224}224}
225 225
226#ifdef CONFIG_ZONE_DEVICE226#ifdef CONFIG_ZONE_DEVICE
227void zone_device_page_init(struct page *page, unsigned int order);227void zone_device_page_init(struct page *page, struct dev_pagemap *pgmap,
228 unsigned int order);
228void *memremap_pages(struct dev_pagemap *pgmap, int nid);229void *memremap_pages(struct dev_pagemap *pgmap, int nid);
229void memunmap_pages(struct dev_pagemap *pgmap);230void memunmap_pages(struct dev_pagemap *pgmap);
230void *devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap);231void *devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap);
@@ -234,9 +235,11 @@ bool pgmap_pfn_valid(struct dev_pagemap *pgmap, unsigned long pfn); · 3 unmodified lines
234 235
235unsigned long memremap_compat_align(void);236unsigned long memremap_compat_align(void);
236 237
237static inline void zone_device_folio_init(struct folio *folio, unsigned int order)238static inline void zone_device_folio_init(struct folio *folio,
239 struct dev_pagemap *pgmap,
240 unsigned int order)
238{241{
239 zone_device_page_init(&folio->page, order);242 zone_device_page_init(&folio->page, pgmap, order);
240 if (order)243 if (order)
241 folio_set_large_rmappable(folio);244 folio_set_large_rmappable(folio);
242}245}
kernel/liveupdate/kexec_handover.c+11−1
@@ -255,6 +255,14 @@ static struct page *kho_restore_page(phys_addr_t phys, bool is_folio) · 254 unmodified lines
255 if (is_folio && info.order)255 if (is_folio && info.order)
256 prep_compound_page(page, info.order);256 prep_compound_page(page, info.order);
257 257
258 /* Always mark headpage's codetag as empty to avoid accounting mismatch */
259 clear_page_tag_ref(page);
260 if (!is_folio) {
261 /* Also do that for the non-compound tail pages */
262 for (unsigned int i = 1; i < nr_pages; i++)
263 clear_page_tag_ref(page + i);
264 }
265
258 adjust_managed_page_count(page, nr_pages);266 adjust_managed_page_count(page, nr_pages);
259 return page;267 return page;
260}268}
@@ -1006,8 +1014,10 @@ int kho_preserve_vmalloc(void *ptr, struct kho_vmalloc *preservation) · 745 unmodified lines
1006 chunk->phys[idx++] = phys;1014 chunk->phys[idx++] = phys;
1007 if (idx == ARRAY_SIZE(chunk->phys)) {1015 if (idx == ARRAY_SIZE(chunk->phys)) {
1008 chunk = new_vmalloc_chunk(chunk);1016 chunk = new_vmalloc_chunk(chunk);
1009 if (!chunk)1017 if (!chunk) {
1018 err = -ENOMEM;
1010 goto err_free;1019 goto err_free;
1020 }
1011 idx = 0;1021 idx = 0;
1012 }1022 }
1013 }1023 }
kernel/vmcore_info.c+5−1
@@ -36,7 +36,11 @@ struct hwerr_info { · 35 unmodified lines
36 time64_t timestamp;36 time64_t timestamp;
37};37};
38 38
39static struct hwerr_info hwerr_data[HWERR_RECOV_MAX];39/*
40 * The hwerr_data[] array is declared with global scope so that it remains
41 * accessible to vmcoreinfo even when Link Time Optimization (LTO) is enabled.
42 */
43struct hwerr_info hwerr_data[HWERR_RECOV_MAX];
40 44
41Elf_Word *append_elf_note(Elf_Word *buf, char *name, unsigned int type,45Elf_Word *append_elf_note(Elf_Word *buf, char *name, unsigned int type,
42 void *data, size_t data_len)46 void *data, size_t data_len)
lib/flex_proportions.c+3−2
@@ -64,13 +64,14 @@ void fprop_global_destroy(struct fprop_global *p) · 63 unmodified lines
64bool fprop_new_period(struct fprop_global *p, int periods)64bool fprop_new_period(struct fprop_global *p, int periods)
65{65{
66 s64 events = percpu_counter_sum(&p->events);66 s64 events = percpu_counter_sum(&p->events);
67 unsigned long flags;
67 68
68 /*69 /*
69 * Don't do anything if there are no events.70 * Don't do anything if there are no events.
70 */71 */
71 if (events <= 1)72 if (events <= 1)
72 return false;73 return false;
73 preempt_disable_nested();74 local_irq_save(flags);
74 write_seqcount_begin(&p->sequence);75 write_seqcount_begin(&p->sequence);
75 if (periods < 64)76 if (periods < 64)
76 events -= events >> periods;77 events -= events >> periods;
@@ -78,7 +79,7 @@ bool fprop_new_period(struct fprop_global *p, int periods) · 1 unmodified line
78 percpu_counter_add(&p->events, -events);79 percpu_counter_add(&p->events, -events);
79 p->period += periods;80 p->period += periods;
80 write_seqcount_end(&p->sequence);81 write_seqcount_end(&p->sequence);
81 preempt_enable_nested();82 local_irq_restore(flags);
82 83
83 return true;84 return true;
84}85}
lib/test_hmm.c+3−1
@@ -662,7 +662,9 @@ static struct page *dmirror_devmem_alloc_page(struct dmirror *dmirror, · 661 unmodified lines
662 goto error;662 goto error;
663 }663 }
664 664
665 zone_device_folio_init(page_folio(dpage), order);665 zone_device_folio_init(page_folio(dpage),
666 page_pgmap(folio_page(page_folio(dpage), 0)),
667 order);
666 dpage->zone_device_data = rpage;668 dpage->zone_device_data = rpage;
667 return dpage;669 return dpage;
668 670
mm/kasan/common.c+21−0
@@ -606,4 +606,25 @@ void __kasan_unpoison_vmap_areas(struct vm_struct **vms, int nr_vms, · 605 unmodified lines
606 __kasan_unpoison_vmalloc(addr, size, flags | KASAN_VMALLOC_KEEP_TAG);606 __kasan_unpoison_vmalloc(addr, size, flags | KASAN_VMALLOC_KEEP_TAG);
607 }607 }
608}608}
609
610void __kasan_vrealloc(const void *addr, unsigned long old_size,
611 unsigned long new_size)
612{
613 if (new_size < old_size) {
614 kasan_poison_last_granule(addr, new_size);
615
616 new_size = round_up(new_size, KASAN_GRANULE_SIZE);
617 old_size = round_up(old_size, KASAN_GRANULE_SIZE);
618 if (new_size < old_size)
619 __kasan_poison_vmalloc(addr + new_size,
620 old_size - new_size);
621 } else if (new_size > old_size) {
622 old_size = round_down(old_size, KASAN_GRANULE_SIZE);
623 __kasan_unpoison_vmalloc(addr + old_size,
624 new_size - old_size,
625 KASAN_VMALLOC_PROT_NORMAL |
626 KASAN_VMALLOC_VM_ALLOC |
627 KASAN_VMALLOC_KEEP_TAG);
628 }
629}
609#endif630#endif
mm/kfence/core.c+19−4
@@ -596,7 +596,7 @@ static void rcu_guarded_free(struct rcu_head *h) · 595 unmodified lines
596static unsigned long kfence_init_pool(void)596static unsigned long kfence_init_pool(void)
597{597{
598 unsigned long addr, start_pfn;598 unsigned long addr, start_pfn;
599 int i;599 int i, rand;
600 600
601 if (!arch_kfence_init_pool())601 if (!arch_kfence_init_pool())
602 return (unsigned long)__kfence_pool;602 return (unsigned long)__kfence_pool;
@@ -647,13 +647,27 @@ static unsigned long kfence_init_pool(void) · 44 unmodified lines
647 INIT_LIST_HEAD(&meta->list);647 INIT_LIST_HEAD(&meta->list);
648 raw_spin_lock_init(&meta->lock);648 raw_spin_lock_init(&meta->lock);
649 meta->state = KFENCE_OBJECT_UNUSED;649 meta->state = KFENCE_OBJECT_UNUSED;
650 meta->addr = addr; /* Initialize for validation in metadata_to_pageaddr(). */650 /* Use addr to randomize the freelist. */
651 list_add_tail(&meta->list, &kfence_freelist);651 meta->addr = i;
652 652
653 /* Protect the right redzone. */653 /* Protect the right redzone. */
654 if (unlikely(!kfence_protect(addr + PAGE_SIZE)))654 if (unlikely(!kfence_protect(addr + 2 * i * PAGE_SIZE + PAGE_SIZE)))
655 goto reset_slab;655 goto reset_slab;
656 }
657
658 for (i = CONFIG_KFENCE_NUM_OBJECTS; i > 0; i--) {
659 rand = get_random_u32_below(i);
660 swap(kfence_metadata_init[i - 1].addr, kfence_metadata_init[rand].addr);
661 }
656 662
663 for (i = 0; i < CONFIG_KFENCE_NUM_OBJECTS; i++) {
664 struct kfence_metadata *meta_1 = &kfence_metadata_init[i];
665 struct kfence_metadata *meta_2 = &kfence_metadata_init[meta_1->addr];
666
667 list_add_tail(&meta_2->list, &kfence_freelist);
668 }
669 for (i = 0; i < CONFIG_KFENCE_NUM_OBJECTS; i++) {
670 kfence_metadata_init[i].addr = addr;
657 addr += 2 * PAGE_SIZE;671 addr += 2 * PAGE_SIZE;
658 }672 }
659 673
@@ -666,6 +680,7 @@ static unsigned long kfence_init_pool(void) · 6 unmodified lines
666 return 0;680 return 0;
667 681
668reset_slab:682reset_slab:
683 addr += 2 * i * PAGE_SIZE;
669 for (i = 0; i < KFENCE_POOL_SIZE / PAGE_SIZE; i++) {684 for (i = 0; i < KFENCE_POOL_SIZE / PAGE_SIZE; i++) {
670 struct page *page;685 struct page *page;
671 686
mm/memfd.c+2−2
@@ -456,7 +456,7 @@ err_name: · 455 unmodified lines
456 return ERR_PTR(error);456 return ERR_PTR(error);
457}457}
458 458
459static struct file *alloc_file(const char *name, unsigned int flags)459struct file *memfd_alloc_file(const char *name, unsigned int flags)
460{460{
461 unsigned int *file_seals;461 unsigned int *file_seals;
462 struct file *file;462 struct file *file;
@@ -520,5 +520,5 @@ SYSCALL_DEFINE2(memfd_create, · 57 unmodified lines
520 return PTR_ERR(name);520 return PTR_ERR(name);
521 521
522 fd_flags = (flags & MFD_CLOEXEC) ? O_CLOEXEC : 0;522 fd_flags = (flags & MFD_CLOEXEC) ? O_CLOEXEC : 0;
523 return FD_ADD(fd_flags, alloc_file(name, flags));523 return FD_ADD(fd_flags, memfd_alloc_file(name, flags));
524}524}
mm/memfd_luo.c+6−4
@@ -78,6 +78,7 @@ · 77 unmodified lines
78#include <linux/liveupdate.h>78#include <linux/liveupdate.h>
79#include <linux/shmem_fs.h>79#include <linux/shmem_fs.h>
80#include <linux/vmalloc.h>80#include <linux/vmalloc.h>
81#include <linux/memfd.h>
81#include "internal.h"82#include "internal.h"
82 83
83static int memfd_luo_preserve_folios(struct file *file,84static int memfd_luo_preserve_folios(struct file *file,
@@ -443,11 +444,11 @@ static int memfd_luo_retrieve(struct liveupdate_file_op_args *args) · 359 unmodified lines
443 if (!ser)444 if (!ser)
444 return -EINVAL;445 return -EINVAL;
445 446
446 file = shmem_file_setup("", 0, VM_NORESERVE);447 file = memfd_alloc_file("", 0);
447
448 if (IS_ERR(file)) {448 if (IS_ERR(file)) {
449 pr_err("failed to setup file: %pe\n", file);449 pr_err("failed to setup file: %pe\n", file);
450 return PTR_ERR(file);450 err = PTR_ERR(file);
451 goto free_ser;
451 }452 }
452 453
453 vfs_setpos(file, ser->pos, MAX_LFS_FILESIZE);454 vfs_setpos(file, ser->pos, MAX_LFS_FILESIZE);
@@ -473,7 +474,8 @@ static int memfd_luo_retrieve(struct liveupdate_file_op_args *args) · 19 unmodified lines
473 474
474put_file:475put_file:
475 fput(file);476 fput(file);
476 477free_ser:
478 kho_restore_free(ser);
477 return err;479 return err;
478}480}
479 481
mm/memory-failure.c+60−39
@@ -692,6 +692,8 @@ static int check_hwpoisoned_entry(pte_t pte, unsigned long addr, short shift, · 691 unmodified lines
692 unsigned long poisoned_pfn, struct to_kill *tk)692 unsigned long poisoned_pfn, struct to_kill *tk)
693{693{
694 unsigned long pfn = 0;694 unsigned long pfn = 0;
695 unsigned long hwpoison_vaddr;
696 unsigned long mask;
695 697
696 if (pte_present(pte)) {698 if (pte_present(pte)) {
697 pfn = pte_pfn(pte);699 pfn = pte_pfn(pte);
@@ -702,10 +704,12 @@ static int check_hwpoisoned_entry(pte_t pte, unsigned long addr, short shift, · 4 unmodified lines
702 pfn = softleaf_to_pfn(entry);704 pfn = softleaf_to_pfn(entry);
703 }705 }
704 706
705 if (!pfn || pfn != poisoned_pfn)707 mask = ~((1UL << (shift - PAGE_SHIFT)) - 1);
708 if (!pfn || pfn != (poisoned_pfn & mask))
706 return 0;709 return 0;
707 710
708 set_to_kill(tk, addr, shift);711 hwpoison_vaddr = addr + ((poisoned_pfn - pfn) << PAGE_SHIFT);
712 set_to_kill(tk, hwpoison_vaddr, shift);
709 return 1;713 return 1;
710}714}
711 715
@@ -1883,12 +1887,22 @@ static unsigned long __folio_free_raw_hwp(struct folio *folio, bool move_flag) · 1171 unmodified lines
1883 return count;1887 return count;
1884}1888}
1885 1889
1886static int folio_set_hugetlb_hwpoison(struct folio *folio, struct page *page)1890#define MF_HUGETLB_FREED 0 /* freed hugepage */
1891#define MF_HUGETLB_IN_USED 1 /* in-use hugepage */
1892#define MF_HUGETLB_NON_HUGEPAGE 2 /* not a hugepage */
1893#define MF_HUGETLB_FOLIO_PRE_POISONED 3 /* folio already poisoned */
1894#define MF_HUGETLB_PAGE_PRE_POISONED 4 /* exact page already poisoned */
1895#define MF_HUGETLB_RETRY 5 /* hugepage is busy, retry */
1896/*
1897 * Set hugetlb folio as hwpoisoned, update folio private raw hwpoison list
1898 * to keep track of the poisoned pages.
1899 */
1900static int hugetlb_update_hwpoison(struct folio *folio, struct page *page)
1887{1901{
1888 struct llist_head *head;1902 struct llist_head *head;
1889 struct raw_hwp_page *raw_hwp;1903 struct raw_hwp_page *raw_hwp;
1890 struct raw_hwp_page *p;1904 struct raw_hwp_page *p;
1891 int ret = folio_test_set_hwpoison(folio) ? -EHWPOISON : 0;1905 int ret = folio_test_set_hwpoison(folio) ? MF_HUGETLB_FOLIO_PRE_POISONED : 0;
1892 1906
1893 /*1907 /*
1894 * Once the hwpoison hugepage has lost reliable raw error info,1908 * Once the hwpoison hugepage has lost reliable raw error info,
@@ -1896,20 +1910,17 @@ static int folio_set_hugetlb_hwpoison(struct folio *folio, struct page *page) · 1 unmodified line
1896 * so skip to add additional raw error info.1910 * so skip to add additional raw error info.
1897 */1911 */
1898 if (folio_test_hugetlb_raw_hwp_unreliable(folio))1912 if (folio_test_hugetlb_raw_hwp_unreliable(folio))
1899 return -EHWPOISON;1913 return MF_HUGETLB_FOLIO_PRE_POISONED;
1900 head = raw_hwp_list_head(folio);1914 head = raw_hwp_list_head(folio);
1901 llist_for_each_entry(p, head->first, node) {1915 llist_for_each_entry(p, head->first, node) {
1902 if (p->page == page)1916 if (p->page == page)
1903 return -EHWPOISON;1917 return MF_HUGETLB_PAGE_PRE_POISONED;
1904 }1918 }
1905 1919
1906 raw_hwp = kmalloc(sizeof(struct raw_hwp_page), GFP_ATOMIC);1920 raw_hwp = kmalloc(sizeof(struct raw_hwp_page), GFP_ATOMIC);
1907 if (raw_hwp) {1921 if (raw_hwp) {
1908 raw_hwp->page = page;1922 raw_hwp->page = page;
1909 llist_add(&raw_hwp->node, head);1923 llist_add(&raw_hwp->node, head);
1910 /* the first error event will be counted in action_result(). */
1911 if (ret)
1912 num_poisoned_pages_inc(page_to_pfn(page));
1913 } else {1924 } else {
1914 /*1925 /*
1915 * Failed to save raw error info. We no longer trace all1926 * Failed to save raw error info. We no longer trace all
@@ -1957,42 +1968,39 @@ void folio_clear_hugetlb_hwpoison(struct folio *folio) · 41 unmodified lines
1957 1968
1958/*1969/*
1959 * Called from hugetlb code with hugetlb_lock held.1970 * Called from hugetlb code with hugetlb_lock held.
1960 *
1961 * Return values:
1962 * 0 - free hugepage
1963 * 1 - in-use hugepage
1964 * 2 - not a hugepage
1965 * -EBUSY - the hugepage is busy (try to retry)
1966 * -EHWPOISON - the hugepage is already hwpoisoned
1967 */1971 */
1968int __get_huge_page_for_hwpoison(unsigned long pfn, int flags,1972int __get_huge_page_for_hwpoison(unsigned long pfn, int flags,
1969 bool *migratable_cleared)1973 bool *migratable_cleared)
1970{1974{
1971 struct page *page = pfn_to_page(pfn);1975 struct page *page = pfn_to_page(pfn);
1972 struct folio *folio = page_folio(page);1976 struct folio *folio = page_folio(page);
1973 int ret = 2; /* fallback to normal page handling */
1974 bool count_increased = false;1977 bool count_increased = false;
1978 int ret, rc;
1975 1979
1976 if (!folio_test_hugetlb(folio))1980 if (!folio_test_hugetlb(folio)) {
1981 ret = MF_HUGETLB_NON_HUGEPAGE;
1977 goto out;1982 goto out;
1978 1983 } else if (flags & MF_COUNT_INCREASED) {
1979 if (flags & MF_COUNT_INCREASED) {1984 ret = MF_HUGETLB_IN_USED;
1980 ret = 1;
1981 count_increased = true;1985 count_increased = true;
1982 } else if (folio_test_hugetlb_freed(folio)) {1986 } else if (folio_test_hugetlb_freed(folio)) {
1983 ret = 0;1987 ret = MF_HUGETLB_FREED;
1984 } else if (folio_test_hugetlb_migratable(folio)) {1988 } else if (folio_test_hugetlb_migratable(folio)) {
1985 ret = folio_try_get(folio);1989 if (folio_try_get(folio)) {
1986 if (ret)1990 ret = MF_HUGETLB_IN_USED;
1987 count_increased = true;1991 count_increased = true;
1992 } else {
1993 ret = MF_HUGETLB_FREED;
1994 }
1988 } else {1995 } else {
1989 ret = -EBUSY;1996 ret = MF_HUGETLB_RETRY;
1990 if (!(flags & MF_NO_RETRY))1997 if (!(flags & MF_NO_RETRY))
1991 goto out;1998 goto out;
1992 }1999 }
1993 2000
1994 if (folio_set_hugetlb_hwpoison(folio, page)) {2001 rc = hugetlb_update_hwpoison(folio, page);
1995 ret = -EHWPOISON;2002 if (rc >= MF_HUGETLB_FOLIO_PRE_POISONED) {
2003 ret = rc;
1996 goto out;2004 goto out;
1997 }2005 }
1998 2006
@@ -2017,10 +2025,16 @@ out: · 18 unmodified lines
2017 * with basic operations like hugepage allocation/free/demotion.2025 * with basic operations like hugepage allocation/free/demotion.
2018 * So some of prechecks for hwpoison (pinning, and testing/setting2026 * So some of prechecks for hwpoison (pinning, and testing/setting
2019 * PageHWPoison) should be done in single hugetlb_lock range.2027 * PageHWPoison) should be done in single hugetlb_lock range.
2028 * Returns:
2029 * 0 - not hugetlb, or recovered
2030 * -EBUSY - not recovered
2031 * -EOPNOTSUPP - hwpoison_filter'ed
2032 * -EHWPOISON - folio or exact page already poisoned
2033 * -EFAULT - kill_accessing_process finds current->mm null
2020 */2034 */
2021static int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlb)2035static int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlb)
2022{2036{
2023 int res;2037 int res, rv;
2024 struct page *p = pfn_to_page(pfn);2038 struct page *p = pfn_to_page(pfn);
2025 struct folio *folio;2039 struct folio *folio;
2026 unsigned long page_flags;2040 unsigned long page_flags;
@@ -2029,22 +2043,29 @@ static int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlb · 2 unmodified lines
2029 *hugetlb = 1;2043 *hugetlb = 1;
2030retry:2044retry:
2031 res = get_huge_page_for_hwpoison(pfn, flags, &migratable_cleared);2045 res = get_huge_page_for_hwpoison(pfn, flags, &migratable_cleared);
2032 if (res == 2) { /* fallback to normal page handling */2046 switch (res) {
2047 case MF_HUGETLB_NON_HUGEPAGE: /* fallback to normal page handling */
2033 *hugetlb = 0;2048 *hugetlb = 0;
2034 return 0;2049 return 0;
2035 } else if (res == -EHWPOISON) {2050 case MF_HUGETLB_RETRY:
2036 if (flags & MF_ACTION_REQUIRED) {
2037 folio = page_folio(p);
2038 res = kill_accessing_process(current, folio_pfn(folio), flags);
2039 }
2040 action_result(pfn, MF_MSG_ALREADY_POISONED, MF_FAILED);
2041 return res;
2042 } else if (res == -EBUSY) {
2043 if (!(flags & MF_NO_RETRY)) {2051 if (!(flags & MF_NO_RETRY)) {
2044 flags |= MF_NO_RETRY;2052 flags |= MF_NO_RETRY;
2045 goto retry;2053 goto retry;
2046 }2054 }
2047 return action_result(pfn, MF_MSG_GET_HWPOISON, MF_IGNORED);2055 return action_result(pfn, MF_MSG_GET_HWPOISON, MF_IGNORED);
2056 case MF_HUGETLB_FOLIO_PRE_POISONED:
2057 case MF_HUGETLB_PAGE_PRE_POISONED:
2058 rv = -EHWPOISON;
2059 if (flags & MF_ACTION_REQUIRED)
2060 rv = kill_accessing_process(current, pfn, flags);
2061 if (res == MF_HUGETLB_PAGE_PRE_POISONED)
2062 action_result(pfn, MF_MSG_ALREADY_POISONED, MF_FAILED);
2063 else
2064 action_result(pfn, MF_MSG_HUGE, MF_FAILED);
2065 return rv;
2066 default:
2067 WARN_ON((res != MF_HUGETLB_FREED) && (res != MF_HUGETLB_IN_USED));
2068 break;
2048 }2069 }
2049 2070
2050 folio = page_folio(p);2071 folio = page_folio(p);
@@ -2055,7 +2076,7 @@ retry: · 4 unmodified lines
2055 if (migratable_cleared)2076 if (migratable_cleared)
2056 folio_set_hugetlb_migratable(folio);2077 folio_set_hugetlb_migratable(folio);
2057 folio_unlock(folio);2078 folio_unlock(folio);
2058 if (res == 1)2079 if (res == MF_HUGETLB_IN_USED)
2059 folio_put(folio);2080 folio_put(folio);
2060 return -EOPNOTSUPP;2081 return -EOPNOTSUPP;
2061 }2082 }
@@ -2064,7 +2085,7 @@ retry: · 2 unmodified lines
2064 * Handling free hugepage. The possible race with hugepage allocation2085 * Handling free hugepage. The possible race with hugepage allocation
2065 * or demotion can be prevented by PageHWPoison flag.2086 * or demotion can be prevented by PageHWPoison flag.
2066 */2087 */
2067 if (res == 0) {2088 if (res == MF_HUGETLB_FREED) {
2068 folio_unlock(folio);2089 folio_unlock(folio);
2069 if (__page_handle_poison(p) > 0) {2090 if (__page_handle_poison(p) > 0) {
2070 page_ref_inc(p);2091 page_ref_inc(p);
mm/memremap.c+34−1
@@ -477,10 +477,43 @@ void free_zone_device_folio(struct folio *folio) · 476 unmodified lines
477 }477 }
478}478}
479 479
480void zone_device_page_init(struct page *page, unsigned int order)480void zone_device_page_init(struct page *page, struct dev_pagemap *pgmap,
481 unsigned int order)
481{482{
483 struct page *new_page = page;
484 unsigned int i;
485
482 VM_WARN_ON_ONCE(order > MAX_ORDER_NR_PAGES);486 VM_WARN_ON_ONCE(order > MAX_ORDER_NR_PAGES);
483 487
488 for (i = 0; i < (1UL << order); ++i, ++new_page) {
489 struct folio *new_folio = (struct folio *)new_page;
490
491 /*
492 * new_page could have been part of previous higher order folio
493 * which encodes the order, in page + 1, in the flags bits. We
494 * blindly clear bits which could have set my order field here,
495 * including page head.
496 */
497 new_page->flags.f &= ~0xffUL; /* Clear possible order, page head */
498
499#ifdef NR_PAGES_IN_LARGE_FOLIO
500 /*
501 * This pointer math looks odd, but new_page could have been
502 * part of a previous higher order folio, which sets _nr_pages
503 * in page + 1 (new_page). Therefore, we use pointer casting to
504 * correctly locate the _nr_pages bits within new_page which
505 * could have modified by previous higher order folio.
506 */
507 ((struct folio *)(new_page - 1))->_nr_pages = 0;
508#endif
509
510 new_folio->mapping = NULL;
511 new_folio->pgmap = pgmap; /* Also clear compound head */
512 new_folio->share = 0; /* fsdax only, unused for device private */
513 VM_WARN_ON_FOLIO(folio_ref_count(new_folio), new_folio);
514 VM_WARN_ON_FOLIO(!folio_is_zone_device(new_folio), new_folio);
515 }
516
484 /*517 /*
485 * Drivers shouldn't be allocating pages after calling518 * Drivers shouldn't be allocating pages after calling
486 * memunmap_pages().519 * memunmap_pages().
mm/mm_init.c+6−6
@@ -2059,7 +2059,7 @@ static unsigned long __init deferred_init_pages(struct zone *zone, · 2058 unmodified lines
2059 */2059 */
2060static unsigned long __init2060static unsigned long __init
2061deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn,2061deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn,
2062 struct zone *zone)2062 struct zone *zone, bool can_resched)
2063{2063{
2064 int nid = zone_to_nid(zone);2064 int nid = zone_to_nid(zone);
2065 unsigned long nr_pages = 0;2065 unsigned long nr_pages = 0;
@@ -2085,10 +2085,10 @@ deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, · 19 unmodified lines
2085 2085
2086 spfn = chunk_end;2086 spfn = chunk_end;
2087 2087
2088 if (irqs_disabled())2088 if (can_resched)
2089 touch_nmi_watchdog();
2090 else
2091 cond_resched();2089 cond_resched();
2090 else
2091 touch_nmi_watchdog();
2092 }2092 }
2093 }2093 }
2094 2094
@@ -2101,7 +2101,7 @@ deferred_init_memmap_job(unsigned long start_pfn, unsigned long end_pfn, · 6 unmodified lines
2101{2101{
2102 struct zone *zone = arg;2102 struct zone *zone = arg;
2103 2103
2104 deferred_init_memmap_chunk(start_pfn, end_pfn, zone);2104 deferred_init_memmap_chunk(start_pfn, end_pfn, zone, true);
2105}2105}
2106 2106
2107static unsigned int __init2107static unsigned int __init
@@ -2216,7 +2216,7 @@ bool __init deferred_grow_zone(struct zone *zone, unsigned int order) · 108 unmodified lines
2216 for (spfn = first_deferred_pfn, epfn = SECTION_ALIGN_UP(spfn + 1);2216 for (spfn = first_deferred_pfn, epfn = SECTION_ALIGN_UP(spfn + 1);
2217 nr_pages < nr_pages_needed && spfn < zone_end_pfn(zone);2217 nr_pages < nr_pages_needed && spfn < zone_end_pfn(zone);
2218 spfn = epfn, epfn += PAGES_PER_SECTION) {2218 spfn = epfn, epfn += PAGES_PER_SECTION) {
2219 nr_pages += deferred_init_memmap_chunk(spfn, epfn, zone);2219 nr_pages += deferred_init_memmap_chunk(spfn, epfn, zone, false);
2220 }2220 }
2221 2221
2222 /*2222 /*
mm/shmem.c+34−11
@@ -962,17 +962,29 @@ static void shmem_delete_from_page_cache(struct folio *folio, void *radswap) · 961 unmodified lines
962 * being freed).962 * being freed).
963 */963 */
964static long shmem_free_swap(struct address_space *mapping,964static long shmem_free_swap(struct address_space *mapping,
965 pgoff_t index, void *radswap)965 pgoff_t index, pgoff_t end, void *radswap)
966{966{
967 int order = xa_get_order(&mapping->i_pages, index);967 XA_STATE(xas, &mapping->i_pages, index);
968 void *old;968 unsigned int nr_pages = 0;
969 pgoff_t base;
970 void *entry;
969 971
970 old = xa_cmpxchg_irq(&mapping->i_pages, index, radswap, NULL, 0);972 xas_lock_irq(&xas);
971 if (old != radswap)973 entry = xas_load(&xas);
972 return 0;974 if (entry == radswap) {
973 free_swap_and_cache_nr(radix_to_swp_entry(radswap), 1 << order);975 nr_pages = 1 << xas_get_order(&xas);
976 base = round_down(xas.xa_index, nr_pages);
977 if (base < index || base + nr_pages - 1 > end)
978 nr_pages = 0;
979 else
980 xas_store(&xas, NULL);
981 }
982 xas_unlock_irq(&xas);
983
984 if (nr_pages)
985 free_swap_and_cache_nr(radix_to_swp_entry(radswap), nr_pages);
974 986
975 return 1 << order;987 return nr_pages;
976}988}
977 989
978/*990/*
@@ -1124,8 +1136,8 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend, · 145 unmodified lines
1124 if (xa_is_value(folio)) {1136 if (xa_is_value(folio)) {
1125 if (unfalloc)1137 if (unfalloc)
1126 continue;1138 continue;
1127 nr_swaps_freed += shmem_free_swap(mapping,1139 nr_swaps_freed += shmem_free_swap(mapping, indices[i],
1128 indices[i], folio);1140 end - 1, folio);
1129 continue;1141 continue;
1130 }1142 }
1131 1143
@@ -1191,12 +1203,23 @@ whole_folios: · 59 unmodified lines
1191 folio = fbatch.folios[i];1203 folio = fbatch.folios[i];
1192 1204
1193 if (xa_is_value(folio)) {1205 if (xa_is_value(folio)) {
1206 int order;
1194 long swaps_freed;1207 long swaps_freed;
1195 1208
1196 if (unfalloc)1209 if (unfalloc)
1197 continue;1210 continue;
1198 swaps_freed = shmem_free_swap(mapping, indices[i], folio);1211 swaps_freed = shmem_free_swap(mapping, indices[i],
1212 end - 1, folio);
1199 if (!swaps_freed) {1213 if (!swaps_freed) {
1214 /*
1215 * If found a large swap entry cross the end border,
1216 * skip it as the truncate_inode_partial_folio above
1217 * should have at least zerod its content once.
1218 */
1219 order = shmem_confirm_swap(mapping, indices[i],
1220 radix_to_swp_entry(folio));
1221 if (order > 0 && indices[i] + (1 << order) > end)
1222 continue;
1200 /* Swap was replaced by page: retry */1223 /* Swap was replaced by page: retry */
1201 index = indices[i];1224 index = indices[i];
1202 break;1225 break;
mm/swap.h+1−1
@@ -198,7 +198,7 @@ int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug); · 197 unmodified lines
198void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug);198void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug);
199 199
200/* linux/mm/swap_state.c */200/* linux/mm/swap_state.c */
201extern struct address_space swap_space __ro_after_init;201extern struct address_space swap_space __read_mostly;
202static inline struct address_space *swap_address_space(swp_entry_t entry)202static inline struct address_space *swap_address_space(swp_entry_t entry)
203{203{
204 return &swap_space;204 return &swap_space;
mm/swap_state.c+1−2
@@ -37,8 +37,7 @@ static const struct address_space_operations swap_aops = { · 36 unmodified lines
37#endif37#endif
38};38};
39 39
40/* Set swap_space as read only as swap cache is handled by swap table */40struct address_space swap_space __read_mostly = {
41struct address_space swap_space __ro_after_init = {
42 .a_ops = &swap_aops,41 .a_ops = &swap_aops,
43};42};
44 43
mm/vmalloc.c+2−5
@@ -4322,7 +4322,7 @@ void *vrealloc_node_align_noprof(const void *p, size_t size, unsigned long align · 4321 unmodified lines
4322 if (want_init_on_free() || want_init_on_alloc(flags))4322 if (want_init_on_free() || want_init_on_alloc(flags))
4323 memset((void *)p + size, 0, old_size - size);4323 memset((void *)p + size, 0, old_size - size);
4324 vm->requested_size = size;4324 vm->requested_size = size;
4325 kasan_poison_vmalloc(p + size, old_size - size);4325 kasan_vrealloc(p, old_size, size);
4326 return (void *)p;4326 return (void *)p;
4327 }4327 }
4328 4328
@@ -4330,16 +4330,13 @@ void *vrealloc_node_align_noprof(const void *p, size_t size, unsigned long align · 1 unmodified line
4330 * We already have the bytes available in the allocation; use them.4330 * We already have the bytes available in the allocation; use them.
4331 */4331 */
4332 if (size <= alloced_size) {4332 if (size <= alloced_size) {
4333 kasan_unpoison_vmalloc(p + old_size, size - old_size,
4334 KASAN_VMALLOC_PROT_NORMAL |
4335 KASAN_VMALLOC_VM_ALLOC |
4336 KASAN_VMALLOC_KEEP_TAG);
4337 /*4333 /*
4338 * No need to zero memory here, as unused memory will have4334 * No need to zero memory here, as unused memory will have
4339 * already been zeroed at initial allocation time or during4335 * already been zeroed at initial allocation time or during
4340 * realloc shrink time.4336 * realloc shrink time.
4341 */4337 */
4342 vm->requested_size = size;4338 vm->requested_size = size;
4339 kasan_vrealloc(p, old_size, size);
4343 return (void *)p;4340 return (void *)p;
4344 }4341 }
4345 4342