History

Merge branch 'akpm' (patches from Andrew)

Merge updates from Andrew Morton: "Incoming: - a small number of updates to scripts/, ocfs2 and fs/buffer.c - most of MM I still have quite a lot of material (mostly not MM) staged after linux-next due to -next dependencies. I'll send those across next week as the preprequisites get merged up" * emailed patches from Andrew Morton <akpm@linux-foundation.org>: (135 commits) mm/page_io.c: annotate refault stalls from swap_readpage mm/Kconfig: fix trivial help text punctuation mm/Kconfig: fix indentation mm/memory_hotplug.c: remove __online_page_set_limits() mm: fix typos in comments when calling __SetPageUptodate() mm: fix struct member name in function comments mm/shmem.c: cast the type of unmap_start to u64 mm: shmem: use proper gfp flags for shmem_writepage() mm/shmem.c: make array 'values' static const, makes object smaller userfaultfd: require CAP_SYS_PTRACE for UFFD_FEATURE_EVENT_FORK fs/userfaultfd.c: wp: clear VM_UFFD_MISSING or VM_UFFD_WP during userfaultfd_register() userfaultfd: wrap the common dst_vma check into an inlined function userfaultfd: remove unnecessary WARN_ON() in __mcopy_atomic_hugetlb() userfaultfd: use vma_pagesize for all huge page size calculation mm/madvise.c: use PAGE_ALIGN[ED] for range checking mm/madvise.c: replace with page_size() in madvise_inject_error() mm/mmap.c: make vma_merge() comment more easy to understand mm/hwpoison-inject: use DEFINE_DEBUGFS_ATTRIBUTE to define debugfs fops autonuma: reduce cache footprint when scanning page tables autonuma: fix watermark checking in migrate_balanced_pgdat() ...

Linus Torvalds committed 6 years ago

Commit 596cf452 parents c3bfc5d 937790695 files · +2661 −1506Browse files

Unified · Split

95 files
  1. Documentation/admin-guide/cgroup-v2.rst +6 −1
  2. Documentation/dev-tools/kasan.rst +63 −0
  3. arch/Kconfig +5 −4
  4. arch/arc/include/asm/pgtable.h +0 −1
  5. arch/arc/mm/fault.c +8 −2
  6. arch/arc/mm/highmem.c +3 −1
  7. arch/powerpc/include/asm/book3s/64/pgtable-4k.h +0 −3
  8. arch/powerpc/include/asm/book3s/64/pgtable-64k.h +0 −3
  9. arch/powerpc/mm/book3s64/radix_pgtable.c +1 −0
  10. arch/x86/Kconfig +1 −0
  11. arch/x86/mm/kasan_init_64.c +61 −0
  12. drivers/base/memory.c +15 −25
  13. drivers/hv/hv_balloon.c +1 −3
  14. drivers/xen/balloon.c +0 −1
  15. fs/buffer.c +4 −2
  16. fs/direct-io.c +0 −21
  17. fs/hugetlbfs/inode.c +48 −15
  18. fs/ocfs2/acl.c +2 −2
  19. fs/userfaultfd.c +13 −8
  20. include/asm-generic/4level-fixup.h +0 −1
  21. include/asm-generic/5level-fixup.h +0 −1
  22. include/asm-generic/pgtable-nop4d.h +1 −1
  23. include/asm-generic/pgtable-nopmd.h +1 −1
  24. include/asm-generic/pgtable-nopud.h +1 −1
  25. include/asm-generic/pgtable.h +51 −0
  26. include/asm-generic/tlb.h +0 −4
  27. include/linux/fs.h +5 −1
  28. include/linux/gfp.h +2 −0
  29. include/linux/hugetlb.h +116 −24
  30. include/linux/kasan.h +31 −0
  31. include/linux/memblock.h +3 −0
  32. include/linux/memcontrol.h +22 −27
  33. include/linux/memory_hotplug.h +4 −7
  34. include/linux/mm.h +12 −22
  35. include/linux/mmzone.h +20 −14
  36. include/linux/moduleloader.h +1 −1
  37. include/linux/page-isolation.h +2 −2
  38. include/linux/slab.h +0 −20
  39. include/linux/string.h +2 −0
  40. include/linux/swap.h +1 −1
  41. include/linux/vmalloc.h +12 −0
  42. include/trace/events/kmem.h +47 −0
  43. kernel/events/uprobes.c +1 −1
  44. kernel/fork.c +4 −0
  45. kernel/sysctl.c +1 −1
  46. lib/Kconfig.kasan +16 −0
  47. lib/test_kasan.c +26 −0
  48. lib/vsprintf.c +29 −11
  49. mm/Kconfig +20 −20
  50. mm/cma.c +2 −4
  51. mm/cma_debug.c +5 −5
  52. mm/filemap.c +29 −25
  53. mm/gup.c +27 −13
  54. mm/huge_memory.c +1 −1
  55. mm/hugetlb.c +91 −197
  56. mm/hwpoison-inject.c +2 −2
  57. mm/internal.h +26 −1
  58. mm/kasan/common.c +233 −0
  59. mm/kasan/generic_report.c +3 −0
  60. mm/kasan/kasan.h +1 −0
  61. mm/khugepaged.c +18 −0
  62. mm/madvise.c +7 −7
  63. mm/memblock.c +75 −36
  64. mm/memcontrol.c +33 −134
  65. mm/memory-failure.c +23 −38
  66. mm/memory.c +40 −12
  67. mm/memory_hotplug.c +56 −30
  68. mm/mempolicy.c +31 −16
  69. mm/migrate.c +6 −10
  70. mm/mmap.c +21 −42
  71. mm/mprotect.c +4 −4
  72. mm/mremap.c +2 −2
  73. mm/nommu.c +2 −8
  74. mm/page_alloc.c +119 −18
  75. mm/page_io.c +13 −2
  76. mm/page_isolation.c +6 −6
  77. mm/pgtable-generic.c +9 −0
  78. mm/rmap.c +45 −20
  79. mm/shmem.c +17 −12
  80. mm/slab.c +4 −3
  81. mm/slab.h +3 −3
  82. mm/slab_common.c +53 −46
  83. mm/slub.c +16 −20
  84. mm/sparse.c +10 −8
  85. mm/swap.c +24 −5
  86. mm/swapfile.c +7 −0
  87. mm/userfaultfd.c +37 −36
  88. mm/util.c +16 −6
  89. mm/vmalloc.c +134 −58
  90. mm/vmscan.c +327 −335
  91. mm/workingset.c +53 −16
  92. mm/z3fold.c +303 −72
  93. scripts/spelling.txt +28 −0
  94. tools/testing/selftests/memfd/memfd_test.c +36 −0
  95. tools/testing/selftests/vm/config +1 −0
Documentation/admin-guide/cgroup-v2.rst+6−1
@@ -1288,7 +1288,12 @@ PAGE_SIZE multiple when read back. · 1287 unmodified lines
12881288 inactive_anon, active_anon, inactive_file, active_file, unevictable
12891289 Amount of memory, swap-backed and filesystem-backed,
12901290 on the internal memory management lists used by the
1291− page reclaim algorithm
1291+ page reclaim algorithm.
1292+
1293+ As these represent internal list state (eg. shmem pages are on anon
1294+ memory management lists), inactive_foo + active_foo may not be equal to
1295+ the value for the foo counter, since the foo counter is type-based, not
1296+ list-based.
12921297
12931298 slab_reclaimable
12941299 Part of "slab" that might be reclaimed, such as
Documentation/dev-tools/kasan.rst+63−0
@@ -218,3 +218,66 @@ brk handler is used to print bug reports. · 217 unmodified lines
218218 A potential expansion of this mode is a hardware tag-based mode, which would
219219 use hardware memory tagging support instead of compiler instrumentation and
220220 manual shadow memory manipulation.
221+
222+What memory accesses are sanitised by KASAN?
223+--------------------------------------------
224+
225+The kernel maps memory in a number of different parts of the address
226+space. This poses something of a problem for KASAN, which requires
227+that all addresses accessed by instrumented code have a valid shadow
228+region.
229+
230+The range of kernel virtual addresses is large: there is not enough
231+real memory to support a real shadow region for every address that
232+could be accessed by the kernel.
233+
234+By default
235+~~~~~~~~~~
236+
237+By default, architectures only map real memory over the shadow region
238+for the linear mapping (and potentially other small areas). For all
239+other areas - such as vmalloc and vmemmap space - a single read-only
240+page is mapped over the shadow area. This read-only shadow page
241+declares all memory accesses as permitted.
242+
243+This presents a problem for modules: they do not live in the linear
244+mapping, but in a dedicated module space. By hooking in to the module
245+allocator, KASAN can temporarily map real shadow memory to cover
246+them. This allows detection of invalid accesses to module globals, for
247+example.
248+
249+This also creates an incompatibility with ``VMAP_STACK``: if the stack
250+lives in vmalloc space, it will be shadowed by the read-only page, and
251+the kernel will fault when trying to set up the shadow data for stack
252+variables.
253+
254+CONFIG_KASAN_VMALLOC
255+~~~~~~~~~~~~~~~~~~~~
256+
257+With ``CONFIG_KASAN_VMALLOC``, KASAN can cover vmalloc space at the
258+cost of greater memory usage. Currently this is only supported on x86.
259+
260+This works by hooking into vmalloc and vmap, and dynamically
261+allocating real shadow memory to back the mappings.
262+
263+Most mappings in vmalloc space are small, requiring less than a full
264+page of shadow space. Allocating a full shadow page per mapping would
265+therefore be wasteful. Furthermore, to ensure that different mappings
266+use different shadow pages, mappings would have to be aligned to
267+``KASAN_SHADOW_SCALE_SIZE * PAGE_SIZE``.
268+
269+Instead, we share backing space across multiple mappings. We allocate
270+a backing page when a mapping in vmalloc space uses a particular page
271+of the shadow region. This page can be shared by other vmalloc
272+mappings later on.
273+
274+We hook in to the vmap infrastructure to lazily clean up unused shadow
275+memory.
276+
277+To avoid the difficulties around swapping mappings around, we expect
278+that the part of the shadow region that covers the vmalloc space will
279+not be covered by the early shadow page, but will be left
280+unmapped. This will require changes in arch-specific code.
281+
282+This allows ``VMAP_STACK`` support on x86, and can simplify support of
283+architectures that do not have a fixed module region.
arch/Kconfig+5−4
@@ -836,16 +836,17 @@ config HAVE_ARCH_VMAP_STACK · 835 unmodified lines
836836 config VMAP_STACK
837837 default y
838838 bool "Use a virtually-mapped stack"
839− depends on HAVE_ARCH_VMAP_STACK && !KASAN
839+ depends on HAVE_ARCH_VMAP_STACK
840+ depends on !KASAN || KASAN_VMALLOC
840841 ---help---
841842 Enable this if you want the use virtually-mapped kernel stacks
842843 with guard pages. This causes kernel stack overflows to be
843844 caught immediately rather than causing difficult-to-diagnose
844845 corruption.
845846
846− This is presently incompatible with KASAN because KASAN expects
847− the stack to map directly to the KASAN shadow map using a formula
848− that is incorrect if the stack is in vmalloc space.
847+ To use this with KASAN, the architecture must support backing
848+ virtual mappings with real shadow memory, and KASAN_VMALLOC must
849+ be enabled.
849850
850851 config ARCH_OPTIONAL_KERNEL_RWX
851852 def_bool n
arch/arc/include/asm/pgtable.h+0−1
@@ -33,7 +33,6 @@ · 32 unmodified lines
3333 #define _ASM_ARC_PGTABLE_H
3434
3535 #include <linux/bits.h>
36−#define __ARCH_USE_5LEVEL_HACK
3736 #include <asm-generic/pgtable-nopmd.h>
3837 #include <asm/page.h>
3938 #include <asm/mmu.h> /* to propagate CONFIG_ARC_MMU_VER <n> */
arch/arc/mm/fault.c+8−2
@@ -30,6 +30,7 @@ noinline static int handle_kernel_vaddr_fault(unsigned long address) · 29 unmodified lines
3030 * with the 'reference' page table.
3131 */
3232 pgd_t *pgd, *pgd_k;
33+ p4d_t *p4d, *p4d_k;
3334 pud_t *pud, *pud_k;
3435 pmd_t *pmd, *pmd_k;
3536
@@ -39,8 +40,13 @@ noinline static int handle_kernel_vaddr_fault(unsigned long address) · 3 unmodified lines
3940 if (!pgd_present(*pgd_k))
4041 goto bad_area;
4142
42− pud = pud_offset(pgd, address);
43− pud_k = pud_offset(pgd_k, address);
43+ p4d = p4d_offset(pgd, address);
44+ p4d_k = p4d_offset(pgd_k, address);
45+ if (!p4d_present(*p4d_k))
46+ goto bad_area;
47+
48+ pud = pud_offset(p4d, address);
49+ pud_k = pud_offset(p4d_k, address);
4450 if (!pud_present(*pud_k))
4551 goto bad_area;
4652
arch/arc/mm/highmem.c+3−1
@@ -111,12 +111,14 @@ EXPORT_SYMBOL(__kunmap_atomic); · 110 unmodified lines
111111 static noinline pte_t * __init alloc_kmap_pgtable(unsigned long kvaddr)
112112 {
113113 pgd_t *pgd_k;
114+ p4d_t *p4d_k;
114115 pud_t *pud_k;
115116 pmd_t *pmd_k;
116117 pte_t *pte_k;
117118
118119 pgd_k = pgd_offset_k(kvaddr);
119− pud_k = pud_offset(pgd_k, kvaddr);
120+ p4d_k = p4d_offset(pgd_k, kvaddr);
121+ pud_k = pud_offset(p4d_k, kvaddr);
120122 pmd_k = pmd_offset(pud_k, kvaddr);
121123
122124 pte_k = (pte_t *)memblock_alloc_low(PAGE_SIZE, PAGE_SIZE);
arch/powerpc/include/asm/book3s/64/pgtable-4k.h+0−3
@@ -70,9 +70,6 @@ static inline int get_hugepd_cache_index(int index) · 69 unmodified lines
7070 /* should not reach */
7171 }
7272
73−#else /* !CONFIG_HUGETLB_PAGE */
74−static inline int pmd_huge(pmd_t pmd) { return 0; }
75−static inline int pud_huge(pud_t pud) { return 0; }
7673 #endif /* CONFIG_HUGETLB_PAGE */
7774
7875 #endif /* __ASSEMBLY__ */
arch/powerpc/include/asm/book3s/64/pgtable-64k.h+0−3
@@ -59,9 +59,6 @@ static inline int get_hugepd_cache_index(int index) · 58 unmodified lines
5959 BUG();
6060 }
6161
62−#else /* !CONFIG_HUGETLB_PAGE */
63−static inline int pmd_huge(pmd_t pmd) { return 0; }
64−static inline int pud_huge(pud_t pud) { return 0; }
6562 #endif /* CONFIG_HUGETLB_PAGE */
6663
6764 static inline int remap_4k_pfn(struct vm_area_struct *vma, unsigned long addr,
arch/powerpc/mm/book3s64/radix_pgtable.c+1−0
@@ -13,6 +13,7 @@ · 12 unmodified lines
1313 #include <linux/memblock.h>
1414 #include <linux/of_fdt.h>
1515 #include <linux/mm.h>
16+#include <linux/hugetlb.h>
1617 #include <linux/string_helpers.h>
1718 #include <linux/stop_machine.h>
1819
arch/x86/Kconfig+1−0
@@ -134,6 +134,7 @@ config X86 · 133 unmodified lines
134134 select HAVE_ARCH_JUMP_LABEL
135135 select HAVE_ARCH_JUMP_LABEL_RELATIVE
136136 select HAVE_ARCH_KASAN if X86_64
137+ select HAVE_ARCH_KASAN_VMALLOC if X86_64
137138 select HAVE_ARCH_KGDB
138139 select HAVE_ARCH_MMAP_RND_BITS if MMU
139140 select HAVE_ARCH_MMAP_RND_COMPAT_BITS if MMU && COMPAT
arch/x86/mm/kasan_init_64.c+61−0
@@ -245,6 +245,49 @@ static void __init kasan_map_early_shadow(pgd_t *pgd) · 244 unmodified lines
245245 } while (pgd++, addr = next, addr != end);
246246 }
247247
248+static void __init kasan_shallow_populate_p4ds(pgd_t *pgd,
249+ unsigned long addr,
250+ unsigned long end)
251+{
252+ p4d_t *p4d;
253+ unsigned long next;
254+ void *p;
255+
256+ p4d = p4d_offset(pgd, addr);
257+ do {
258+ next = p4d_addr_end(addr, end);
259+
260+ if (p4d_none(*p4d)) {
261+ p = early_alloc(PAGE_SIZE, NUMA_NO_NODE, true);
262+ p4d_populate(&init_mm, p4d, p);
263+ }
264+ } while (p4d++, addr = next, addr != end);
265+}
266+
267+static void __init kasan_shallow_populate_pgds(void *start, void *end)
268+{
269+ unsigned long addr, next;
270+ pgd_t *pgd;
271+ void *p;
272+
273+ addr = (unsigned long)start;
274+ pgd = pgd_offset_k(addr);
275+ do {
276+ next = pgd_addr_end(addr, (unsigned long)end);
277+
278+ if (pgd_none(*pgd)) {
279+ p = early_alloc(PAGE_SIZE, NUMA_NO_NODE, true);
280+ pgd_populate(&init_mm, pgd, p);
281+ }
282+
283+ /*
284+ * we need to populate p4ds to be synced when running in
285+ * four level mode - see sync_global_pgds_l4()
286+ */
287+ kasan_shallow_populate_p4ds(pgd, addr, next);
288+ } while (pgd++, addr = next, addr != (unsigned long)end);
289+}
290+
248291 #ifdef CONFIG_KASAN_INLINE
249292 static int kasan_die_handler(struct notifier_block *self,
250293 unsigned long val,
@@ -354,6 +397,24 @@ void __init kasan_init(void) · 103 unmodified lines
354397
355398 kasan_populate_early_shadow(
356399 kasan_mem_to_shadow((void *)PAGE_OFFSET + MAXMEM),
400+ kasan_mem_to_shadow((void *)VMALLOC_START));
401+
402+ /*
403+ * If we're in full vmalloc mode, don't back vmalloc space with early
404+ * shadow pages. Instead, prepopulate pgds/p4ds so they are synced to
405+ * the global table and we can populate the lower levels on demand.
406+ */
407+ if (IS_ENABLED(CONFIG_KASAN_VMALLOC))
408+ kasan_shallow_populate_pgds(
409+ kasan_mem_to_shadow((void *)VMALLOC_START),
410+ kasan_mem_to_shadow((void *)VMALLOC_END));
411+ else
412+ kasan_populate_early_shadow(
413+ kasan_mem_to_shadow((void *)VMALLOC_START),
414+ kasan_mem_to_shadow((void *)VMALLOC_END));
415+
416+ kasan_populate_early_shadow(
417+ kasan_mem_to_shadow((void *)VMALLOC_END + 1),
357418 shadow_cpu_entry_begin);
358419
359420 kasan_populate_shadow((unsigned long)shadow_cpu_entry_begin,
drivers/base/memory.c+15−25
@@ -19,15 +19,12 @@ · 18 unmodified lines
1919 #include <linux/memory.h>
2020 #include <linux/memory_hotplug.h>
2121 #include <linux/mm.h>
22−#include <linux/mutex.h>
2322 #include <linux/stat.h>
2423 #include <linux/slab.h>
2524
2625 #include <linux/atomic.h>
2726 #include <linux/uaccess.h>
2827
29−static DEFINE_MUTEX(mem_sysfs_mutex);
30−
3128 #define MEMORY_CLASS_NAME "memory"
3229
3330 #define to_memory_block(dev) container_of(dev, struct memory_block, dev)
@@ -538,12 +535,7 @@ static ssize_t soft_offline_page_store(struct device *dev, · 504 unmodified lines
538535 if (kstrtoull(buf, 0, &pfn) < 0)
539536 return -EINVAL;
540537 pfn >>= PAGE_SHIFT;
541− if (!pfn_valid(pfn))
542− return -ENXIO;
543− /* Only online pages can be soft-offlined (esp., not ZONE_DEVICE). */
544− if (!pfn_to_online_page(pfn))
545− return -EIO;
546− ret = soft_offline_page(pfn_to_page(pfn), 0);
538+ ret = soft_offline_page(pfn, 0);
547539 return ret == 0 ? count : ret;
548540 }
549541
@@ -705,6 +697,8 @@ static void unregister_memory(struct memory_block *memory) · 155 unmodified lines
705697 * Create memory block devices for the given memory area. Start and size
706698 * have to be aligned to memory block granularity. Memory block devices
707699 * will be initialized as offline.
700+ *
701+ * Called under device_hotplug_lock.
708702 */
709703 int create_memory_block_devices(unsigned long start, unsigned long size)
710704 {
@@ -718,7 +712,6 @@ int create_memory_block_devices(unsigned long start, unsigned long size) · 7 unmodified lines
718712 !IS_ALIGNED(size, memory_block_size_bytes())))
719713 return -EINVAL;
720714
721− mutex_lock(&mem_sysfs_mutex);
722715 for (block_id = start_block_id; block_id != end_block_id; block_id++) {
723716 ret = init_memory_block(&mem, block_id, MEM_OFFLINE);
724717 if (ret)
@@ -730,11 +723,12 @@ int create_memory_block_devices(unsigned long start, unsigned long size) · 5 unmodified lines
730723 for (block_id = start_block_id; block_id != end_block_id;
731724 block_id++) {
732725 mem = find_memory_block_by_id(block_id);
726+ if (WARN_ON_ONCE(!mem))
727+ continue;
733728 mem->section_count = 0;
734729 unregister_memory(mem);
735730 }
736731 }
737− mutex_unlock(&mem_sysfs_mutex);
738732 return ret;
739733 }
740734
@@ -742,6 +736,8 @@ int create_memory_block_devices(unsigned long start, unsigned long size) · 1 unmodified line
742736 * Remove memory block devices for the given memory area. Start and size
743737 * have to be aligned to memory block granularity. Memory block devices
744738 * have to be offline.
739+ *
740+ * Called under device_hotplug_lock.
745741 */
746742 void remove_memory_block_devices(unsigned long start, unsigned long size)
747743 {
@@ -754,7 +750,6 @@ void remove_memory_block_devices(unsigned long start, unsigned long size) · 6 unmodified lines
754750 !IS_ALIGNED(size, memory_block_size_bytes())))
755751 return;
756752
757− mutex_lock(&mem_sysfs_mutex);
758753 for (block_id = start_block_id; block_id != end_block_id; block_id++) {
759754 mem = find_memory_block_by_id(block_id);
760755 if (WARN_ON_ONCE(!mem))
@@ -763,7 +758,6 @@ void remove_memory_block_devices(unsigned long start, unsigned long size) · 2 unmodified lines
763758 unregister_memory_block_under_nodes(mem);
764759 unregister_memory(mem);
765760 }
766− mutex_unlock(&mem_sysfs_mutex);
767761 }
768762
769763 /* return true if the memory block is offlined, otherwise, return false */
@@ -797,12 +791,13 @@ static const struct attribute_group *memory_root_attr_groups[] = { · 27 unmodified lines
797791 };
798792
799793 /*
800− * Initialize the sysfs support for memory devices...
794+ * Initialize the sysfs support for memory devices. At the time this function
795+ * is called, we cannot have concurrent creation/deletion of memory block
796+ * devices, the device_hotplug_lock is not needed.
801797 */
802798 void __init memory_dev_init(void)
803799 {
804800 int ret;
805− int err;
806801 unsigned long block_sz, nr;
807802
808803 /* Validate the configured memory block size */
@@ -813,24 +808,19 @@ void __init memory_dev_init(void) · 4 unmodified lines
813808
814809 ret = subsys_system_register(&memory_subsys, memory_root_attr_groups);
815810 if (ret)
816− goto out;
811+ panic("%s() failed to register subsystem: %d\n", __func__, ret);
817812
818813 /*
819814 * Create entries for memory sections that were found
820815 * during boot and have been initialized
821816 */
822− mutex_lock(&mem_sysfs_mutex);
823817 for (nr = 0; nr <= __highest_present_section_nr;
824818 nr += sections_per_block) {
825− err = add_memory_block(nr);
826− if (!ret)
827− ret = err;
819+ ret = add_memory_block(nr);
820+ if (ret)
821+ panic("%s() failed to add memory block: %d\n", __func__,
822+ ret);
828823 }
829− mutex_unlock(&mem_sysfs_mutex);
830−
831−out:
832− if (ret)
833− panic("%s() failed: %d\n", __func__, ret);
834824 }
835825
836826 /**
drivers/hv/hv_balloon.c+1−3
@@ -682,9 +682,7 @@ static void hv_page_online_one(struct hv_hotadd_state *has, struct page *pg) · 681 unmodified lines
682682 __ClearPageOffline(pg);
683683
684684 /* This frame is currently backed; online the page. */
685− __online_page_set_limits(pg);
686− __online_page_increment_counters(pg);
687− __online_page_free(pg);
685+ generic_online_page(pg, 0);
688686
689687 lockdep_assert_held(&dm_device.ha_lock);
690688 dm_device.num_pages_onlined++;
drivers/xen/balloon.c+0−1
@@ -374,7 +374,6 @@ static void xen_online_page(struct page *page, unsigned int order) · 373 unmodified lines
374374 mutex_lock(&balloon_mutex);
375375 for (i = 0; i < size; i++) {
376376 p = pfn_to_page(start_pfn + i);
377− __online_page_set_limits(p);
378377 balloon_append(p);
379378 }
380379 mutex_unlock(&balloon_mutex);
fs/buffer.c+4−2
@@ -49,6 +49,8 @@ · 48 unmodified lines
4949 #include <trace/events/block.h>
5050 #include <linux/fscrypt.h>
5151
52+#include "internal.h"
53+
5254 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
5355 static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh,
5456 enum rw_hint hint, struct writeback_control *wbc);
@@ -1423,10 +1425,10 @@ static bool has_bh_in_lru(int cpu, void *dummy) · 1368 unmodified lines
14231425
14241426 for (i = 0; i < BH_LRU_SIZE; i++) {
14251427 if (b->bhs[i])
1426− return 1;
1428+ return true;
14271429 }
14281430
1429− return 0;
1431+ return false;
14301432 }
14311433
14321434 void invalidate_bh_lrus(void)
fs/direct-io.c+0−21
@@ -220,27 +220,6 @@ static inline struct page *dio_get_page(struct dio *dio, · 219 unmodified lines
220220 return dio->pages[sdio->head];
221221 }
222222
223−/*
224− * Warn about a page cache invalidation failure during a direct io write.
225− */
226−void dio_warn_stale_pagecache(struct file *filp)
227−{
228− static DEFINE_RATELIMIT_STATE(_rs, 86400 * HZ, DEFAULT_RATELIMIT_BURST);
229− char pathname[128];
230− struct inode *inode = file_inode(filp);
231− char *path;
232−
233− errseq_set(&inode->i_mapping->wb_err, -EIO);
234− if (__ratelimit(&_rs)) {
235− path = file_path(filp, pathname, sizeof(pathname));
236− if (IS_ERR(path))
237− path = "(unknown)";
238− pr_crit("Page cache invalidation failure on direct I/O. Possible data corruption due to collision with buffered I/O!\n");
239− pr_crit("File: %s PID: %d Comm: %.20s\n", path, current->pid,
240− current->comm);
241− }
242−}
243−
244223 /*
245224 * dio_complete() - called when all DIO BIO I/O has been completed
246225 *
fs/hugetlbfs/inode.c+48−15
@@ -440,7 +440,7 @@ static void remove_inode_hugepages(struct inode *inode, loff_t lstart, · 439 unmodified lines
440440 u32 hash;
441441
442442 index = page->index;
443− hash = hugetlb_fault_mutex_hash(h, mapping, index, 0);
443+ hash = hugetlb_fault_mutex_hash(mapping, index);
444444 mutex_lock(&hugetlb_fault_mutex_table[hash]);
445445
446446 /*
@@ -644,7 +644,7 @@ static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset, · 197 unmodified lines
644644 addr = index * hpage_size;
645645
646646 /* mutex taken here, fault path and hole punch */
647− hash = hugetlb_fault_mutex_hash(h, mapping, index, addr);
647+ hash = hugetlb_fault_mutex_hash(mapping, index);
648648 mutex_lock(&hugetlb_fault_mutex_table[hash]);
649649
650650 /* See if already present in mapping to avoid alloc/free */
@@ -815,8 +815,11 @@ static struct inode *hugetlbfs_get_inode(struct super_block *sb, · 164 unmodified lines
815815 /*
816816 * File creation. Allocate an inode, and we're done..
817817 */
818−static int hugetlbfs_mknod(struct inode *dir,
819− struct dentry *dentry, umode_t mode, dev_t dev)
818+static int do_hugetlbfs_mknod(struct inode *dir,
819+ struct dentry *dentry,
820+ umode_t mode,
821+ dev_t dev,
822+ bool tmpfile)
820823 {
821824 struct inode *inode;
822825 int error = -ENOSPC;
@@ -824,13 +827,23 @@ static int hugetlbfs_mknod(struct inode *dir, · 1 unmodified line
824827 inode = hugetlbfs_get_inode(dir->i_sb, dir, mode, dev);
825828 if (inode) {
826829 dir->i_ctime = dir->i_mtime = current_time(dir);
827− d_instantiate(dentry, inode);
828− dget(dentry); /* Extra count - pin the dentry in core */
830+ if (tmpfile) {
831+ d_tmpfile(dentry, inode);
832+ } else {
833+ d_instantiate(dentry, inode);
834+ dget(dentry);/* Extra count - pin the dentry in core */
835+ }
829836 error = 0;
830837 }
831838 return error;
832839 }
833840
841+static int hugetlbfs_mknod(struct inode *dir,
842+ struct dentry *dentry, umode_t mode, dev_t dev)
843+{
844+ return do_hugetlbfs_mknod(dir, dentry, mode, dev, false);
845+}
846+
834847 static int hugetlbfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
835848 {
836849 int retval = hugetlbfs_mknod(dir, dentry, mode | S_IFDIR, 0);
@@ -844,6 +857,12 @@ static int hugetlbfs_create(struct inode *dir, struct dentry *dentry, umode_t mo · 7 unmodified lines
844857 return hugetlbfs_mknod(dir, dentry, mode | S_IFREG, 0);
845858 }
846859
860+static int hugetlbfs_tmpfile(struct inode *dir,
861+ struct dentry *dentry, umode_t mode)
862+{
863+ return do_hugetlbfs_mknod(dir, dentry, mode | S_IFREG, 0, true);
864+}
865+
847866 static int hugetlbfs_symlink(struct inode *dir,
848867 struct dentry *dentry, const char *symname)
849868 {
@@ -1102,6 +1121,7 @@ static const struct inode_operations hugetlbfs_dir_inode_operations = { · 252 unmodified lines
11021121 .mknod = hugetlbfs_mknod,
11031122 .rename = simple_rename,
11041123 .setattr = hugetlbfs_setattr,
1124+ .tmpfile = hugetlbfs_tmpfile,
11051125 };
11061126
11071127 static const struct inode_operations hugetlbfs_inode_operations = {
@@ -1461,28 +1481,41 @@ static int __init init_hugetlbfs_fs(void) · 353 unmodified lines
14611481 sizeof(struct hugetlbfs_inode_info),
14621482 0, SLAB_ACCOUNT, init_once);
14631483 if (hugetlbfs_inode_cachep == NULL)
1464− goto out2;
1484+ goto out;
14651485
14661486 error = register_filesystem(&hugetlbfs_fs_type);
14671487 if (error)
1468− goto out;
1488+ goto out_free;
14691489
1490+ /* default hstate mount is required */
1491+ mnt = mount_one_hugetlbfs(&hstates[default_hstate_idx]);
1492+ if (IS_ERR(mnt)) {
1493+ error = PTR_ERR(mnt);
1494+ goto out_unreg;
1495+ }
1496+ hugetlbfs_vfsmount[default_hstate_idx] = mnt;
1497+
1498+ /* other hstates are optional */
14701499 i = 0;
14711500 for_each_hstate(h) {
1501+ if (i == default_hstate_idx)
1502+ continue;
1503+
14721504 mnt = mount_one_hugetlbfs(h);
1473− if (IS_ERR(mnt) && i == 0) {
1474− error = PTR_ERR(mnt);
1475− goto out;
1476− }
1477− hugetlbfs_vfsmount[i] = mnt;
1505+ if (IS_ERR(mnt))
1506+ hugetlbfs_vfsmount[i] = NULL;
1507+ else
1508+ hugetlbfs_vfsmount[i] = mnt;
14781509 i++;
14791510 }
14801511
14811512 return 0;
14821513
1483− out:
1514+ out_unreg:
1515+ (void)unregister_filesystem(&hugetlbfs_fs_type);
1516+ out_free:
14841517 kmem_cache_destroy(hugetlbfs_inode_cachep);
1485− out2:
1518+ out:
14861519 return error;
14871520 }
14881521 fs_initcall(init_hugetlbfs_fs)
fs/ocfs2/acl.c+2−2
@@ -327,8 +327,8 @@ int ocfs2_acl_chmod(struct inode *inode, struct buffer_head *bh) · 326 unmodified lines
327327 down_read(&OCFS2_I(inode)->ip_xattr_sem);
328328 acl = ocfs2_get_acl_nolock(inode, ACL_TYPE_ACCESS, bh);
329329 up_read(&OCFS2_I(inode)->ip_xattr_sem);
330− if (IS_ERR(acl) || !acl)
331− return PTR_ERR(acl);
330+ if (IS_ERR_OR_NULL(acl))
331+ return PTR_ERR_OR_ZERO(acl);
332332 ret = __posix_acl_chmod(&acl, GFP_KERNEL, inode->i_mode);
333333 if (ret)
334334 return ret;
fs/userfaultfd.c+13−8
@@ -1460,7 +1460,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, · 1459 unmodified lines
14601460 start = vma->vm_start;
14611461 vma_end = min(end, vma->vm_end);
14621462
1463− new_flags = (vma->vm_flags & ~vm_flags) | vm_flags;
1463+ new_flags = (vma->vm_flags &
1464+ ~(VM_UFFD_MISSING|VM_UFFD_WP)) | vm_flags;
14641465 prev = vma_merge(mm, prev, start, vma_end, new_flags,
14651466 vma->anon_vma, vma->vm_file, vma->vm_pgoff,
14661467 vma_policy(vma),
@@ -1834,13 +1835,12 @@ static int userfaultfd_api(struct userfaultfd_ctx *ctx, · 367 unmodified lines
18341835 if (copy_from_user(&uffdio_api, buf, sizeof(uffdio_api)))
18351836 goto out;
18361837 features = uffdio_api.features;
1837− if (uffdio_api.api != UFFD_API || (features & ~UFFD_API_FEATURES)) {
1838− memset(&uffdio_api, 0, sizeof(uffdio_api));
1839− if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
1840− goto out;
1841− ret = -EINVAL;
1842− goto out;
1843− }
1838+ ret = -EINVAL;
1839+ if (uffdio_api.api != UFFD_API || (features & ~UFFD_API_FEATURES))
1840+ goto err_out;
1841+ ret = -EPERM;
1842+ if ((features & UFFD_FEATURE_EVENT_FORK) && !capable(CAP_SYS_PTRACE))
1843+ goto err_out;
18441844 /* report all available features and ioctls to userland */
18451845 uffdio_api.features = UFFD_API_FEATURES;
18461846 uffdio_api.ioctls = UFFD_API_IOCTLS;
@@ -1853,6 +1853,11 @@ static int userfaultfd_api(struct userfaultfd_ctx *ctx, · 6 unmodified lines
18531853 ret = 0;
18541854 out:
18551855 return ret;
1856+err_out:
1857+ memset(&uffdio_api, 0, sizeof(uffdio_api));
1858+ if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
1859+ ret = -EFAULT;
1860+ goto out;
18561861 }
18571862
18581863 static long userfaultfd_ioctl(struct file *file, unsigned cmd,
include/asm-generic/4level-fixup.h+0−1
@@ -30,7 +30,6 @@ · 29 unmodified lines
3030 #undef pud_free_tlb
3131 #define pud_free_tlb(tlb, x, addr) do { } while (0)
3232 #define pud_free(mm, x) do { } while (0)
33−#define __pud_free_tlb(tlb, x, addr) do { } while (0)
3433
3534 #undef pud_addr_end
3635 #define pud_addr_end(addr, end) (end)
include/asm-generic/5level-fixup.h+0−1
@@ -51,7 +51,6 @@ static inline int p4d_present(p4d_t p4d) · 50 unmodified lines
5151 #undef p4d_free_tlb
5252 #define p4d_free_tlb(tlb, x, addr) do { } while (0)
5353 #define p4d_free(mm, x) do { } while (0)
54−#define __p4d_free_tlb(tlb, x, addr) do { } while (0)
5554
5655 #undef p4d_addr_end
5756 #define p4d_addr_end(addr, end) (end)
include/asm-generic/pgtable-nop4d.h+1−1
@@ -50,7 +50,7 @@ static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) · 49 unmodified lines
5050 */
5151 #define p4d_alloc_one(mm, address) NULL
5252 #define p4d_free(mm, x) do { } while (0)
53−#define __p4d_free_tlb(tlb, x, a) do { } while (0)
53+#define p4d_free_tlb(tlb, x, a) do { } while (0)
5454
5555 #undef p4d_addr_end
5656 #define p4d_addr_end(addr, end) (end)
include/asm-generic/pgtable-nopmd.h+1−1
@@ -60,7 +60,7 @@ static inline pmd_t * pmd_offset(pud_t * pud, unsigned long address) · 59 unmodified lines
6060 static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
6161 {
6262 }
63−#define __pmd_free_tlb(tlb, x, a) do { } while (0)
63+#define pmd_free_tlb(tlb, x, a) do { } while (0)
6464
6565 #undef pmd_addr_end
6666 #define pmd_addr_end(addr, end) (end)
include/asm-generic/pgtable-nopud.h+1−1
@@ -59,7 +59,7 @@ static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) · 58 unmodified lines
5959 */
6060 #define pud_alloc_one(mm, address) NULL
6161 #define pud_free(mm, x) do { } while (0)
62−#define __pud_free_tlb(tlb, x, a) do { } while (0)
62+#define pud_free_tlb(tlb, x, a) do { } while (0)
6363
6464 #undef pud_addr_end
6565 #define pud_addr_end(addr, end) (end)
include/asm-generic/pgtable.h+51−0
@@ -558,8 +558,19 @@ static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) · 557 unmodified lines
558558 * Do the tests inline, but report and clear the bad entry in mm/memory.c.
559559 */
560560 void pgd_clear_bad(pgd_t *);
561+
562+#ifndef __PAGETABLE_P4D_FOLDED
561563 void p4d_clear_bad(p4d_t *);
564+#else
565+#define p4d_clear_bad(p4d) do { } while (0)
566+#endif
567+
568+#ifndef __PAGETABLE_PUD_FOLDED
562569 void pud_clear_bad(pud_t *);
570+#else
571+#define pud_clear_bad(p4d) do { } while (0)
572+#endif
573+
563574 void pmd_clear_bad(pmd_t *);
564575
565576 static inline int pgd_none_or_clear_bad(pgd_t *pgd)
@@ -903,6 +914,21 @@ static inline int pud_write(pud_t pud) · 337 unmodified lines
903914 }
904915 #endif /* pud_write */
905916
917+#if !defined(CONFIG_ARCH_HAS_PTE_DEVMAP) || !defined(CONFIG_TRANSPARENT_HUGEPAGE)
918+static inline int pmd_devmap(pmd_t pmd)
919+{
920+ return 0;
921+}
922+static inline int pud_devmap(pud_t pud)
923+{
924+ return 0;
925+}
926+static inline int pgd_devmap(pgd_t pgd)
927+{
928+ return 0;
929+}
930+#endif
931+
906932 #if !defined(CONFIG_TRANSPARENT_HUGEPAGE) || \
907933 (defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
908934 !defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD))
@@ -912,6 +938,31 @@ static inline int pud_trans_huge(pud_t pud) · 3 unmodified lines
912938 }
913939 #endif
914940
941+/* See pmd_none_or_trans_huge_or_clear_bad for discussion. */
942+static inline int pud_none_or_trans_huge_or_dev_or_clear_bad(pud_t *pud)
943+{
944+ pud_t pudval = READ_ONCE(*pud);
945+
946+ if (pud_none(pudval) || pud_trans_huge(pudval) || pud_devmap(pudval))
947+ return 1;
948+ if (unlikely(pud_bad(pudval))) {
949+ pud_clear_bad(pud);
950+ return 1;
951+ }
952+ return 0;
953+}
954+
955+/* See pmd_trans_unstable for discussion. */
956+static inline int pud_trans_unstable(pud_t *pud)
957+{
958+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
959+ defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
960+ return pud_none_or_trans_huge_or_dev_or_clear_bad(pud);
961+#else
962+ return 0;
963+#endif
964+}
965+
915966 #ifndef pmd_read_atomic
916967 static inline pmd_t pmd_read_atomic(pmd_t *pmdp)
917968 {
include/asm-generic/tlb.h+0−4
@@ -584,7 +584,6 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm · 583 unmodified lines
584584 } while (0)
585585 #endif
586586
587−#ifndef __ARCH_HAS_4LEVEL_HACK
588587 #ifndef pud_free_tlb
589588 #define pud_free_tlb(tlb, pudp, address) \
590589 do { \
@@ -594,9 +593,7 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm · 3 unmodified lines
594593 __pud_free_tlb(tlb, pudp, address); \
595594 } while (0)
596595 #endif
597−#endif
598596
599−#ifndef __ARCH_HAS_5LEVEL_HACK
600597 #ifndef p4d_free_tlb
601598 #define p4d_free_tlb(tlb, pudp, address) \
602599 do { \
@@ -605,7 +602,6 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm · 2 unmodified lines
605602 __p4d_free_tlb(tlb, pudp, address); \
606603 } while (0)
607604 #endif
608−#endif
609605
610606 #endif /* CONFIG_MMU */
611607
include/linux/fs.h+5−1
@@ -3156,7 +3156,6 @@ enum { · 3155 unmodified lines
31563156 };
31573157
31583158 void dio_end_io(struct bio *bio);
3159−void dio_warn_stale_pagecache(struct file *filp);
31603159
31613160 ssize_t __blockdev_direct_IO(struct kiocb *iocb, struct inode *inode,
31623161 struct block_device *bdev, struct iov_iter *iter,
@@ -3201,6 +3200,11 @@ static inline void inode_dio_end(struct inode *inode) · 38 unmodified lines
32013200 wake_up_bit(&inode->i_state, __I_DIO_WAKEUP);
32023201 }
32033202
3203+/*
3204+ * Warn about a page cache invalidation failure diring a direct I/O write.
3205+ */
3206+void dio_warn_stale_pagecache(struct file *filp);
3207+
32043208 extern void inode_set_flags(struct inode *inode, unsigned int flags,
32053209 unsigned int mask);
32063210
include/linux/gfp.h+2−0
@@ -612,6 +612,8 @@ static inline bool pm_suspended_storage(void) · 611 unmodified lines
612612 /* The below functions must be run on a range from a single zone. */
613613 extern int alloc_contig_range(unsigned long start, unsigned long end,
614614 unsigned migratetype, gfp_t gfp_mask);
615+extern struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask,
616+ int nid, nodemask_t *nodemask);
615617 #endif
616618 void free_contig_range(unsigned long pfn, unsigned int nr_pages);
617619
include/linux/hugetlb.h+116−24
@@ -105,8 +105,7 @@ void move_hugetlb_state(struct page *oldpage, struct page *newpage, int reason); · 104 unmodified lines
105105 void free_huge_page(struct page *page);
106106 void hugetlb_fix_reserve_counts(struct inode *inode);
107107 extern struct mutex *hugetlb_fault_mutex_table;
108−u32 hugetlb_fault_mutex_hash(struct hstate *h, struct address_space *mapping,
109− pgoff_t idx, unsigned long address);
108+u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx);
110109
111110 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud);
112111
@@ -164,38 +163,130 @@ static inline void adjust_range_if_pmd_sharing_possible( · 51 unmodified lines
164163 {
165164 }
166165
167−#define follow_hugetlb_page(m,v,p,vs,a,b,i,w,n) ({ BUG(); 0; })
168−#define follow_huge_addr(mm, addr, write) ERR_PTR(-EINVAL)
169−#define copy_hugetlb_page_range(src, dst, vma) ({ BUG(); 0; })
166+static inline long follow_hugetlb_page(struct mm_struct *mm,
167+ struct vm_area_struct *vma, struct page **pages,
168+ struct vm_area_struct **vmas, unsigned long *position,
169+ unsigned long *nr_pages, long i, unsigned int flags,
170+ int *nonblocking)
171+{
172+ BUG();
173+ return 0;
174+}
175+
176+static inline struct page *follow_huge_addr(struct mm_struct *mm,
177+ unsigned long address, int write)
178+{
179+ return ERR_PTR(-EINVAL);
180+}
181+
182+static inline int copy_hugetlb_page_range(struct mm_struct *dst,
183+ struct mm_struct *src, struct vm_area_struct *vma)
184+{
185+ BUG();
186+ return 0;
187+}
188+
170189 static inline void hugetlb_report_meminfo(struct seq_file *m)
171190 {
172191 }
173−#define hugetlb_report_node_meminfo(n, buf) 0
192+
193+static inline int hugetlb_report_node_meminfo(int nid, char *buf)
194+{
195+ return 0;
196+}
197+
174198 static inline void hugetlb_show_meminfo(void)
175199 {
176200 }
177−#define follow_huge_pd(vma, addr, hpd, flags, pdshift) NULL
178−#define follow_huge_pmd(mm, addr, pmd, flags) NULL
179−#define follow_huge_pud(mm, addr, pud, flags) NULL
180−#define follow_huge_pgd(mm, addr, pgd, flags) NULL
181−#define prepare_hugepage_range(file, addr, len) (-EINVAL)
182−#define pmd_huge(x) 0
183−#define pud_huge(x) 0
184−#define is_hugepage_only_range(mm, addr, len) 0
185−#define hugetlb_free_pgd_range(tlb, addr, end, floor, ceiling) ({BUG(); 0; })
186−#define hugetlb_mcopy_atomic_pte(dst_mm, dst_pte, dst_vma, dst_addr, \
187− src_addr, pagep) ({ BUG(); 0; })
188−#define huge_pte_offset(mm, address, sz) 0
201+
202+static inline struct page *follow_huge_pd(struct vm_area_struct *vma,
203+ unsigned long address, hugepd_t hpd, int flags,
204+ int pdshift)
205+{
206+ return NULL;
207+}
208+
209+static inline struct page *follow_huge_pmd(struct mm_struct *mm,
210+ unsigned long address, pmd_t *pmd, int flags)
211+{
212+ return NULL;
213+}
214+
215+static inline struct page *follow_huge_pud(struct mm_struct *mm,
216+ unsigned long address, pud_t *pud, int flags)
217+{
218+ return NULL;
219+}
220+
221+static inline struct page *follow_huge_pgd(struct mm_struct *mm,
222+ unsigned long address, pgd_t *pgd, int flags)
223+{
224+ return NULL;
225+}
226+
227+static inline int prepare_hugepage_range(struct file *file,
228+ unsigned long addr, unsigned long len)
229+{
230+ return -EINVAL;
231+}
232+
233+static inline int pmd_huge(pmd_t pmd)
234+{
235+ return 0;
236+}
237+
238+static inline int pud_huge(pud_t pud)
239+{
240+ return 0;
241+}
242+
243+static inline int is_hugepage_only_range(struct mm_struct *mm,
244+ unsigned long addr, unsigned long len)
245+{
246+ return 0;
247+}
248+
249+static inline void hugetlb_free_pgd_range(struct mmu_gather *tlb,
250+ unsigned long addr, unsigned long end,
251+ unsigned long floor, unsigned long ceiling)
252+{
253+ BUG();
254+}
255+
256+static inline int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm,
257+ pte_t *dst_pte,
258+ struct vm_area_struct *dst_vma,
259+ unsigned long dst_addr,
260+ unsigned long src_addr,
261+ struct page **pagep)
262+{
263+ BUG();
264+ return 0;
265+}
266+
267+static inline pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr,
268+ unsigned long sz)
269+{
270+ return NULL;
271+}
189272
190273 static inline bool isolate_huge_page(struct page *page, struct list_head *list)
191274 {
192275 return false;
193276 }
194−#define putback_active_hugepage(p) do {} while (0)
195−#define move_hugetlb_state(old, new, reason) do {} while (0)
196277
197−static inline unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
198− unsigned long address, unsigned long end, pgprot_t newprot)
278+static inline void putback_active_hugepage(struct page *page)
279+{
280+}
281+
282+static inline void move_hugetlb_state(struct page *oldpage,
283+ struct page *newpage, int reason)
284+{
285+}
286+
287+static inline unsigned long hugetlb_change_protection(
288+ struct vm_area_struct *vma, unsigned long address,
289+ unsigned long end, pgprot_t newprot)
199290 {
200291 return 0;
201292 }
@@ -213,9 +304,10 @@ static inline void __unmap_hugepage_range(struct mmu_gather *tlb, · 11 unmodified lines
213304 {
214305 BUG();
215306 }
307+
216308 static inline vm_fault_t hugetlb_fault(struct mm_struct *mm,
217− struct vm_area_struct *vma, unsigned long address,
218− unsigned int flags)
309+ struct vm_area_struct *vma, unsigned long address,
310+ unsigned int flags)
219311 {
220312 BUG();
221313 return 0;
include/linux/kasan.h+31−0
@@ -70,8 +70,18 @@ struct kasan_cache { · 69 unmodified lines
7070 int free_meta_offset;
7171 };
7272
73+/*
74+ * These functions provide a special case to support backing module
75+ * allocations with real shadow memory. With KASAN vmalloc, the special
76+ * case is unnecessary, as the work is handled in the generic case.
77+ */
78+#ifndef CONFIG_KASAN_VMALLOC
7379 int kasan_module_alloc(void *addr, size_t size);
7480 void kasan_free_shadow(const struct vm_struct *vm);
81+#else
82+static inline int kasan_module_alloc(void *addr, size_t size) { return 0; }
83+static inline void kasan_free_shadow(const struct vm_struct *vm) {}
84+#endif
7585
7686 int kasan_add_zero_shadow(void *start, unsigned long size);
7787 void kasan_remove_zero_shadow(void *start, unsigned long size);
@@ -194,4 +204,25 @@ static inline void *kasan_reset_tag(const void *addr) · 116 unmodified lines
194204
195205 #endif /* CONFIG_KASAN_SW_TAGS */
196206
207+#ifdef CONFIG_KASAN_VMALLOC
208+int kasan_populate_vmalloc(unsigned long requested_size,
209+ struct vm_struct *area);
210+void kasan_poison_vmalloc(void *start, unsigned long size);
211+void kasan_release_vmalloc(unsigned long start, unsigned long end,
212+ unsigned long free_region_start,
213+ unsigned long free_region_end);
214+#else
215+static inline int kasan_populate_vmalloc(unsigned long requested_size,
216+ struct vm_struct *area)
217+{
218+ return 0;
219+}
220+
221+static inline void kasan_poison_vmalloc(void *start, unsigned long size) {}
222+static inline void kasan_release_vmalloc(unsigned long start,
223+ unsigned long end,
224+ unsigned long free_region_start,
225+ unsigned long free_region_end) {}
226+#endif
227+
197228 #endif /* LINUX_KASAN_H */
include/linux/memblock.h+3−0
@@ -358,6 +358,9 @@ static inline phys_addr_t memblock_phys_alloc(phys_addr_t size, · 357 unmodified lines
358358 MEMBLOCK_ALLOC_ACCESSIBLE);
359359 }
360360
361+void *memblock_alloc_exact_nid_raw(phys_addr_t size, phys_addr_t align,
362+ phys_addr_t min_addr, phys_addr_t max_addr,
363+ int nid);
361364 void *memblock_alloc_try_nid_raw(phys_addr_t size, phys_addr_t align,
362365 phys_addr_t min_addr, phys_addr_t max_addr,
363366 int nid);
include/linux/memcontrol.h+22−27
@@ -58,7 +58,6 @@ enum mem_cgroup_protection { · 57 unmodified lines
5858
5959 struct mem_cgroup_reclaim_cookie {
6060 pg_data_t *pgdat;
61− int priority;
6261 unsigned int generation;
6362 };
6463
@@ -81,7 +80,6 @@ struct mem_cgroup_id { · 16 unmodified lines
8180 enum mem_cgroup_events_target {
8281 MEM_CGROUP_TARGET_THRESH,
8382 MEM_CGROUP_TARGET_SOFTLIMIT,
84− MEM_CGROUP_TARGET_NUMAINFO,
8583 MEM_CGROUP_NTARGETS,
8684 };
8785
@@ -112,7 +110,7 @@ struct memcg_shrinker_map { · 24 unmodified lines
112110 };
113111
114112 /*
115− * per-zone information in memory controller.
113+ * per-node information in memory controller.
116114 */
117115 struct mem_cgroup_per_node {
118116 struct lruvec lruvec;
@@ -126,7 +124,7 @@ struct mem_cgroup_per_node { · 7 unmodified lines
126124
127125 unsigned long lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS];
128126
129− struct mem_cgroup_reclaim_iter iter[DEF_PRIORITY + 1];
127+ struct mem_cgroup_reclaim_iter iter;
130128
131129 struct memcg_shrinker_map __rcu *shrinker_map;
132130
@@ -134,9 +132,6 @@ struct mem_cgroup_per_node { · 1 unmodified line
134132 unsigned long usage_in_excess;/* Set to the value by which */
135133 /* the soft limit is exceeded*/
136134 bool on_tree;
137− bool congested; /* memcg has many dirty pages */
138− /* backed by a congested BDI */
139−
140135 struct mem_cgroup *memcg; /* Back pointer, we cannot */
141136 /* use container_of */
142137 };
@@ -313,13 +308,6 @@ struct mem_cgroup { · 170 unmodified lines
313308 struct list_head kmem_caches;
314309 #endif
315310
316− int last_scanned_node;
317−#if MAX_NUMNODES > 1
318− nodemask_t scan_nodes;
319− atomic_t numainfo_events;
320− atomic_t numainfo_updating;
321−#endif
322−
323311 #ifdef CONFIG_CGROUP_WRITEBACK
324312 struct list_head cgwb_list;
325313 struct wb_domain cgwb_domain;
@@ -394,25 +382,27 @@ mem_cgroup_nodeinfo(struct mem_cgroup *memcg, int nid) · 68 unmodified lines
394382 }
395383
396384 /**
397− * mem_cgroup_lruvec - get the lru list vector for a node or a memcg zone
398− * @node: node of the wanted lruvec
385+ * mem_cgroup_lruvec - get the lru list vector for a memcg & node
399386 * @memcg: memcg of the wanted lruvec
400387 *
401− * Returns the lru list vector holding pages for a given @node or a given
402− * @memcg and @zone. This can be the node lruvec, if the memory controller
403− * is disabled.
388+ * Returns the lru list vector holding pages for a given @memcg &
389+ * @node combination. This can be the node lruvec, if the memory
390+ * controller is disabled.
404391 */
405−static inline struct lruvec *mem_cgroup_lruvec(struct pglist_data *pgdat,
406− struct mem_cgroup *memcg)
392+static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,
393+ struct pglist_data *pgdat)
407394 {
408395 struct mem_cgroup_per_node *mz;
409396 struct lruvec *lruvec;
410397
411398 if (mem_cgroup_disabled()) {
412− lruvec = node_lruvec(pgdat);
399+ lruvec = &pgdat->__lruvec;
413400 goto out;
414401 }
415402
403+ if (!memcg)
404+ memcg = root_mem_cgroup;
405+
416406 mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
417407 lruvec = &mz->lruvec;
418408 out:
@@ -728,7 +718,7 @@ static inline void __mod_lruvec_page_state(struct page *page, · 309 unmodified lines
728718 return;
729719 }
730720
731− lruvec = mem_cgroup_lruvec(pgdat, page->mem_cgroup);
721+ lruvec = mem_cgroup_lruvec(page->mem_cgroup, pgdat);
732722 __mod_lruvec_state(lruvec, idx, val);
733723 }
734724
@@ -899,16 +889,21 @@ static inline void mem_cgroup_migrate(struct page *old, struct page *new) · 164 unmodified lines
899889 {
900890 }
901891
902−static inline struct lruvec *mem_cgroup_lruvec(struct pglist_data *pgdat,
903− struct mem_cgroup *memcg)
892+static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,
893+ struct pglist_data *pgdat)
904894 {
905− return node_lruvec(pgdat);
895+ return &pgdat->__lruvec;
906896 }
907897
908898 static inline struct lruvec *mem_cgroup_page_lruvec(struct page *page,
909899 struct pglist_data *pgdat)
910900 {
911− return &pgdat->lruvec;
901+ return &pgdat->__lruvec;
902+}
903+
904+static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
905+{
906+ return NULL;
912907 }
913908
914909 static inline bool mm_match_cgroup(struct mm_struct *mm,
include/linux/memory_hotplug.h+4−7
@@ -102,13 +102,10 @@ extern unsigned long __offline_isolated_pages(unsigned long start_pfn, · 101 unmodified lines
102102
103103 typedef void (*online_page_callback_t)(struct page *page, unsigned int order);
104104
105+extern void generic_online_page(struct page *page, unsigned int order);
105106 extern int set_online_page_callback(online_page_callback_t callback);
106107 extern int restore_online_page_callback(online_page_callback_t callback);
107108
108−extern void __online_page_set_limits(struct page *page);
109−extern void __online_page_increment_counters(struct page *page);
110−extern void __online_page_free(struct page *page);
111−
112109 extern int try_online_node(int nid);
113110
114111 extern int arch_add_memory(int nid, u64 start, u64 size,
@@ -229,9 +226,6 @@ void put_online_mems(void); · 114 unmodified lines
229226 void mem_hotplug_begin(void);
230227 void mem_hotplug_done(void);
231228
232−extern void set_zone_contiguous(struct zone *zone);
233−extern void clear_zone_contiguous(struct zone *zone);
234−
235229 #else /* ! CONFIG_MEMORY_HOTPLUG */
236230 #define pfn_to_online_page(pfn) \
237231 ({ \
@@ -339,6 +333,9 @@ static inline int remove_memory(int nid, u64 start, u64 size) · 101 unmodified lines
339333 static inline void __remove_memory(int nid, u64 start, u64 size) {}
340334 #endif /* CONFIG_MEMORY_HOTREMOVE */
341335
336+extern void set_zone_contiguous(struct zone *zone);
337+extern void clear_zone_contiguous(struct zone *zone);
338+
342339 extern void __ref free_area_init_core_hotplug(int nid);
343340 extern int __add_memory(int nid, u64 start, u64 size);
344341 extern int add_memory(int nid, u64 start, u64 size);
include/linux/mm.h+12−22
@@ -564,21 +564,6 @@ int vma_is_stack_for_current(struct vm_area_struct *vma); · 563 unmodified lines
564564 struct mmu_gather;
565565 struct inode;
566566
567−#if !defined(CONFIG_ARCH_HAS_PTE_DEVMAP) || !defined(CONFIG_TRANSPARENT_HUGEPAGE)
568−static inline int pmd_devmap(pmd_t pmd)
569−{
570− return 0;
571−}
572−static inline int pud_devmap(pud_t pud)
573−{
574− return 0;
575−}
576−static inline int pgd_devmap(pgd_t pgd)
577−{
578− return 0;
579−}
580−#endif
581−
582567 /*
583568 * FIXME: take this include out, include page-flags.h in
584569 * files which need it (119 of them)
@@ -1643,19 +1628,27 @@ static inline unsigned long get_mm_counter(struct mm_struct *mm, int member) · 1058 unmodified lines
16431628 return (unsigned long)val;
16441629 }
16451630
1631+void mm_trace_rss_stat(struct mm_struct *mm, int member, long count);
1632+
16461633 static inline void add_mm_counter(struct mm_struct *mm, int member, long value)
16471634 {
1648− atomic_long_add(value, &mm->rss_stat.count[member]);
1635+ long count = atomic_long_add_return(value, &mm->rss_stat.count[member]);
1636+
1637+ mm_trace_rss_stat(mm, member, count);
16491638 }
16501639
16511640 static inline void inc_mm_counter(struct mm_struct *mm, int member)
16521641 {
1653− atomic_long_inc(&mm->rss_stat.count[member]);
1642+ long count = atomic_long_inc_return(&mm->rss_stat.count[member]);
1643+
1644+ mm_trace_rss_stat(mm, member, count);
16541645 }
16551646
16561647 static inline void dec_mm_counter(struct mm_struct *mm, int member)
16571648 {
1658− atomic_long_dec(&mm->rss_stat.count[member]);
1649+ long count = atomic_long_dec_return(&mm->rss_stat.count[member]);
1650+
1651+ mm_trace_rss_stat(mm, member, count);
16591652 }
16601653
16611654 /* Optimized variant when page is already known not to be PageAnon */
@@ -2214,9 +2207,6 @@ void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...); · 552 unmodified lines
22142207
22152208 extern void setup_per_cpu_pageset(void);
22162209
2217−extern void zone_pcp_update(struct zone *zone);
2218−extern void zone_pcp_reset(struct zone *zone);
2219−
22202210 /* page_alloc.c */
22212211 extern int min_free_kbytes;
22222212 extern int watermark_boost_factor;
@@ -2780,7 +2770,7 @@ extern int sysctl_memory_failure_early_kill; · 557 unmodified lines
27802770 extern int sysctl_memory_failure_recovery;
27812771 extern void shake_page(struct page *p, int access);
27822772 extern atomic_long_t num_poisoned_pages __read_mostly;
2783−extern int soft_offline_page(struct page *page, int flags);
2773+extern int soft_offline_page(unsigned long pfn, int flags);
27842774
27852775
27862776 /*
include/linux/mmzone.h+20−14
@@ -273,12 +273,12 @@ enum lru_list { · 272 unmodified lines
273273
274274 #define for_each_evictable_lru(lru) for (lru = 0; lru <= LRU_ACTIVE_FILE; lru++)
275275
276−static inline int is_file_lru(enum lru_list lru)
276+static inline bool is_file_lru(enum lru_list lru)
277277 {
278278 return (lru == LRU_INACTIVE_FILE || lru == LRU_ACTIVE_FILE);
279279 }
280280
281−static inline int is_active_lru(enum lru_list lru)
281+static inline bool is_active_lru(enum lru_list lru)
282282 {
283283 return (lru == LRU_ACTIVE_ANON || lru == LRU_ACTIVE_FILE);
284284 }
@@ -296,6 +296,12 @@ struct zone_reclaim_stat { · 11 unmodified lines
296296 unsigned long recent_scanned[2];
297297 };
298298
299+enum lruvec_flags {
300+ LRUVEC_CONGESTED, /* lruvec has many dirty pages
301+ * backed by a congested BDI
302+ */
303+};
304+
299305 struct lruvec {
300306 struct list_head lists[NR_LRU_LISTS];
301307 struct zone_reclaim_stat reclaim_stat;
@@ -303,12 +309,14 @@ struct lruvec { · 1 unmodified line
303309 atomic_long_t inactive_age;
304310 /* Refaults at the time of last reclaim cycle */
305311 unsigned long refaults;
312+ /* Various lruvec state flags (enum lruvec_flags) */
313+ unsigned long flags;
306314 #ifdef CONFIG_MEMCG
307315 struct pglist_data *pgdat;
308316 #endif
309317 };
310318
311−/* Isolate unmapped file */
319+/* Isolate unmapped pages */
312320 #define ISOLATE_UNMAPPED ((__force isolate_mode_t)0x2)
313321 /* Isolate for asynchronous migration */
314322 #define ISOLATE_ASYNC_MIGRATE ((__force isolate_mode_t)0x4)
@@ -572,9 +580,6 @@ struct zone { · 257 unmodified lines
572580 } ____cacheline_internodealigned_in_smp;
573581
574582 enum pgdat_flags {
575− PGDAT_CONGESTED, /* pgdat has many dirty pages backed by
576− * a congested BDI
577− */
578583 PGDAT_DIRTY, /* reclaim scanning has recently found
579584 * many dirty file pages at the tail
580585 * of the LRU.
@@ -777,7 +782,13 @@ typedef struct pglist_data { · 196 unmodified lines
777782 #endif
778783
779784 /* Fields commonly accessed by the page reclaim scanner */
780− struct lruvec lruvec;
785+
786+ /*
787+ * NOTE: THIS IS UNUSED IF MEMCG IS ENABLED.
788+ *
789+ * Use mem_cgroup_lruvec() to look up lruvecs.
790+ */
791+ struct lruvec __lruvec;
781792
782793 unsigned long flags;
783794
@@ -800,11 +811,6 @@ typedef struct pglist_data { · 16 unmodified lines
800811 #define node_start_pfn(nid) (NODE_DATA(nid)->node_start_pfn)
801812 #define node_end_pfn(nid) pgdat_end_pfn(NODE_DATA(nid))
802813
803−static inline struct lruvec *node_lruvec(struct pglist_data *pgdat)
804−{
805− return &pgdat->lruvec;
806−}
807−
808814 static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat)
809815 {
810816 return pgdat->node_start_pfn + pgdat->node_spanned_pages;
@@ -842,7 +848,7 @@ static inline struct pglist_data *lruvec_pgdat(struct lruvec *lruvec) · 31 unmodified lines
842848 #ifdef CONFIG_MEMCG
843849 return lruvec->pgdat;
844850 #else
845− return container_of(lruvec, struct pglist_data, lruvec);
851+ return container_of(lruvec, struct pglist_data, __lruvec);
846852 #endif
847853 }
848854
@@ -1079,7 +1085,7 @@ static inline struct zoneref *first_zones_zonelist(struct zonelist *zonelist, · 230 unmodified lines
10791085 /**
10801086 * for_each_zone_zonelist_nodemask - helper macro to iterate over valid zones in a zonelist at or below a given zone index and within a nodemask
10811087 * @zone - The current zone in the iterator
1082− * @z - The current pointer within zonelist->zones being iterated
1088+ * @z - The current pointer within zonelist->_zonerefs being iterated
10831089 * @zlist - The zonelist being iterated
10841090 * @highidx - The zone index of the highest zone to return
10851091 * @nodemask - Nodemask allowed by the allocator
include/linux/moduleloader.h+1−1
@@ -91,7 +91,7 @@ void module_arch_cleanup(struct module *mod); · 90 unmodified lines
9191 /* Any cleanup before freeing mod->module_init */
9292 void module_arch_freeing_init(struct module *mod);
9393
94−#ifdef CONFIG_KASAN
94+#if defined(CONFIG_KASAN) && !defined(CONFIG_KASAN_VMALLOC)
9595 #include <linux/kasan.h>
9696 #define MODULE_ALIGN (PAGE_SIZE << KASAN_SHADOW_SCALE_SHIFT)
9797 #else
include/linux/page-isolation.h+2−2
@@ -30,7 +30,7 @@ static inline bool is_migrate_isolate(int migratetype) · 29 unmodified lines
3030 }
3131 #endif
3232
33−#define SKIP_HWPOISON 0x1
33+#define MEMORY_OFFLINE 0x1
3434 #define REPORT_FAILURE 0x2
3535
3636 bool has_unmovable_pages(struct zone *zone, struct page *page, int count,
@@ -58,7 +58,7 @@ undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, · 21 unmodified lines
5858 * Test all pages in [start_pfn, end_pfn) are isolated or not.
5959 */
6060 int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn,
61− bool skip_hwpoisoned_pages);
61+ int isol_flags);
6262
6363 struct page *alloc_migrate_target(struct page *page, unsigned long private);
6464
include/linux/slab.h+0−20
@@ -561,26 +561,6 @@ static __always_inline void *kmalloc(size_t size, gfp_t flags) · 560 unmodified lines
561561 return __kmalloc(size, flags);
562562 }
563563
564−/*
565− * Determine size used for the nth kmalloc cache.
566− * return size or 0 if a kmalloc cache for that
567− * size does not exist
568− */
569−static __always_inline unsigned int kmalloc_size(unsigned int n)
570−{
571−#ifndef CONFIG_SLOB
572− if (n > 2)
573− return 1U << n;
574−
575− if (n == 1 && KMALLOC_MIN_SIZE <= 32)
576− return 96;
577−
578− if (n == 2 && KMALLOC_MIN_SIZE <= 64)
579− return 192;
580−#endif
581− return 0;
582−}
583−
584564 static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
585565 {
586566 #ifndef CONFIG_SLOB
include/linux/string.h+2−0
@@ -216,6 +216,8 @@ int bprintf(u32 *bin_buf, size_t size, const char *fmt, ...) __printf(3, 4); · 215 unmodified lines
216216 extern ssize_t memory_read_from_buffer(void *to, size_t count, loff_t *ppos,
217217 const void *from, size_t available);
218218
219+int ptr_to_hashval(const void *ptr, unsigned long *hashval_out);
220+
219221 /**
220222 * strstarts - does @str start with @prefix?
221223 * @str: string to examine
include/linux/swap.h+1−1
@@ -307,7 +307,7 @@ struct vma_swap_readahead { · 306 unmodified lines
307307 };
308308
309309 /* linux/mm/workingset.c */
310−void *workingset_eviction(struct page *page);
310+void *workingset_eviction(struct page *page, struct mem_cgroup *target_memcg);
311311 void workingset_refault(struct page *page, void *shadow);
312312 void workingset_activation(struct page *page);
313313
include/linux/vmalloc.h+12−0
@@ -22,6 +22,18 @@ struct notifier_block; /* in notifier.h */ · 21 unmodified lines
2222 #define VM_UNINITIALIZED 0x00000020 /* vm_struct is not fully initialized */
2323 #define VM_NO_GUARD 0x00000040 /* don't add guard page */
2424 #define VM_KASAN 0x00000080 /* has allocated kasan shadow memory */
25+
26+/*
27+ * VM_KASAN is used slighly differently depending on CONFIG_KASAN_VMALLOC.
28+ *
29+ * If IS_ENABLED(CONFIG_KASAN_VMALLOC), VM_KASAN is set on a vm_struct after
30+ * shadow memory has been mapped. It's used to handle allocation errors so that
31+ * we don't try to poision shadow on free if it was never allocated.
32+ *
33+ * Otherwise, VM_KASAN is set for kasan_module_alloc() allocations and used to
34+ * determine which allocations need the module shadow freed.
35+ */
36+
2537 /*
2638 * Memory with VM_FLUSH_RESET_PERMS cannot be freed in an interrupt or with
2739 * vfree_atomic().
include/trace/events/kmem.h+47−0
@@ -316,6 +316,53 @@ TRACE_EVENT(mm_page_alloc_extfrag, · 315 unmodified lines
316316 __entry->change_ownership)
317317 );
318318
319+/*
320+ * Required for uniquely and securely identifying mm in rss_stat tracepoint.
321+ */
322+#ifndef __PTR_TO_HASHVAL
323+static unsigned int __maybe_unused mm_ptr_to_hash(const void *ptr)
324+{
325+ int ret;
326+ unsigned long hashval;
327+
328+ ret = ptr_to_hashval(ptr, &hashval);
329+ if (ret)
330+ return 0;
331+
332+ /* The hashed value is only 32-bit */
333+ return (unsigned int)hashval;
334+}
335+#define __PTR_TO_HASHVAL
336+#endif
337+
338+TRACE_EVENT(rss_stat,
339+
340+ TP_PROTO(struct mm_struct *mm,
341+ int member,
342+ long count),
343+
344+ TP_ARGS(mm, member, count),
345+
346+ TP_STRUCT__entry(
347+ __field(unsigned int, mm_id)
348+ __field(unsigned int, curr)
349+ __field(int, member)
350+ __field(long, size)
351+ ),
352+
353+ TP_fast_assign(
354+ __entry->mm_id = mm_ptr_to_hash(mm);
355+ __entry->curr = !!(current->mm == mm);
356+ __entry->member = member;
357+ __entry->size = (count << PAGE_SHIFT);
358+ ),
359+
360+ TP_printk("mm_id=%u curr=%d member=%d size=%ldB",
361+ __entry->mm_id,
362+ __entry->curr,
363+ __entry->member,
364+ __entry->size)
365+ );
319366 #endif /* _TRACE_KMEM_H */
320367
321368 /* This part must be outside protection */
kernel/events/uprobes.c+1−1
@@ -1457,7 +1457,7 @@ static int xol_add_vma(struct mm_struct *mm, struct xol_area *area) · 1456 unmodified lines
14571457 /* Try to map as high as possible, this is only a hint. */
14581458 area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE,
14591459 PAGE_SIZE, 0, 0);
1460− if (area->vaddr & ~PAGE_MASK) {
1460+ if (IS_ERR_VALUE(area->vaddr)) {
14611461 ret = area->vaddr;
14621462 goto fail;
14631463 }
kernel/fork.c+4−0
@@ -93,6 +93,7 @@ · 92 unmodified lines
9393 #include <linux/livepatch.h>
9494 #include <linux/thread_info.h>
9595 #include <linux/stackleak.h>
96+#include <linux/kasan.h>
9697
9798 #include <asm/pgtable.h>
9899 #include <asm/pgalloc.h>
@@ -223,6 +224,9 @@ static unsigned long *alloc_thread_stack_node(struct task_struct *tsk, int node) · 124 unmodified lines
223224 if (!s)
224225 continue;
225226
227+ /* Clear the KASAN shadow of the stack. */
228+ kasan_unpoison_shadow(s->addr, THREAD_SIZE);
229+
226230 /* Clear stale pointers from reused stack. */
227231 memset(s->addr, 0, THREAD_SIZE);
228232
kernel/sysctl.c+1−1
@@ -1466,7 +1466,7 @@ static struct ctl_table vm_table[] = { · 1465 unmodified lines
14661466 .procname = "drop_caches",
14671467 .data = &sysctl_drop_caches,
14681468 .maxlen = sizeof(int),
1469− .mode = 0644,
1469+ .mode = 0200,
14701470 .proc_handler = drop_caches_sysctl_handler,
14711471 .extra1 = SYSCTL_ONE,
14721472 .extra2 = &four,
lib/Kconfig.kasan+16−0
@@ -6,6 +6,9 @@ config HAVE_ARCH_KASAN · 5 unmodified lines
66 config HAVE_ARCH_KASAN_SW_TAGS
77 bool
88
9+config HAVE_ARCH_KASAN_VMALLOC
10+ bool
11+
912 config CC_HAS_KASAN_GENERIC
1013 def_bool $(cc-option, -fsanitize=kernel-address)
1114
@@ -142,6 +145,19 @@ config KASAN_SW_TAGS_IDENTIFY · 130 unmodified lines
142145 (use-after-free or out-of-bounds) at the cost of increased
143146 memory consumption.
144147
148+config KASAN_VMALLOC
149+ bool "Back mappings in vmalloc space with real shadow memory"
150+ depends on KASAN && HAVE_ARCH_KASAN_VMALLOC
151+ help
152+ By default, the shadow region for vmalloc space is the read-only
153+ zero page. This means that KASAN cannot detect errors involving
154+ vmalloc space.
155+
156+ Enabling this option will hook in to vmap/vmalloc and back those
157+ mappings with real shadow memory allocated on demand. This allows
158+ for KASAN to detect more sorts of errors (and to support vmapped
159+ stacks), but at the cost of higher memory usage.
160+
145161 config TEST_KASAN
146162 tristate "Module for testing KASAN for bug detection"
147163 depends on m && KASAN
lib/test_kasan.c+26−0
@@ -19,6 +19,7 @@ · 18 unmodified lines
1919 #include <linux/string.h>
2020 #include <linux/uaccess.h>
2121 #include <linux/io.h>
22+#include <linux/vmalloc.h>
2223
2324 #include <asm/page.h>
2425
@@ -748,6 +749,30 @@ static noinline void __init kmalloc_double_kzfree(void) · 723 unmodified lines
748749 kzfree(ptr);
749750 }
750751
752+#ifdef CONFIG_KASAN_VMALLOC
753+static noinline void __init vmalloc_oob(void)
754+{
755+ void *area;
756+
757+ pr_info("vmalloc out-of-bounds\n");
758+
759+ /*
760+ * We have to be careful not to hit the guard page.
761+ * The MMU will catch that and crash us.
762+ */
763+ area = vmalloc(3000);
764+ if (!area) {
765+ pr_err("Allocation failed\n");
766+ return;
767+ }
768+
769+ ((volatile char *)area)[3100];
770+ vfree(area);
771+}
772+#else
773+static void __init vmalloc_oob(void) {}
774+#endif
775+
751776 static int __init kmalloc_tests_init(void)
752777 {
753778 /*
@@ -793,6 +818,7 @@ static int __init kmalloc_tests_init(void) · 39 unmodified lines
793818 kasan_strings();
794819 kasan_bitops();
795820 kmalloc_double_kzfree();
821+ vmalloc_oob();
796822
797823 kasan_restore_multi_shot(multishot);
798824
lib/vsprintf.c+29−11
@@ -761,11 +761,38 @@ static int __init initialize_ptr_random(void) · 760 unmodified lines
761761 early_initcall(initialize_ptr_random);
762762
763763 /* Maps a pointer to a 32 bit unique identifier. */
764+static inline int __ptr_to_hashval(const void *ptr, unsigned long *hashval_out)
765+{
766+ unsigned long hashval;
767+
768+ if (static_branch_unlikely(&not_filled_random_ptr_key))
769+ return -EAGAIN;
770+
771+#ifdef CONFIG_64BIT
772+ hashval = (unsigned long)siphash_1u64((u64)ptr, &ptr_key);
773+ /*
774+ * Mask off the first 32 bits, this makes explicit that we have
775+ * modified the address (and 32 bits is plenty for a unique ID).
776+ */
777+ hashval = hashval & 0xffffffff;
778+#else
779+ hashval = (unsigned long)siphash_1u32((u32)ptr, &ptr_key);
780+#endif
781+ *hashval_out = hashval;
782+ return 0;
783+}
784+
785+int ptr_to_hashval(const void *ptr, unsigned long *hashval_out)
786+{
787+ return __ptr_to_hashval(ptr, hashval_out);
788+}
789+
764790 static char *ptr_to_id(char *buf, char *end, const void *ptr,
765791 struct printf_spec spec)
766792 {
767793 const char *str = sizeof(ptr) == 8 ? "(____ptrval____)" : "(ptrval)";
768794 unsigned long hashval;
795+ int ret;
769796
770797 /* When debugging early boot use non-cryptographically secure hash. */
771798 if (unlikely(debug_boot_weak_hash)) {
@@ -773,22 +800,13 @@ static char *ptr_to_id(char *buf, char *end, const void *ptr, · 1 unmodified line
773800 return pointer_string(buf, end, (const void *)hashval, spec);
774801 }
775802
776− if (static_branch_unlikely(&not_filled_random_ptr_key)) {
803+ ret = __ptr_to_hashval(ptr, &hashval);
804+ if (ret) {
777805 spec.field_width = 2 * sizeof(ptr);
778806 /* string length must be less than default_width */
779807 return error_string(buf, end, str, spec);
780808 }
781809
782−#ifdef CONFIG_64BIT
783− hashval = (unsigned long)siphash_1u64((u64)ptr, &ptr_key);
784− /*
785− * Mask off the first 32 bits, this makes explicit that we have
786− * modified the address (and 32 bits is plenty for a unique ID).
787− */
788− hashval = hashval & 0xffffffff;
789−#else
790− hashval = (unsigned long)siphash_1u32((u32)ptr, &ptr_key);
791−#endif
792810 return pointer_string(buf, end, (const void *)hashval, spec);
793811 }
794812
mm/Kconfig+20−20
@@ -29,7 +29,7 @@ config FLATMEM_MANUAL · 28 unmodified lines
2929
3030 For systems that have holes in their physical address
3131 spaces and for features like NUMA and memory hotplug,
32− choose "Sparse Memory"
32+ choose "Sparse Memory".
3333
3434 If unsure, choose this option (Flat Memory) over any other.
3535
@@ -122,9 +122,9 @@ config SPARSEMEM_VMEMMAP · 86 unmodified lines
122122 depends on SPARSEMEM && SPARSEMEM_VMEMMAP_ENABLE
123123 default y
124124 help
125− SPARSEMEM_VMEMMAP uses a virtually mapped memmap to optimise
126− pfn_to_page and page_to_pfn operations. This is the most
127− efficient option when sufficient kernel resources are available.
125+ SPARSEMEM_VMEMMAP uses a virtually mapped memmap to optimise
126+ pfn_to_page and page_to_pfn operations. This is the most
127+ efficient option when sufficient kernel resources are available.
128128
129129 config HAVE_MEMBLOCK_NODE_MAP
130130 bool
@@ -160,9 +160,9 @@ config MEMORY_HOTPLUG_SPARSE · 29 unmodified lines
160160 depends on SPARSEMEM && MEMORY_HOTPLUG
161161
162162 config MEMORY_HOTPLUG_DEFAULT_ONLINE
163− bool "Online the newly added memory blocks by default"
164− depends on MEMORY_HOTPLUG
165− help
163+ bool "Online the newly added memory blocks by default"
164+ depends on MEMORY_HOTPLUG
165+ help
166166 This option sets the default policy setting for memory hotplug
167167 onlining policy (/sys/devices/system/memory/auto_online_blocks) which
168168 determines what happens to newly added memory regions. Policy setting
@@ -227,14 +227,14 @@ config COMPACTION · 58 unmodified lines
227227 select MIGRATION
228228 depends on MMU
229229 help
230− Compaction is the only memory management component to form
231− high order (larger physically contiguous) memory blocks
232− reliably. The page allocator relies on compaction heavily and
233− the lack of the feature can lead to unexpected OOM killer
234− invocations for high order memory requests. You shouldn't
235− disable this option unless there really is a strong reason for
236− it and then we would be really interested to hear about that at
237− linux-mm@kvack.org.
230+ Compaction is the only memory management component to form
231+ high order (larger physically contiguous) memory blocks
232+ reliably. The page allocator relies on compaction heavily and
233+ the lack of the feature can lead to unexpected OOM killer
234+ invocations for high order memory requests. You shouldn't
235+ disable this option unless there really is a strong reason for
236+ it and then we would be really interested to hear about that at
237+ linux-mm@kvack.org.
238238
239239 #
240240 # support for page migration
@@ -258,7 +258,7 @@ config ARCH_ENABLE_THP_MIGRATION · 17 unmodified lines
258258 bool
259259
260260 config CONTIG_ALLOC
261− def_bool (MEMORY_ISOLATION && COMPACTION) || CMA
261+ def_bool (MEMORY_ISOLATION && COMPACTION) || CMA
262262
263263 config PHYS_ADDR_T_64BIT
264264 def_bool 64BIT
@@ -302,10 +302,10 @@ config KSM · 37 unmodified lines
302302 root has set /sys/kernel/mm/ksm/run to 1 (if CONFIG_SYSFS is set).
303303
304304 config DEFAULT_MMAP_MIN_ADDR
305− int "Low address space to protect from user allocation"
305+ int "Low address space to protect from user allocation"
306306 depends on MMU
307− default 4096
308− help
307+ default 4096
308+ help
309309 This is the portion of low virtual memory which should be protected
310310 from userspace allocation. Keeping a user from writing to low pages
311311 can help reduce the impact of kernel NULL pointer bugs.
@@ -408,7 +408,7 @@ choice · 96 unmodified lines
408408 endchoice
409409
410410 config ARCH_WANTS_THP_SWAP
411− def_bool n
411+ def_bool n
412412
413413 config THP_SWAP
414414 def_bool y
mm/cma.c+2−4
@@ -95,13 +95,11 @@ static void cma_clear_bitmap(struct cma *cma, unsigned long pfn, · 94 unmodified lines
9595
9696 static int __init cma_activate_area(struct cma *cma)
9797 {
98− int bitmap_size = BITS_TO_LONGS(cma_bitmap_maxno(cma)) * sizeof(long);
9998 unsigned long base_pfn = cma->base_pfn, pfn = base_pfn;
10099 unsigned i = cma->count >> pageblock_order;
101100 struct zone *zone;
102101
103− cma->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
104−
102+ cma->bitmap = bitmap_zalloc(cma_bitmap_maxno(cma), GFP_KERNEL);
105103 if (!cma->bitmap) {
106104 cma->count = 0;
107105 return -ENOMEM;
@@ -139,7 +137,7 @@ static int __init cma_activate_area(struct cma *cma) · 31 unmodified lines
139137
140138 not_in_zone:
141139 pr_err("CMA area %s could not be activated\n", cma->name);
142− kfree(cma->bitmap);
140+ bitmap_free(cma->bitmap);
143141 cma->count = 0;
144142 return -EINVAL;
145143 }
mm/cma_debug.c+5−5
@@ -29,7 +29,7 @@ static int cma_debugfs_get(void *data, u64 *val) · 28 unmodified lines
2929
3030 return 0;
3131 }
32−DEFINE_SIMPLE_ATTRIBUTE(cma_debugfs_fops, cma_debugfs_get, NULL, "%llu\n");
32+DEFINE_DEBUGFS_ATTRIBUTE(cma_debugfs_fops, cma_debugfs_get, NULL, "%llu\n");
3333
3434 static int cma_used_get(void *data, u64 *val)
3535 {
@@ -44,7 +44,7 @@ static int cma_used_get(void *data, u64 *val) · 8 unmodified lines
4444
4545 return 0;
4646 }
47−DEFINE_SIMPLE_ATTRIBUTE(cma_used_fops, cma_used_get, NULL, "%llu\n");
47+DEFINE_DEBUGFS_ATTRIBUTE(cma_used_fops, cma_used_get, NULL, "%llu\n");
4848
4949 static int cma_maxchunk_get(void *data, u64 *val)
5050 {
@@ -66,7 +66,7 @@ static int cma_maxchunk_get(void *data, u64 *val) · 15 unmodified lines
6666
6767 return 0;
6868 }
69−DEFINE_SIMPLE_ATTRIBUTE(cma_maxchunk_fops, cma_maxchunk_get, NULL, "%llu\n");
69+DEFINE_DEBUGFS_ATTRIBUTE(cma_maxchunk_fops, cma_maxchunk_get, NULL, "%llu\n");
7070
7171 static void cma_add_to_cma_mem_list(struct cma *cma, struct cma_mem *mem)
7272 {
@@ -126,7 +126,7 @@ static int cma_free_write(void *data, u64 val) · 53 unmodified lines
126126
127127 return cma_free_mem(cma, pages);
128128 }
129−DEFINE_SIMPLE_ATTRIBUTE(cma_free_fops, NULL, cma_free_write, "%llu\n");
129+DEFINE_DEBUGFS_ATTRIBUTE(cma_free_fops, NULL, cma_free_write, "%llu\n");
130130
131131 static int cma_alloc_mem(struct cma *cma, int count)
132132 {
@@ -158,7 +158,7 @@ static int cma_alloc_write(void *data, u64 val) · 25 unmodified lines
158158
159159 return cma_alloc_mem(cma, pages);
160160 }
161−DEFINE_SIMPLE_ATTRIBUTE(cma_alloc_fops, NULL, cma_alloc_write, "%llu\n");
161+DEFINE_DEBUGFS_ATTRIBUTE(cma_alloc_fops, NULL, cma_alloc_write, "%llu\n");
162162
163163 static void cma_debugfs_add_one(struct cma *cma, struct dentry *root_dentry)
164164 {
mm/filemap.c+29−25
@@ -2329,27 +2329,6 @@ EXPORT_SYMBOL(generic_file_read_iter); · 2328 unmodified lines
23292329
23302330 #ifdef CONFIG_MMU
23312331 #define MMAP_LOTSAMISS (100)
2332−static struct file *maybe_unlock_mmap_for_io(struct vm_fault *vmf,
2333− struct file *fpin)
2334−{
2335− int flags = vmf->flags;
2336−
2337− if (fpin)
2338− return fpin;
2339−
2340− /*
2341− * FAULT_FLAG_RETRY_NOWAIT means we don't want to wait on page locks or
2342− * anything, so we only pin the file and drop the mmap_sem if only
2343− * FAULT_FLAG_ALLOW_RETRY is set.
2344− */
2345− if ((flags & (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT)) ==
2346− FAULT_FLAG_ALLOW_RETRY) {
2347− fpin = get_file(vmf->vma->vm_file);
2348− up_read(&vmf->vma->vm_mm->mmap_sem);
2349− }
2350− return fpin;
2351−}
2352−
23532332 /*
23542333 * lock_page_maybe_drop_mmap - lock the page, possibly dropping the mmap_sem
23552334 * @vmf - the vm_fault for this fault.
@@ -3161,6 +3140,27 @@ int pagecache_write_end(struct file *file, struct address_space *mapping, · 805 unmodified lines
31613140 }
31623141 EXPORT_SYMBOL(pagecache_write_end);
31633142
3143+/*
3144+ * Warn about a page cache invalidation failure during a direct I/O write.
3145+ */
3146+void dio_warn_stale_pagecache(struct file *filp)
3147+{
3148+ static DEFINE_RATELIMIT_STATE(_rs, 86400 * HZ, DEFAULT_RATELIMIT_BURST);
3149+ char pathname[128];
3150+ struct inode *inode = file_inode(filp);
3151+ char *path;
3152+
3153+ errseq_set(&inode->i_mapping->wb_err, -EIO);
3154+ if (__ratelimit(&_rs)) {
3155+ path = file_path(filp, pathname, sizeof(pathname));
3156+ if (IS_ERR(path))
3157+ path = "(unknown)";
3158+ pr_crit("Page cache invalidation failure on direct I/O. Possible data corruption due to collision with buffered I/O!\n");
3159+ pr_crit("File: %s PID: %d Comm: %.20s\n", path, current->pid,
3160+ current->comm);
3161+ }
3162+}
3163+
31643164 ssize_t
31653165 generic_file_direct_write(struct kiocb *iocb, struct iov_iter *from)
31663166 {
@@ -3218,11 +3218,15 @@ generic_file_direct_write(struct kiocb *iocb, struct iov_iter *from) · 51 unmodified lines
32183218 * Most of the time we do not need this since dio_complete() will do
32193219 * the invalidation for us. However there are some file systems that
32203220 * do not end up with dio_complete() being called, so let's not break
3221− * them by removing it completely
3221+ * them by removing it completely.
3222+ *
3223+ * Noticeable example is a blkdev_direct_IO().
3224+ *
3225+ * Skip invalidation for async writes or if mapping has no pages.
32223226 */
3223− if (mapping->nrpages)
3224− invalidate_inode_pages2_range(mapping,
3225− pos >> PAGE_SHIFT, end);
3227+ if (written > 0 && mapping->nrpages &&
3228+ invalidate_inode_pages2_range(mapping, pos >> PAGE_SHIFT, end))
3229+ dio_warn_stale_pagecache(file);
32263230
32273231 if (written > 0) {
32283232 pos += written;
mm/gup.c+27−13
@@ -734,11 +734,17 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags) · 733 unmodified lines
734734 * Or NULL if the caller does not require them.
735735 * @nonblocking: whether waiting for disk IO or mmap_sem contention
736736 *
737− * Returns number of pages pinned. This may be fewer than the number
738− * requested. If nr_pages is 0 or negative, returns 0. If no pages
739− * were pinned, returns -errno. Each page returned must be released
740− * with a put_page() call when it is finished with. vmas will only
741− * remain valid while mmap_sem is held.
737+ * Returns either number of pages pinned (which may be less than the
738+ * number requested), or an error. Details about the return value:
739+ *
740+ * -- If nr_pages is 0, returns 0.
741+ * -- If nr_pages is >0, but no pages were pinned, returns -errno.
742+ * -- If nr_pages is >0, and some pages were pinned, returns the number of
743+ * pages pinned. Again, this may be less than nr_pages.
744+ *
745+ * The caller is responsible for releasing returned @pages, via put_page().
746+ *
747+ * @vmas are valid only as long as mmap_sem is held.
742748 *
743749 * Must be called with mmap_sem held. It may be released. See below.
744750 *
@@ -1107,11 +1113,17 @@ static __always_inline long __get_user_pages_locked(struct task_struct *tsk, · 362 unmodified lines
11071113 * subsequently whether VM_FAULT_RETRY functionality can be
11081114 * utilised. Lock must initially be held.
11091115 *
1110− * Returns number of pages pinned. This may be fewer than the number
1111− * requested. If nr_pages is 0 or negative, returns 0. If no pages
1112− * were pinned, returns -errno. Each page returned must be released
1113− * with a put_page() call when it is finished with. vmas will only
1114− * remain valid while mmap_sem is held.
1116+ * Returns either number of pages pinned (which may be less than the
1117+ * number requested), or an error. Details about the return value:
1118+ *
1119+ * -- If nr_pages is 0, returns 0.
1120+ * -- If nr_pages is >0, but no pages were pinned, returns -errno.
1121+ * -- If nr_pages is >0, and some pages were pinned, returns the number of
1122+ * pages pinned. Again, this may be less than nr_pages.
1123+ *
1124+ * The caller is responsible for releasing returned @pages, via put_page().
1125+ *
1126+ * @vmas are valid only as long as mmap_sem is held.
11151127 *
11161128 * Must be called with mmap_sem held for read or write.
11171129 *
@@ -1443,6 +1455,7 @@ static long check_and_migrate_cma_pages(struct task_struct *tsk, · 325 unmodified lines
14431455 bool drain_allow = true;
14441456 bool migrate_allow = true;
14451457 LIST_HEAD(cma_page_list);
1458+ long ret = nr_pages;
14461459
14471460 check_again:
14481461 for (i = 0; i < nr_pages;) {
@@ -1504,17 +1517,18 @@ check_again: · 55 unmodified lines
15041517 * again migrating any new CMA pages which we failed to isolate
15051518 * earlier.
15061519 */
1507− nr_pages = __get_user_pages_locked(tsk, mm, start, nr_pages,
1520+ ret = __get_user_pages_locked(tsk, mm, start, nr_pages,
15081521 pages, vmas, NULL,
15091522 gup_flags);
15101523
1511− if ((nr_pages > 0) && migrate_allow) {
1524+ if ((ret > 0) && migrate_allow) {
1525+ nr_pages = ret;
15121526 drain_allow = true;
15131527 goto check_again;
15141528 }
15151529 }
15161530
1517− return nr_pages;
1531+ return ret;
15181532 }
15191533 #else
15201534 static long check_and_migrate_cma_pages(struct task_struct *tsk,
mm/huge_memory.c+1−1
@@ -3003,7 +3003,7 @@ next: · 3002 unmodified lines
30033003
30043004 return 0;
30053005 }
3006−DEFINE_SIMPLE_ATTRIBUTE(split_huge_pages_fops, NULL, split_huge_pages_set,
3006+DEFINE_DEBUGFS_ATTRIBUTE(split_huge_pages_fops, NULL, split_huge_pages_set,
30073007 "%llu\n");
30083008
30093009 static int __init split_huge_pages_debugfs(void)
mm/hugetlb.c+91−197
@@ -244,16 +244,66 @@ struct file_region { · 243 unmodified lines
244244 long to;
245245 };
246246
247+/* Must be called with resv->lock held. Calling this with count_only == true
248+ * will count the number of pages to be added but will not modify the linked
249+ * list.
250+ */
251+static long add_reservation_in_range(struct resv_map *resv, long f, long t,
252+ bool count_only)
253+{
254+ long chg = 0;
255+ struct list_head *head = &resv->regions;
256+ struct file_region *rg = NULL, *trg = NULL, *nrg = NULL;
257+
258+ /* Locate the region we are before or in. */
259+ list_for_each_entry(rg, head, link)
260+ if (f <= rg->to)
261+ break;
262+
263+ /* Round our left edge to the current segment if it encloses us. */
264+ if (f > rg->from)
265+ f = rg->from;
266+
267+ chg = t - f;
268+
269+ /* Check for and consume any regions we now overlap with. */
270+ nrg = rg;
271+ list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
272+ if (&rg->link == head)
273+ break;
274+ if (rg->from > t)
275+ break;
276+
277+ /* We overlap with this area, if it extends further than
278+ * us then we must extend ourselves. Account for its
279+ * existing reservation.
280+ */
281+ if (rg->to > t) {
282+ chg += rg->to - t;
283+ t = rg->to;
284+ }
285+ chg -= rg->to - rg->from;
286+
287+ if (!count_only && rg != nrg) {
288+ list_del(&rg->link);
289+ kfree(rg);
290+ }
291+ }
292+
293+ if (!count_only) {
294+ nrg->from = f;
295+ nrg->to = t;
296+ }
297+
298+ return chg;
299+}
300+
247301 /*
248302 * Add the huge page range represented by [f, t) to the reserve
249− * map. In the normal case, existing regions will be expanded
250− * to accommodate the specified range. Sufficient regions should
251− * exist for expansion due to the previous call to region_chg
252− * with the same range. However, it is possible that region_del
253− * could have been called after region_chg and modifed the map
254− * in such a way that no region exists to be expanded. In this
255− * case, pull a region descriptor from the cache associated with
256− * the map and use that for the new range.
303+ * map. Existing regions will be expanded to accommodate the specified
304+ * range, or a region will be taken from the cache. Sufficient regions
305+ * must exist in the cache due to the previous call to region_chg with
306+ * the same range.
257307 *
258308 * Return the number of new huge pages added to the map. This
259309 * number is greater than or equal to zero.
@@ -261,7 +311,7 @@ struct file_region { · 1 unmodified line
261311 static long region_add(struct resv_map *resv, long f, long t)
262312 {
263313 struct list_head *head = &resv->regions;
264− struct file_region *rg, *nrg, *trg;
314+ struct file_region *rg, *nrg;
265315 long add = 0;
266316
267317 spin_lock(&resv->lock);
@@ -272,9 +322,8 @@ static long region_add(struct resv_map *resv, long f, long t) · 4 unmodified lines
272322
273323 /*
274324 * If no region exists which can be expanded to include the
275− * specified range, the list must have been modified by an
276− * interleving call to region_del(). Pull a region descriptor
277− * from the cache and use it for this range.
325+ * specified range, pull a region descriptor from the cache
326+ * and use it for this range.
278327 */
279328 if (&rg->link == head || t < rg->from) {
280329 VM_BUG_ON(resv->region_cache_count <= 0);
@@ -292,38 +341,7 @@ static long region_add(struct resv_map *resv, long f, long t) · 11 unmodified lines
292341 goto out_locked;
293342 }
294343
295− /* Round our left edge to the current segment if it encloses us. */
296− if (f > rg->from)
297− f = rg->from;
298−
299− /* Check for and consume any regions we now overlap with. */
300− nrg = rg;
301− list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
302− if (&rg->link == head)
303− break;
304− if (rg->from > t)
305− break;
306−
307− /* If this area reaches higher then extend our area to
308− * include it completely. If this is not the first area
309− * which we intend to reuse, free it. */
310− if (rg->to > t)
311− t = rg->to;
312− if (rg != nrg) {
313− /* Decrement return value by the deleted range.
314− * Another range will span this area so that by
315− * end of routine add will be >= zero
316− */
317− add -= (rg->to - rg->from);
318− list_del(&rg->link);
319− kfree(rg);
320− }
321− }
322−
323− add += (nrg->from - f); /* Added to beginning of region */
324− nrg->from = f;
325− add += t - nrg->to; /* Added to end of region */
326− nrg->to = t;
344+ add = add_reservation_in_range(resv, f, t, false);
327345
328346 out_locked:
329347 resv->adds_in_progress--;
@@ -339,15 +357,9 @@ out_locked: · 9 unmodified lines
339357 * call to region_add that will actually modify the reserve
340358 * map to add the specified range [f, t). region_chg does
341359 * not change the number of huge pages represented by the
342− * map. However, if the existing regions in the map can not
343− * be expanded to represent the new range, a new file_region
344− * structure is added to the map as a placeholder. This is
345− * so that the subsequent region_add call will have all the
346− * regions it needs and will not fail.
347− *
348− * Upon entry, region_chg will also examine the cache of region descriptors
349− * associated with the map. If there are not enough descriptors cached, one
350− * will be allocated for the in progress add operation.
360+ * map. A new file_region structure is added to the cache
361+ * as a placeholder, so that the subsequent region_add
362+ * call will have all the regions it needs and will not fail.
351363 *
352364 * Returns the number of huge pages that need to be added to the existing
353365 * reservation map for the range [f, t). This number is greater or equal to
@@ -356,11 +368,8 @@ out_locked: · 2 unmodified lines
356368 */
357369 static long region_chg(struct resv_map *resv, long f, long t)
358370 {
359− struct list_head *head = &resv->regions;
360− struct file_region *rg, *nrg = NULL;
361371 long chg = 0;
362372
363−retry:
364373 spin_lock(&resv->lock);
365374 retry_locked:
366375 resv->adds_in_progress++;
@@ -378,10 +387,8 @@ retry_locked: · 11 unmodified lines
378387 spin_unlock(&resv->lock);
379388
380389 trg = kmalloc(sizeof(*trg), GFP_KERNEL);
381− if (!trg) {
382− kfree(nrg);
390+ if (!trg)
383391 return -ENOMEM;
384− }
385392
386393 spin_lock(&resv->lock);
387394 list_add(&trg->link, &resv->region_cache);
@@ -389,61 +396,8 @@ retry_locked: · 1 unmodified line
389396 goto retry_locked;
390397 }
391398
392− /* Locate the region we are before or in. */
393− list_for_each_entry(rg, head, link)
394− if (f <= rg->to)
395− break;
399+ chg = add_reservation_in_range(resv, f, t, true);
396400
397− /* If we are below the current region then a new region is required.
398− * Subtle, allocate a new region at the position but make it zero
399− * size such that we can guarantee to record the reservation. */
400− if (&rg->link == head || t < rg->from) {
401− if (!nrg) {
402− resv->adds_in_progress--;
403− spin_unlock(&resv->lock);
404− nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
405− if (!nrg)
406− return -ENOMEM;
407−
408− nrg->from = f;
409− nrg->to = f;
410− INIT_LIST_HEAD(&nrg->link);
411− goto retry;
412− }
413−
414− list_add(&nrg->link, rg->link.prev);
415− chg = t - f;
416− goto out_nrg;
417− }
418−
419− /* Round our left edge to the current segment if it encloses us. */
420− if (f > rg->from)
421− f = rg->from;
422− chg = t - f;
423−
424− /* Check for and consume any regions we now overlap with. */
425− list_for_each_entry(rg, rg->link.prev, link) {
426− if (&rg->link == head)
427− break;
428− if (rg->from > t)
429− goto out;
430−
431− /* We overlap with this area, if it extends further than
432− * us then we must extend ourselves. Account for its
433− * existing reservation. */
434− if (rg->to > t) {
435− chg += rg->to - t;
436− t = rg->to;
437− }
438− chg -= rg->to - rg->from;
439− }
440−
441−out:
442− spin_unlock(&resv->lock);
443− /* We already know we raced and no longer need the new region */
444− kfree(nrg);
445− return chg;
446−out_nrg:
447401 spin_unlock(&resv->lock);
448402 return chg;
449403 }
@@ -1069,85 +1023,12 @@ static void free_gigantic_page(struct page *page, unsigned int order) · 619 unmodified lines
10691023 }
10701024
10711025 #ifdef CONFIG_CONTIG_ALLOC
1072−static int __alloc_gigantic_page(unsigned long start_pfn,
1073− unsigned long nr_pages, gfp_t gfp_mask)
1074−{
1075− unsigned long end_pfn = start_pfn + nr_pages;
1076− return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE,
1077− gfp_mask);
1078−}
1079−
1080−static bool pfn_range_valid_gigantic(struct zone *z,
1081− unsigned long start_pfn, unsigned long nr_pages)
1082−{
1083− unsigned long i, end_pfn = start_pfn + nr_pages;
1084− struct page *page;
1085−
1086− for (i = start_pfn; i < end_pfn; i++) {
1087− page = pfn_to_online_page(i);
1088− if (!page)
1089− return false;
1090−
1091− if (page_zone(page) != z)
1092− return false;
1093−
1094− if (PageReserved(page))
1095− return false;
1096−
1097− if (page_count(page) > 0)
1098− return false;
1099−
1100− if (PageHuge(page))
1101− return false;
1102− }
1103−
1104− return true;
1105−}
1106−
1107−static bool zone_spans_last_pfn(const struct zone *zone,
1108− unsigned long start_pfn, unsigned long nr_pages)
1109−{
1110− unsigned long last_pfn = start_pfn + nr_pages - 1;
1111− return zone_spans_pfn(zone, last_pfn);
1112−}
1113−
11141026 static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
11151027 int nid, nodemask_t *nodemask)
11161028 {
1117− unsigned int order = huge_page_order(h);
1118− unsigned long nr_pages = 1 << order;
1119− unsigned long ret, pfn, flags;
1120− struct zonelist *zonelist;
1121− struct zone *zone;
1122− struct zoneref *z;
1029+ unsigned long nr_pages = 1UL << huge_page_order(h);
11231030
1124− zonelist = node_zonelist(nid, gfp_mask);
1125− for_each_zone_zonelist_nodemask(zone, z, zonelist, gfp_zone(gfp_mask), nodemask) {
1126− spin_lock_irqsave(&zone->lock, flags);
1127−
1128− pfn = ALIGN(zone->zone_start_pfn, nr_pages);
1129− while (zone_spans_last_pfn(zone, pfn, nr_pages)) {
1130− if (pfn_range_valid_gigantic(zone, pfn, nr_pages)) {
1131− /*
1132− * We release the zone lock here because
1133− * alloc_contig_range() will also lock the zone
1134− * at some point. If there's an allocation
1135− * spinning on this lock, it may win the race
1136− * and cause alloc_contig_range() to fail...
1137− */
1138− spin_unlock_irqrestore(&zone->lock, flags);
1139− ret = __alloc_gigantic_page(pfn, nr_pages, gfp_mask);
1140− if (!ret)
1141− return pfn_to_page(pfn);
1142− spin_lock_irqsave(&zone->lock, flags);
1143− }
1144− pfn += nr_pages;
1145− }
1146−
1147− spin_unlock_irqrestore(&zone->lock, flags);
1148− }
1149−
1150− return NULL;
1031+ return alloc_contig_pages(nr_pages, gfp_mask, nid, nodemask);
11511032 }
11521033
11531034 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid);
@@ -3915,7 +3796,7 @@ retry: · 2761 unmodified lines
39153796 * handling userfault. Reacquire after handling
39163797 * fault to make calling code simpler.
39173798 */
3918− hash = hugetlb_fault_mutex_hash(h, mapping, idx, haddr);
3799+ hash = hugetlb_fault_mutex_hash(mapping, idx);
39193800 mutex_unlock(&hugetlb_fault_mutex_table[hash]);
39203801 ret = handle_userfault(&vmf, VM_UFFD_MISSING);
39213802 mutex_lock(&hugetlb_fault_mutex_table[hash]);
@@ -4042,8 +3923,7 @@ backout_unlocked: · 120 unmodified lines
40423923 }
40433924
40443925 #ifdef CONFIG_SMP
4045−u32 hugetlb_fault_mutex_hash(struct hstate *h, struct address_space *mapping,
4046− pgoff_t idx, unsigned long address)
3926+u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
40473927 {
40483928 unsigned long key[2];
40493929 u32 hash;
@@ -4051,7 +3931,7 @@ u32 hugetlb_fault_mutex_hash(struct hstate *h, struct address_space *mapping, · 1 unmodified line
40513931 key[0] = (unsigned long) mapping;
40523932 key[1] = idx;
40533933
4054− hash = jhash2((u32 *)&key, sizeof(key)/sizeof(u32), 0);
3934+ hash = jhash2((u32 *)&key, sizeof(key)/(sizeof(u32)), 0);
40553935
40563936 return hash & (num_fault_mutexes - 1);
40573937 }
@@ -4060,8 +3940,7 @@ u32 hugetlb_fault_mutex_hash(struct hstate *h, struct address_space *mapping, · 2 unmodified lines
40603940 * For uniprocesor systems we always use a single mutex, so just
40613941 * return 0 and avoid the hashing overhead.
40623942 */
4063−u32 hugetlb_fault_mutex_hash(struct hstate *h, struct address_space *mapping,
4064− pgoff_t idx, unsigned long address)
3943+u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
40653944 {
40663945 return 0;
40673946 }
@@ -4105,7 +3984,7 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, · 37 unmodified lines
41053984 * get spurious allocation failures if two CPUs race to instantiate
41063985 * the same page in the page cache.
41073986 */
4108− hash = hugetlb_fault_mutex_hash(h, mapping, idx, haddr);
3987+ hash = hugetlb_fault_mutex_hash(mapping, idx);
41093988 mutex_lock(&hugetlb_fault_mutex_table[hash]);
41103989
41113990 entry = huge_ptep_get(ptep);
@@ -4459,6 +4338,21 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, · 347 unmodified lines
44594338 break;
44604339 }
44614340 }
4341+
4342+ /*
4343+ * If subpage information not requested, update counters
4344+ * and skip the same_page loop below.
4345+ */
4346+ if (!pages && !vmas && !pfn_offset &&
4347+ (vaddr + huge_page_size(h) < vma->vm_end) &&
4348+ (remainder >= pages_per_huge_page(h))) {
4349+ vaddr += huge_page_size(h);
4350+ remainder -= pages_per_huge_page(h);
4351+ i += pages_per_huge_page(h);
4352+ spin_unlock(ptl);
4353+ continue;
4354+ }
4355+
44624356 same_page:
44634357 if (pages) {
44644358 pages[i] = mem_map_offset(page, pfn_offset);
@@ -4842,7 +4736,7 @@ pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud) · 377 unmodified lines
48424736 if (!vma_shareable(vma, addr))
48434737 return (pte_t *)pmd_alloc(mm, pud, addr);
48444738
4845− i_mmap_lock_write(mapping);
4739+ i_mmap_lock_read(mapping);
48464740 vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
48474741 if (svma == vma)
48484742 continue;
@@ -4872,7 +4766,7 @@ pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud) · 23 unmodified lines
48724766 spin_unlock(ptl);
48734767 out:
48744768 pte = (pte_t *)pmd_alloc(mm, pud, addr);
4875− i_mmap_unlock_write(mapping);
4769+ i_mmap_unlock_read(mapping);
48764770 return pte;
48774771 }
48784772
mm/hwpoison-inject.c+2−2
@@ -67,8 +67,8 @@ static int hwpoison_unpoison(void *data, u64 val) · 66 unmodified lines
6767 return unpoison_memory(val);
6868 }
6969
70−DEFINE_SIMPLE_ATTRIBUTE(hwpoison_fops, NULL, hwpoison_inject, "%lli\n");
71−DEFINE_SIMPLE_ATTRIBUTE(unpoison_fops, NULL, hwpoison_unpoison, "%lli\n");
70+DEFINE_DEBUGFS_ATTRIBUTE(hwpoison_fops, NULL, hwpoison_inject, "%lli\n");
71+DEFINE_DEBUGFS_ATTRIBUTE(unpoison_fops, NULL, hwpoison_unpoison, "%lli\n");
7272
7373 static void pfn_inject_exit(void)
7474 {
mm/internal.h+26−1
@@ -165,6 +165,9 @@ extern void post_alloc_hook(struct page *page, unsigned int order, · 164 unmodified lines
165165 gfp_t gfp_flags);
166166 extern int user_min_free_kbytes;
167167
168+extern void zone_pcp_update(struct zone *zone);
169+extern void zone_pcp_reset(struct zone *zone);
170+
168171 #if defined CONFIG_COMPACTION || defined CONFIG_CMA
169172
170173 /*
@@ -290,7 +293,8 @@ static inline bool is_data_mapping(vm_flags_t flags) · 119 unmodified lines
290293
291294 /* mm/util.c */
292295 void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
293− struct vm_area_struct *prev, struct rb_node *rb_parent);
296+ struct vm_area_struct *prev);
297+void __vma_unlink_list(struct mm_struct *mm, struct vm_area_struct *vma);
294298
295299 #ifdef CONFIG_MMU
296300 extern long populate_vma_page_range(struct vm_area_struct *vma,
@@ -362,6 +366,27 @@ vma_address(struct page *page, struct vm_area_struct *vma) · 65 unmodified lines
362366 return max(start, vma->vm_start);
363367 }
364368
369+static inline struct file *maybe_unlock_mmap_for_io(struct vm_fault *vmf,
370+ struct file *fpin)
371+{
372+ int flags = vmf->flags;
373+
374+ if (fpin)
375+ return fpin;
376+
377+ /*
378+ * FAULT_FLAG_RETRY_NOWAIT means we don't want to wait on page locks or
379+ * anything, so we only pin the file and drop the mmap_sem if only
380+ * FAULT_FLAG_ALLOW_RETRY is set.
381+ */
382+ if ((flags & (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT)) ==
383+ FAULT_FLAG_ALLOW_RETRY) {
384+ fpin = get_file(vmf->vma->vm_file);
385+ up_read(&vmf->vma->vm_mm->mmap_sem);
386+ }
387+ return fpin;
388+}
389+
365390 #else /* !CONFIG_MMU */
366391 static inline void clear_page_mlock(struct page *page) { }
367392 static inline void mlock_vma_page(struct page *page) { }
mm/kasan/common.c+233−0
@@ -36,6 +36,8 @@ · 35 unmodified lines
3636 #include <linux/bug.h>
3737 #include <linux/uaccess.h>
3838
39+#include <asm/tlbflush.h>
40+
3941 #include "kasan.h"
4042 #include "../slab.h"
4143
@@ -590,6 +592,7 @@ void kasan_kfree_large(void *ptr, unsigned long ip) · 548 unmodified lines
590592 /* The object will be poisoned by page_alloc. */
591593 }
592594
595+#ifndef CONFIG_KASAN_VMALLOC
593596 int kasan_module_alloc(void *addr, size_t size)
594597 {
595598 void *ret;
@@ -625,6 +628,7 @@ void kasan_free_shadow(const struct vm_struct *vm) · 29 unmodified lines
625628 if (vm->flags & VM_KASAN)
626629 vfree(kasan_mem_to_shadow(vm->addr));
627630 }
631+#endif
628632
629633 extern void __kasan_report(unsigned long addr, size_t size, bool is_write, unsigned long ip);
630634
@@ -744,3 +748,232 @@ static int __init kasan_memhotplug_init(void) · 113 unmodified lines
744748
745749 core_initcall(kasan_memhotplug_init);
746750 #endif
751+
752+#ifdef CONFIG_KASAN_VMALLOC
753+static int kasan_populate_vmalloc_pte(pte_t *ptep, unsigned long addr,
754+ void *unused)
755+{
756+ unsigned long page;
757+ pte_t pte;
758+
759+ if (likely(!pte_none(*ptep)))
760+ return 0;
761+
762+ page = __get_free_page(GFP_KERNEL);
763+ if (!page)
764+ return -ENOMEM;
765+
766+ memset((void *)page, KASAN_VMALLOC_INVALID, PAGE_SIZE);
767+ pte = pfn_pte(PFN_DOWN(__pa(page)), PAGE_KERNEL);
768+
769+ spin_lock(&init_mm.page_table_lock);
770+ if (likely(pte_none(*ptep))) {
771+ set_pte_at(&init_mm, addr, ptep, pte);
772+ page = 0;
773+ }
774+ spin_unlock(&init_mm.page_table_lock);
775+ if (page)
776+ free_page(page);
777+ return 0;
778+}
779+
780+int kasan_populate_vmalloc(unsigned long requested_size, struct vm_struct *area)
781+{
782+ unsigned long shadow_start, shadow_end;
783+ int ret;
784+
785+ shadow_start = (unsigned long)kasan_mem_to_shadow(area->addr);
786+ shadow_start = ALIGN_DOWN(shadow_start, PAGE_SIZE);
787+ shadow_end = (unsigned long)kasan_mem_to_shadow(area->addr +
788+ area->size);
789+ shadow_end = ALIGN(shadow_end, PAGE_SIZE);
790+
791+ ret = apply_to_page_range(&init_mm, shadow_start,
792+ shadow_end - shadow_start,
793+ kasan_populate_vmalloc_pte, NULL);
794+ if (ret)
795+ return ret;
796+
797+ flush_cache_vmap(shadow_start, shadow_end);
798+
799+ kasan_unpoison_shadow(area->addr, requested_size);
800+
801+ area->flags |= VM_KASAN;
802+
803+ /*
804+ * We need to be careful about inter-cpu effects here. Consider:
805+ *
806+ * CPU#0 CPU#1
807+ * WRITE_ONCE(p, vmalloc(100)); while (x = READ_ONCE(p)) ;
808+ * p[99] = 1;
809+ *
810+ * With compiler instrumentation, that ends up looking like this:
811+ *
812+ * CPU#0 CPU#1
813+ * // vmalloc() allocates memory
814+ * // let a = area->addr
815+ * // we reach kasan_populate_vmalloc
816+ * // and call kasan_unpoison_shadow:
817+ * STORE shadow(a), unpoison_val
818+ * ...
819+ * STORE shadow(a+99), unpoison_val x = LOAD p
820+ * // rest of vmalloc process <data dependency>
821+ * STORE p, a LOAD shadow(x+99)
822+ *
823+ * If there is no barrier between the end of unpoisioning the shadow
824+ * and the store of the result to p, the stores could be committed
825+ * in a different order by CPU#0, and CPU#1 could erroneously observe
826+ * poison in the shadow.
827+ *
828+ * We need some sort of barrier between the stores.
829+ *
830+ * In the vmalloc() case, this is provided by a smp_wmb() in
831+ * clear_vm_uninitialized_flag(). In the per-cpu allocator and in
832+ * get_vm_area() and friends, the caller gets shadow allocated but
833+ * doesn't have any pages mapped into the virtual address space that
834+ * has been reserved. Mapping those pages in will involve taking and
835+ * releasing a page-table lock, which will provide the barrier.
836+ */
837+
838+ return 0;
839+}
840+
841+/*
842+ * Poison the shadow for a vmalloc region. Called as part of the
843+ * freeing process at the time the region is freed.
844+ */
845+void kasan_poison_vmalloc(void *start, unsigned long size)
846+{
847+ size = round_up(size, KASAN_SHADOW_SCALE_SIZE);
848+ kasan_poison_shadow(start, size, KASAN_VMALLOC_INVALID);
849+}
850+
851+static int kasan_depopulate_vmalloc_pte(pte_t *ptep, unsigned long addr,
852+ void *unused)
853+{
854+ unsigned long page;
855+
856+ page = (unsigned long)__va(pte_pfn(*ptep) << PAGE_SHIFT);
857+
858+ spin_lock(&init_mm.page_table_lock);
859+
860+ if (likely(!pte_none(*ptep))) {
861+ pte_clear(&init_mm, addr, ptep);
862+ free_page(page);
863+ }
864+ spin_unlock(&init_mm.page_table_lock);
865+
866+ return 0;
867+}
868+
869+/*
870+ * Release the backing for the vmalloc region [start, end), which
871+ * lies within the free region [free_region_start, free_region_end).
872+ *
873+ * This can be run lazily, long after the region was freed. It runs
874+ * under vmap_area_lock, so it's not safe to interact with the vmalloc/vmap
875+ * infrastructure.
876+ *
877+ * How does this work?
878+ * -------------------
879+ *
880+ * We have a region that is page aligned, labelled as A.
881+ * That might not map onto the shadow in a way that is page-aligned:
882+ *
883+ * start end
884+ * v v
885+ * |????????|????????|AAAAAAAA|AA....AA|AAAAAAAA|????????| < vmalloc
886+ * -------- -------- -------- -------- --------
887+ * | | | | |
888+ * | | | /-------/ |
889+ * \-------\|/------/ |/---------------/
890+ * ||| ||
891+ * |??AAAAAA|AAAAAAAA|AA??????| < shadow
892+ * (1) (2) (3)
893+ *
894+ * First we align the start upwards and the end downwards, so that the
895+ * shadow of the region aligns with shadow page boundaries. In the
896+ * example, this gives us the shadow page (2). This is the shadow entirely
897+ * covered by this allocation.
898+ *
899+ * Then we have the tricky bits. We want to know if we can free the
900+ * partially covered shadow pages - (1) and (3) in the example. For this,
901+ * we are given the start and end of the free region that contains this
902+ * allocation. Extending our previous example, we could have:
903+ *
904+ * free_region_start free_region_end
905+ * | start end |
906+ * v v v v
907+ * |FFFFFFFF|FFFFFFFF|AAAAAAAA|AA....AA|AAAAAAAA|FFFFFFFF| < vmalloc
908+ * -------- -------- -------- -------- --------
909+ * | | | | |
910+ * | | | /-------/ |
911+ * \-------\|/------/ |/---------------/
912+ * ||| ||
913+ * |FFAAAAAA|AAAAAAAA|AAF?????| < shadow
914+ * (1) (2) (3)
915+ *
916+ * Once again, we align the start of the free region up, and the end of
917+ * the free region down so that the shadow is page aligned. So we can free
918+ * page (1) - we know no allocation currently uses anything in that page,
919+ * because all of it is in the vmalloc free region. But we cannot free
920+ * page (3), because we can't be sure that the rest of it is unused.
921+ *
922+ * We only consider pages that contain part of the original region for
923+ * freeing: we don't try to free other pages from the free region or we'd
924+ * end up trying to free huge chunks of virtual address space.
925+ *
926+ * Concurrency
927+ * -----------
928+ *
929+ * How do we know that we're not freeing a page that is simultaneously
930+ * being used for a fresh allocation in kasan_populate_vmalloc(_pte)?
931+ *
932+ * We _can_ have kasan_release_vmalloc and kasan_populate_vmalloc running
933+ * at the same time. While we run under free_vmap_area_lock, the population
934+ * code does not.
935+ *
936+ * free_vmap_area_lock instead operates to ensure that the larger range
937+ * [free_region_start, free_region_end) is safe: because __alloc_vmap_area and
938+ * the per-cpu region-finding algorithm both run under free_vmap_area_lock,
939+ * no space identified as free will become used while we are running. This
940+ * means that so long as we are careful with alignment and only free shadow
941+ * pages entirely covered by the free region, we will not run in to any
942+ * trouble - any simultaneous allocations will be for disjoint regions.
943+ */
944+void kasan_release_vmalloc(unsigned long start, unsigned long end,
945+ unsigned long free_region_start,
946+ unsigned long free_region_end)
947+{
948+ void *shadow_start, *shadow_end;
949+ unsigned long region_start, region_end;
950+
951+ region_start = ALIGN(start, PAGE_SIZE * KASAN_SHADOW_SCALE_SIZE);
952+ region_end = ALIGN_DOWN(end, PAGE_SIZE * KASAN_SHADOW_SCALE_SIZE);
953+
954+ free_region_start = ALIGN(free_region_start,
955+ PAGE_SIZE * KASAN_SHADOW_SCALE_SIZE);
956+
957+ if (start != region_start &&
958+ free_region_start < region_start)
959+ region_start -= PAGE_SIZE * KASAN_SHADOW_SCALE_SIZE;
960+
961+ free_region_end = ALIGN_DOWN(free_region_end,
962+ PAGE_SIZE * KASAN_SHADOW_SCALE_SIZE);
963+
964+ if (end != region_end &&
965+ free_region_end > region_end)
966+ region_end += PAGE_SIZE * KASAN_SHADOW_SCALE_SIZE;
967+
968+ shadow_start = kasan_mem_to_shadow((void *)region_start);
969+ shadow_end = kasan_mem_to_shadow((void *)region_end);
970+
971+ if (shadow_end > shadow_start) {
972+ apply_to_page_range(&init_mm, (unsigned long)shadow_start,
973+ (unsigned long)(shadow_end - shadow_start),
974+ kasan_depopulate_vmalloc_pte, NULL);
975+ flush_tlb_kernel_range((unsigned long)shadow_start,
976+ (unsigned long)shadow_end);
977+ }
978+}
979+#endif
mm/kasan/generic_report.c+3−0
@@ -86,6 +86,9 @@ static const char *get_shadow_bug_type(struct kasan_access_info *info) · 85 unmodified lines
8686 case KASAN_ALLOCA_RIGHT:
8787 bug_type = "alloca-out-of-bounds";
8888 break;
89+ case KASAN_VMALLOC_INVALID:
90+ bug_type = "vmalloc-out-of-bounds";
91+ break;
8992 }
9093
9194 return bug_type;
mm/kasan/kasan.h+1−0
@@ -25,6 +25,7 @@ · 24 unmodified lines
2525 #endif
2626
2727 #define KASAN_GLOBAL_REDZONE 0xFA /* redzone for global variable */
28+#define KASAN_VMALLOC_INVALID 0xF9 /* unallocated space in vmapped page */
2829
2930 /*
3031 * Stack redzone shadow values
mm/khugepaged.c+18−0
@@ -1602,6 +1602,24 @@ static void collapse_file(struct mm_struct *mm, · 1601 unmodified lines
16021602 result = SCAN_FAIL;
16031603 goto xa_unlocked;
16041604 }
1605+ } else if (PageDirty(page)) {
1606+ /*
1607+ * khugepaged only works on read-only fd,
1608+ * so this page is dirty because it hasn't
1609+ * been flushed since first write. There
1610+ * won't be new dirty pages.
1611+ *
1612+ * Trigger async flush here and hope the
1613+ * writeback is done when khugepaged
1614+ * revisits this page.
1615+ *
1616+ * This is a one-off situation. We are not
1617+ * forcing writeback in loop.
1618+ */
1619+ xas_unlock_irq(&xas);
1620+ filemap_flush(mapping);
1621+ result = SCAN_FAIL;
1622+ goto xa_unlocked;
16051623 } else if (trylock_page(page)) {
16061624 get_page(page);
16071625 xas_unlock_irq(&xas);
mm/madvise.c+7−7
@@ -864,13 +864,13 @@ static int madvise_inject_error(int behavior, · 863 unmodified lines
864864 {
865865 struct page *page;
866866 struct zone *zone;
867− unsigned int order;
867+ unsigned long size;
868868
869869 if (!capable(CAP_SYS_ADMIN))
870870 return -EPERM;
871871
872872
873− for (; start < end; start += PAGE_SIZE << order) {
873+ for (; start < end; start += size) {
874874 unsigned long pfn;
875875 int ret;
876876
@@ -882,9 +882,9 @@ static int madvise_inject_error(int behavior, · 5 unmodified lines
882882 /*
883883 * When soft offlining hugepages, after migrating the page
884884 * we dissolve it, therefore in the second loop "page" will
885− * no longer be a compound page, and order will be 0.
885+ * no longer be a compound page.
886886 */
887− order = compound_order(compound_head(page));
887+ size = page_size(compound_head(page));
888888
889889 if (PageHWPoison(page)) {
890890 put_page(page);
@@ -895,7 +895,7 @@ static int madvise_inject_error(int behavior, · 4 unmodified lines
895895 pr_info("Soft offlining pfn %#lx at process virtual address %#lx\n",
896896 pfn, start);
897897
898− ret = soft_offline_page(page, MF_COUNT_INCREASED);
898+ ret = soft_offline_page(pfn, MF_COUNT_INCREASED);
899899 if (ret)
900900 return ret;
901901 continue;
@@ -1059,9 +1059,9 @@ SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior) · 157 unmodified lines
10591059 if (!madvise_behavior_valid(behavior))
10601060 return error;
10611061
1062− if (start & ~PAGE_MASK)
1062+ if (!PAGE_ALIGNED(start))
10631063 return error;
1064− len = (len_in + ~PAGE_MASK) & PAGE_MASK;
1064+ len = PAGE_ALIGN(len_in);
10651065
10661066 /* Check to see whether len was rounded up from small -ve to zero */
10671067 if (len_in && !len)
mm/memblock.c+75−36
@@ -57,42 +57,38 @@ · 56 unmodified lines
5757 * at build time. The region arrays for the "memory" and "reserved"
5858 * types are initially sized to %INIT_MEMBLOCK_REGIONS and for the
5959 * "physmap" type to %INIT_PHYSMEM_REGIONS.
60− * The :c:func:`memblock_allow_resize` enables automatic resizing of
61− * the region arrays during addition of new regions. This feature
62− * should be used with care so that memory allocated for the region
63− * array will not overlap with areas that should be reserved, for
64− * example initrd.
60+ * The memblock_allow_resize() enables automatic resizing of the region
61+ * arrays during addition of new regions. This feature should be used
62+ * with care so that memory allocated for the region array will not
63+ * overlap with areas that should be reserved, for example initrd.
6564 *
6665 * The early architecture setup should tell memblock what the physical
67− * memory layout is by using :c:func:`memblock_add` or
68− * :c:func:`memblock_add_node` functions. The first function does not
69− * assign the region to a NUMA node and it is appropriate for UMA
70− * systems. Yet, it is possible to use it on NUMA systems as well and
71− * assign the region to a NUMA node later in the setup process using
72− * :c:func:`memblock_set_node`. The :c:func:`memblock_add_node`
73− * performs such an assignment directly.
66+ * memory layout is by using memblock_add() or memblock_add_node()
67+ * functions. The first function does not assign the region to a NUMA
68+ * node and it is appropriate for UMA systems. Yet, it is possible to
69+ * use it on NUMA systems as well and assign the region to a NUMA node
70+ * later in the setup process using memblock_set_node(). The
71+ * memblock_add_node() performs such an assignment directly.
7472 *
7573 * Once memblock is setup the memory can be allocated using one of the
7674 * API variants:
7775 *
78− * * :c:func:`memblock_phys_alloc*` - these functions return the
79− * **physical** address of the allocated memory
80− * * :c:func:`memblock_alloc*` - these functions return the **virtual**
81− * address of the allocated memory.
76+ * * memblock_phys_alloc*() - these functions return the **physical**
77+ * address of the allocated memory
78+ * * memblock_alloc*() - these functions return the **virtual** address
79+ * of the allocated memory.
8280 *
8381 * Note, that both API variants use implict assumptions about allowed
8482 * memory ranges and the fallback methods. Consult the documentation
85− * of :c:func:`memblock_alloc_internal` and
86− * :c:func:`memblock_alloc_range_nid` functions for more elaboarte
87− * description.
83+ * of memblock_alloc_internal() and memblock_alloc_range_nid()
84+ * functions for more elaborate description.
8885 *
89− * As the system boot progresses, the architecture specific
90− * :c:func:`mem_init` function frees all the memory to the buddy page
91− * allocator.
86+ * As the system boot progresses, the architecture specific mem_init()
87+ * function frees all the memory to the buddy page allocator.
9288 *
93− * Unless an architecure enables %CONFIG_ARCH_KEEP_MEMBLOCK, the
89+ * Unless an architecture enables %CONFIG_ARCH_KEEP_MEMBLOCK, the
9490 * memblock data structures will be discarded after the system
95− * initialization compltes.
91+ * initialization completes.
9692 */
9793
9894 #ifndef CONFIG_NEED_MULTIPLE_NODES
@@ -1323,12 +1319,13 @@ __next_mem_pfn_range_in_zone(u64 *idx, struct zone *zone, · 1224 unmodified lines
13231319 * @start: the lower bound of the memory region to allocate (phys address)
13241320 * @end: the upper bound of the memory region to allocate (phys address)
13251321 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1322+ * @exact_nid: control the allocation fall back to other nodes
13261323 *
13271324 * The allocation is performed from memory region limited by
1328− * memblock.current_limit if @max_addr == %MEMBLOCK_ALLOC_ACCESSIBLE.
1325+ * memblock.current_limit if @end == %MEMBLOCK_ALLOC_ACCESSIBLE.
13291326 *
1330− * If the specified node can not hold the requested memory the
1331− * allocation falls back to any node in the system
1327+ * If the specified node can not hold the requested memory and @exact_nid
1328+ * is false, the allocation falls back to any node in the system.
13321329 *
13331330 * For systems with memory mirroring, the allocation is attempted first
13341331 * from the regions with mirroring enabled and then retried from any
@@ -1342,7 +1339,8 @@ __next_mem_pfn_range_in_zone(u64 *idx, struct zone *zone, · 7 unmodified lines
13421339 */
13431340 static phys_addr_t __init memblock_alloc_range_nid(phys_addr_t size,
13441341 phys_addr_t align, phys_addr_t start,
1345− phys_addr_t end, int nid)
1342+ phys_addr_t end, int nid,
1343+ bool exact_nid)
13461344 {
13471345 enum memblock_flags flags = choose_memblock_flags();
13481346 phys_addr_t found;
@@ -1362,7 +1360,7 @@ again: · 13 unmodified lines
13621360 if (found && !memblock_reserve(found, size))
13631361 goto done;
13641362
1365− if (nid != NUMA_NO_NODE) {
1363+ if (nid != NUMA_NO_NODE && !exact_nid) {
13661364 found = memblock_find_in_range_node(size, align, start,
13671365 end, NUMA_NO_NODE,
13681366 flags);
@@ -1410,7 +1408,8 @@ phys_addr_t __init memblock_phys_alloc_range(phys_addr_t size, · 41 unmodified lines
14101408 phys_addr_t start,
14111409 phys_addr_t end)
14121410 {
1413− return memblock_alloc_range_nid(size, align, start, end, NUMA_NO_NODE);
1411+ return memblock_alloc_range_nid(size, align, start, end, NUMA_NO_NODE,
1412+ false);
14141413 }
14151414
14161415 /**
@@ -1429,7 +1428,7 @@ phys_addr_t __init memblock_phys_alloc_range(phys_addr_t size, · 12 unmodified lines
14291428 phys_addr_t __init memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid)
14301429 {
14311430 return memblock_alloc_range_nid(size, align, 0,
1432− MEMBLOCK_ALLOC_ACCESSIBLE, nid);
1431+ MEMBLOCK_ALLOC_ACCESSIBLE, nid, false);
14331432 }
14341433
14351434 /**
@@ -1439,6 +1438,7 @@ phys_addr_t __init memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t ali · 3 unmodified lines
14391438 * @min_addr: the lower bound of the memory region to allocate (phys address)
14401439 * @max_addr: the upper bound of the memory region to allocate (phys address)
14411440 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1441+ * @exact_nid: control the allocation fall back to other nodes
14421442 *
14431443 * Allocates memory block using memblock_alloc_range_nid() and
14441444 * converts the returned physical address to virtual.
@@ -1454,7 +1454,7 @@ phys_addr_t __init memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t ali · 9 unmodified lines
14541454 static void * __init memblock_alloc_internal(
14551455 phys_addr_t size, phys_addr_t align,
14561456 phys_addr_t min_addr, phys_addr_t max_addr,
1457− int nid)
1457+ int nid, bool exact_nid)
14581458 {
14591459 phys_addr_t alloc;
14601460
@@ -1469,11 +1469,13 @@ static void * __init memblock_alloc_internal( · 8 unmodified lines
14691469 if (max_addr > memblock.current_limit)
14701470 max_addr = memblock.current_limit;
14711471
1472− alloc = memblock_alloc_range_nid(size, align, min_addr, max_addr, nid);
1472+ alloc = memblock_alloc_range_nid(size, align, min_addr, max_addr, nid,
1473+ exact_nid);
14731474
14741475 /* retry allocation without lower limit */
14751476 if (!alloc && min_addr)
1476− alloc = memblock_alloc_range_nid(size, align, 0, max_addr, nid);
1477+ alloc = memblock_alloc_range_nid(size, align, 0, max_addr, nid,
1478+ exact_nid);
14771479
14781480 if (!alloc)
14791481 return NULL;
@@ -1481,6 +1483,43 @@ static void * __init memblock_alloc_internal( · 1 unmodified line
14811483 return phys_to_virt(alloc);
14821484 }
14831485
1486+/**
1487+ * memblock_alloc_exact_nid_raw - allocate boot memory block on the exact node
1488+ * without zeroing memory
1489+ * @size: size of memory block to be allocated in bytes
1490+ * @align: alignment of the region and block's size
1491+ * @min_addr: the lower bound of the memory region from where the allocation
1492+ * is preferred (phys address)
1493+ * @max_addr: the upper bound of the memory region from where the allocation
1494+ * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
1495+ * allocate only from memory limited by memblock.current_limit value
1496+ * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1497+ *
1498+ * Public function, provides additional debug information (including caller
1499+ * info), if enabled. Does not zero allocated memory.
1500+ *
1501+ * Return:
1502+ * Virtual address of allocated memory block on success, NULL on failure.
1503+ */
1504+void * __init memblock_alloc_exact_nid_raw(
1505+ phys_addr_t size, phys_addr_t align,
1506+ phys_addr_t min_addr, phys_addr_t max_addr,
1507+ int nid)
1508+{
1509+ void *ptr;
1510+
1511+ memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
1512+ __func__, (u64)size, (u64)align, nid, &min_addr,
1513+ &max_addr, (void *)_RET_IP_);
1514+
1515+ ptr = memblock_alloc_internal(size, align,
1516+ min_addr, max_addr, nid, true);
1517+ if (ptr && size > 0)
1518+ page_init_poison(ptr, size);
1519+
1520+ return ptr;
1521+}
1522+
14841523 /**
14851524 * memblock_alloc_try_nid_raw - allocate boot memory block without zeroing
14861525 * memory and without panicking
@@ -1512,7 +1551,7 @@ void * __init memblock_alloc_try_nid_raw( · 25 unmodified lines
15121551 &max_addr, (void *)_RET_IP_);
15131552
15141553 ptr = memblock_alloc_internal(size, align,
1515− min_addr, max_addr, nid);
1554+ min_addr, max_addr, nid, false);
15161555 if (ptr && size > 0)
15171556 page_init_poison(ptr, size);
15181557
@@ -1547,7 +1586,7 @@ void * __init memblock_alloc_try_nid( · 28 unmodified lines
15471586 __func__, (u64)size, (u64)align, nid, &min_addr,
15481587 &max_addr, (void *)_RET_IP_);
15491588 ptr = memblock_alloc_internal(size, align,
1550− min_addr, max_addr, nid);
1589+ min_addr, max_addr, nid, false);
15511590 if (ptr)
15521591 memset(ptr, 0, size);
15531592
mm/memcontrol.c+33−134
@@ -108,7 +108,6 @@ static const char *const mem_cgroup_lru_names[] = { · 107 unmodified lines
108108
109109 #define THRESHOLDS_EVENTS_TARGET 128
110110 #define SOFTLIMIT_EVENTS_TARGET 1024
111−#define NUMAINFO_EVENTS_TARGET 1024
112111
113112 /*
114113 * Cgroups above their limits are maintained in a RB-Tree, independent of
@@ -778,7 +777,7 @@ void __mod_lruvec_slab_state(void *p, enum node_stat_item idx, int val) · 663 unmodified lines
778777 if (!memcg || memcg == root_mem_cgroup) {
779778 __mod_node_page_state(pgdat, idx, val);
780779 } else {
781− lruvec = mem_cgroup_lruvec(pgdat, memcg);
780+ lruvec = mem_cgroup_lruvec(memcg, pgdat);
782781 __mod_lruvec_state(lruvec, idx, val);
783782 }
784783 rcu_read_unlock();
@@ -877,9 +876,6 @@ static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg, · 92 unmodified lines
877876 case MEM_CGROUP_TARGET_SOFTLIMIT:
878877 next = val + SOFTLIMIT_EVENTS_TARGET;
879878 break;
880− case MEM_CGROUP_TARGET_NUMAINFO:
881− next = val + NUMAINFO_EVENTS_TARGET;
882− break;
883879 default:
884880 break;
885881 }
@@ -899,21 +895,12 @@ static void memcg_check_events(struct mem_cgroup *memcg, struct page *page) · 13 unmodified lines
899895 if (unlikely(mem_cgroup_event_ratelimit(memcg,
900896 MEM_CGROUP_TARGET_THRESH))) {
901897 bool do_softlimit;
902− bool do_numainfo __maybe_unused;
903898
904899 do_softlimit = mem_cgroup_event_ratelimit(memcg,
905900 MEM_CGROUP_TARGET_SOFTLIMIT);
906−#if MAX_NUMNODES > 1
907− do_numainfo = mem_cgroup_event_ratelimit(memcg,
908− MEM_CGROUP_TARGET_NUMAINFO);
909−#endif
910901 mem_cgroup_threshold(memcg);
911902 if (unlikely(do_softlimit))
912903 mem_cgroup_update_tree(memcg, page);
913−#if MAX_NUMNODES > 1
914− if (unlikely(do_numainfo))
915− atomic_inc(&memcg->numainfo_events);
916−#endif
917904 }
918905 }
919906
@@ -1052,7 +1039,7 @@ struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root, · 132 unmodified lines
10521039 struct mem_cgroup_per_node *mz;
10531040
10541041 mz = mem_cgroup_nodeinfo(root, reclaim->pgdat->node_id);
1055− iter = &mz->iter[reclaim->priority];
1042+ iter = &mz->iter;
10561043
10571044 if (prev && reclaim->generation != iter->generation)
10581045 goto out_unlock;
@@ -1152,15 +1139,11 @@ static void __invalidate_reclaim_iterators(struct mem_cgroup *from, · 93 unmodified lines
11521139 struct mem_cgroup_reclaim_iter *iter;
11531140 struct mem_cgroup_per_node *mz;
11541141 int nid;
1155− int i;
11561142
11571143 for_each_node(nid) {
11581144 mz = mem_cgroup_nodeinfo(from, nid);
1159− for (i = 0; i <= DEF_PRIORITY; i++) {
1160− iter = &mz->iter[i];
1161− cmpxchg(&iter->position,
1162− dead_memcg, NULL);
1163− }
1145+ iter = &mz->iter;
1146+ cmpxchg(&iter->position, dead_memcg, NULL);
11641147 }
11651148 }
11661149
@@ -1238,7 +1221,7 @@ struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgd · 71 unmodified lines
12381221 struct lruvec *lruvec;
12391222
12401223 if (mem_cgroup_disabled()) {
1241− lruvec = &pgdat->lruvec;
1224+ lruvec = &pgdat->__lruvec;
12421225 goto out;
12431226 }
12441227
@@ -1595,104 +1578,6 @@ static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask, · 350 unmodified lines
15951578 return ret;
15961579 }
15971580
1598−#if MAX_NUMNODES > 1
1599−
1600−/**
1601− * test_mem_cgroup_node_reclaimable
1602− * @memcg: the target memcg
1603− * @nid: the node ID to be checked.
1604− * @noswap : specify true here if the user wants flle only information.
1605− *
1606− * This function returns whether the specified memcg contains any
1607− * reclaimable pages on a node. Returns true if there are any reclaimable
1608− * pages in the node.
1609− */
1610−static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
1611− int nid, bool noswap)
1612−{
1613− struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg);
1614−
1615− if (lruvec_page_state(lruvec, NR_INACTIVE_FILE) ||
1616− lruvec_page_state(lruvec, NR_ACTIVE_FILE))
1617− return true;
1618− if (noswap || !total_swap_pages)
1619− return false;
1620− if (lruvec_page_state(lruvec, NR_INACTIVE_ANON) ||
1621− lruvec_page_state(lruvec, NR_ACTIVE_ANON))
1622− return true;
1623− return false;
1624−
1625−}
1626−
1627−/*
1628− * Always updating the nodemask is not very good - even if we have an empty
1629− * list or the wrong list here, we can start from some node and traverse all
1630− * nodes based on the zonelist. So update the list loosely once per 10 secs.
1631− *
1632− */
1633−static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
1634−{
1635− int nid;
1636− /*
1637− * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
1638− * pagein/pageout changes since the last update.
1639− */
1640− if (!atomic_read(&memcg->numainfo_events))
1641− return;
1642− if (atomic_inc_return(&memcg->numainfo_updating) > 1)
1643− return;
1644−
1645− /* make a nodemask where this memcg uses memory from */
1646− memcg->scan_nodes = node_states[N_MEMORY];
1647−
1648− for_each_node_mask(nid, node_states[N_MEMORY]) {
1649−
1650− if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
1651− node_clear(nid, memcg->scan_nodes);
1652− }
1653−
1654− atomic_set(&memcg->numainfo_events, 0);
1655− atomic_set(&memcg->numainfo_updating, 0);
1656−}
1657−
1658−/*
1659− * Selecting a node where we start reclaim from. Because what we need is just
1660− * reducing usage counter, start from anywhere is O,K. Considering
1661− * memory reclaim from current node, there are pros. and cons.
1662− *
1663− * Freeing memory from current node means freeing memory from a node which
1664− * we'll use or we've used. So, it may make LRU bad. And if several threads
1665− * hit limits, it will see a contention on a node. But freeing from remote
1666− * node means more costs for memory reclaim because of memory latency.
1667− *
1668− * Now, we use round-robin. Better algorithm is welcomed.
1669− */
1670−int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1671−{
1672− int node;
1673−
1674− mem_cgroup_may_update_nodemask(memcg);
1675− node = memcg->last_scanned_node;
1676−
1677− node = next_node_in(node, memcg->scan_nodes);
1678− /*
1679− * mem_cgroup_may_update_nodemask might have seen no reclaimmable pages
1680− * last time it really checked all the LRUs due to rate limiting.
1681− * Fallback to the current node in that case for simplicity.
1682− */
1683− if (unlikely(node == MAX_NUMNODES))
1684− node = numa_node_id();
1685−
1686− memcg->last_scanned_node = node;
1687− return node;
1688−}
1689−#else
1690−int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1691−{
1692− return 0;
1693−}
1694−#endif
1695−
16961581 static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
16971582 pg_data_t *pgdat,
16981583 gfp_t gfp_mask,
@@ -1705,7 +1590,6 @@ static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg, · 6 unmodified lines
17051590 unsigned long nr_scanned;
17061591 struct mem_cgroup_reclaim_cookie reclaim = {
17071592 .pgdat = pgdat,
1708− .priority = 0,
17091593 };
17101594
17111595 excess = soft_limit_excess(root_memcg);
@@ -3750,7 +3634,7 @@ static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css, · 2038 unmodified lines
37503634 static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
37513635 int nid, unsigned int lru_mask)
37523636 {
3753− struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg);
3637+ struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
37543638 unsigned long nr = 0;
37553639 enum lru_list lru;
37563640
@@ -5078,7 +4962,6 @@ static struct mem_cgroup *mem_cgroup_alloc(void) · 1321 unmodified lines
50784962 goto fail;
50794963
50804964 INIT_WORK(&memcg->high_work, high_work_func);
5081− memcg->last_scanned_node = MAX_NUMNODES;
50824965 INIT_LIST_HEAD(&memcg->oom_notify);
50834966 mutex_init(&memcg->thresholds_lock);
50844967 spin_lock_init(&memcg->move_lock);
@@ -5455,8 +5338,8 @@ static int mem_cgroup_move_account(struct page *page, · 370 unmodified lines
54555338 anon = PageAnon(page);
54565339
54575340 pgdat = page_pgdat(page);
5458− from_vec = mem_cgroup_lruvec(pgdat, from);
5459− to_vec = mem_cgroup_lruvec(pgdat, to);
5341+ from_vec = mem_cgroup_lruvec(from, pgdat);
5342+ to_vec = mem_cgroup_lruvec(to, pgdat);
54605343
54615344 spin_lock_irqsave(&from->move_lock, flags);
54625345
@@ -6096,7 +5979,8 @@ static ssize_t memory_high_write(struct kernfs_open_file *of, · 633 unmodified lines
60965979 char *buf, size_t nbytes, loff_t off)
60975980 {
60985981 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6099− unsigned long nr_pages;
5982+ unsigned int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
5983+ bool drained = false;
61005984 unsigned long high;
61015985 int err;
61025986
@@ -6107,12 +5991,29 @@ static ssize_t memory_high_write(struct kernfs_open_file *of, · 4 unmodified lines
61075991
61085992 memcg->high = high;
61095993
6110− nr_pages = page_counter_read(&memcg->memory);
6111− if (nr_pages > high)
6112− try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
6113− GFP_KERNEL, true);
5994+ for (;;) {
5995+ unsigned long nr_pages = page_counter_read(&memcg->memory);
5996+ unsigned long reclaimed;
5997+
5998+ if (nr_pages <= high)
5999+ break;
6000+
6001+ if (signal_pending(current))
6002+ break;
6003+
6004+ if (!drained) {
6005+ drain_all_stock(memcg);
6006+ drained = true;
6007+ continue;
6008+ }
6009+
6010+ reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
6011+ GFP_KERNEL, true);
6012+
6013+ if (!reclaimed && !nr_retries--)
6014+ break;
6015+ }
61146016
6115− memcg_wb_domain_size_changed(memcg);
61166017 return nbytes;
61176018 }
61186019
@@ -6144,10 +6045,8 @@ static ssize_t memory_max_write(struct kernfs_open_file *of, · 25 unmodified lines
61446045 if (nr_pages <= max)
61456046 break;
61466047
6147− if (signal_pending(current)) {
6148− err = -EINTR;
6048+ if (signal_pending(current))
61496049 break;
6150− }
61516050
61526051 if (!drained) {
61536052 drain_all_stock(memcg);
mm/memory-failure.c+23−38
@@ -303,30 +303,24 @@ static unsigned long dev_pagemap_mapping_shift(struct page *page, · 302 unmodified lines
303303 /*
304304 * Schedule a process for later kill.
305305 * Uses GFP_ATOMIC allocations to avoid potential recursions in the VM.
306− * TBD would GFP_NOIO be enough?
307306 */
308307 static void add_to_kill(struct task_struct *tsk, struct page *p,
309308 struct vm_area_struct *vma,
310− struct list_head *to_kill,
311− struct to_kill **tkc)
309+ struct list_head *to_kill)
312310 {
313311 struct to_kill *tk;
314312
315− if (*tkc) {
316− tk = *tkc;
317− *tkc = NULL;
318− } else {
319− tk = kmalloc(sizeof(struct to_kill), GFP_ATOMIC);
320− if (!tk) {
321− pr_err("Memory failure: Out of memory while machine check handling\n");
322− return;
323− }
313+ tk = kmalloc(sizeof(struct to_kill), GFP_ATOMIC);
314+ if (!tk) {
315+ pr_err("Memory failure: Out of memory while machine check handling\n");
316+ return;
324317 }
318+
325319 tk->addr = page_address_in_vma(p, vma);
326320 if (is_zone_device_page(p))
327321 tk->size_shift = dev_pagemap_mapping_shift(p, vma);
328322 else
329− tk->size_shift = compound_order(compound_head(p)) + PAGE_SHIFT;
323+ tk->size_shift = page_shift(compound_head(p));
330324
331325 /*
332326 * Send SIGKILL if "tk->addr == -EFAULT". Also, as
@@ -345,6 +339,7 @@ static void add_to_kill(struct task_struct *tsk, struct page *p, · 12 unmodified lines
345339 kfree(tk);
346340 return;
347341 }
342+
348343 get_task_struct(tsk);
349344 tk->tsk = tsk;
350345 list_add_tail(&tk->nd, to_kill);
@@ -436,7 +431,7 @@ static struct task_struct *task_early_kill(struct task_struct *tsk, · 85 unmodified lines
436431 * Collect processes when the error hit an anonymous page.
437432 */
438433 static void collect_procs_anon(struct page *page, struct list_head *to_kill,
439− struct to_kill **tkc, int force_early)
434+ int force_early)
440435 {
441436 struct vm_area_struct *vma;
442437 struct task_struct *tsk;
@@ -461,7 +456,7 @@ static void collect_procs_anon(struct page *page, struct list_head *to_kill, · 18 unmodified lines
461456 if (!page_mapped_in_vma(page, vma))
462457 continue;
463458 if (vma->vm_mm == t->mm)
464− add_to_kill(t, page, vma, to_kill, tkc);
459+ add_to_kill(t, page, vma, to_kill);
465460 }
466461 }
467462 read_unlock(&tasklist_lock);
@@ -472,7 +467,7 @@ static void collect_procs_anon(struct page *page, struct list_head *to_kill, · 4 unmodified lines
472467 * Collect processes when the error hit a file mapped page.
473468 */
474469 static void collect_procs_file(struct page *page, struct list_head *to_kill,
475− struct to_kill **tkc, int force_early)
470+ int force_early)
476471 {
477472 struct vm_area_struct *vma;
478473 struct task_struct *tsk;
@@ -496,7 +491,7 @@ static void collect_procs_file(struct page *page, struct list_head *to_kill, · 17 unmodified lines
496491 * to be informed of all such data corruptions.
497492 */
498493 if (vma->vm_mm == t->mm)
499− add_to_kill(t, page, vma, to_kill, tkc);
494+ add_to_kill(t, page, vma, to_kill);
500495 }
501496 }
502497 read_unlock(&tasklist_lock);
@@ -505,26 +500,17 @@ static void collect_procs_file(struct page *page, struct list_head *to_kill, · 2 unmodified lines
505500
506501 /*
507502 * Collect the processes who have the corrupted page mapped to kill.
508− * This is done in two steps for locking reasons.
509− * First preallocate one tokill structure outside the spin locks,
510− * so that we can kill at least one process reasonably reliable.
511503 */
512504 static void collect_procs(struct page *page, struct list_head *tokill,
513505 int force_early)
514506 {
515− struct to_kill *tk;
516−
517507 if (!page->mapping)
518508 return;
519509
520− tk = kmalloc(sizeof(struct to_kill), GFP_NOIO);
521− if (!tk)
522− return;
523510 if (PageAnon(page))
524− collect_procs_anon(page, tokill, &tk, force_early);
511+ collect_procs_anon(page, tokill, force_early);
525512 else
526− collect_procs_file(page, tokill, &tk, force_early);
527− kfree(tk);
513+ collect_procs_file(page, tokill, force_early);
528514 }
529515
530516 static const char *action_name[] = {
@@ -1490,7 +1476,7 @@ static void memory_failure_work_func(struct work_struct *work) · 959 unmodified lines
14901476 if (!gotten)
14911477 break;
14921478 if (entry.flags & MF_SOFT_OFFLINE)
1493− soft_offline_page(pfn_to_page(entry.pfn), entry.flags);
1479+ soft_offline_page(entry.pfn, entry.flags);
14941480 else
14951481 memory_failure(entry.pfn, entry.flags);
14961482 }
@@ -1871,7 +1857,7 @@ static int soft_offline_free_page(struct page *page) · 374 unmodified lines
18711857
18721858 /**
18731859 * soft_offline_page - Soft offline a page.
1874− * @page: page to offline
1860+ * @pfn: pfn to soft-offline
18751861 * @flags: flags. Same as memory_failure().
18761862 *
18771863 * Returns 0 on success, otherwise negated errno.
@@ -1891,18 +1877,17 @@ static int soft_offline_free_page(struct page *page) · 13 unmodified lines
18911877 * This is not a 100% solution for all memory, but tries to be
18921878 * ``good enough'' for the majority of memory.
18931879 */
1894−int soft_offline_page(struct page *page, int flags)
1880+int soft_offline_page(unsigned long pfn, int flags)
18951881 {
18961882 int ret;
1897− unsigned long pfn = page_to_pfn(page);
1883+ struct page *page;
18981884
1899− if (is_zone_device_page(page)) {
1900− pr_debug_ratelimited("soft_offline: %#lx page is device page\n",
1901− pfn);
1902− if (flags & MF_COUNT_INCREASED)
1903− put_page(page);
1885+ if (!pfn_valid(pfn))
1886+ return -ENXIO;
1887+ /* Only online pages can be soft-offlined (esp., not ZONE_DEVICE). */
1888+ page = pfn_to_online_page(pfn);
1889+ if (!page)
19041890 return -EIO;
1905− }
19061891
19071892 if (PageHWPoison(page)) {
19081893 pr_info("soft offline: %#lx page already poisoned\n", pfn);
mm/memory.c+40−12
@@ -72,6 +72,8 @@ · 71 unmodified lines
7272 #include <linux/oom.h>
7373 #include <linux/numa.h>
7474
75+#include <trace/events/kmem.h>
76+
7577 #include <asm/io.h>
7678 #include <asm/mmu_context.h>
7779 #include <asm/pgalloc.h>
@@ -152,6 +154,10 @@ static int __init init_zero_pfn(void) · 74 unmodified lines
152154 }
153155 core_initcall(init_zero_pfn);
154156
157+void mm_trace_rss_stat(struct mm_struct *mm, int member, long count)
158+{
159+ trace_rss_stat(mm, member, count);
160+}
155161
156162 #if defined(SPLIT_RSS_COUNTING)
157163
@@ -2289,10 +2295,11 @@ static vm_fault_t do_page_mkwrite(struct vm_fault *vmf) · 2131 unmodified lines
22892295 *
22902296 * The function expects the page to be locked and unlocks it.
22912297 */
2292−static void fault_dirty_shared_page(struct vm_area_struct *vma,
2293− struct page *page)
2298+static vm_fault_t fault_dirty_shared_page(struct vm_fault *vmf)
22942299 {
2300+ struct vm_area_struct *vma = vmf->vma;
22952301 struct address_space *mapping;
2302+ struct page *page = vmf->page;
22962303 bool dirtied;
22972304 bool page_mkwrite = vma->vm_ops && vma->vm_ops->page_mkwrite;
22982305
@@ -2307,16 +2314,30 @@ static void fault_dirty_shared_page(struct vm_area_struct *vma, · 8 unmodified lines
23072314 mapping = page_rmapping(page);
23082315 unlock_page(page);
23092316
2317+ if (!page_mkwrite)
2318+ file_update_time(vma->vm_file);
2319+
2320+ /*
2321+ * Throttle page dirtying rate down to writeback speed.
2322+ *
2323+ * mapping may be NULL here because some device drivers do not
2324+ * set page.mapping but still dirty their pages
2325+ *
2326+ * Drop the mmap_sem before waiting on IO, if we can. The file
2327+ * is pinning the mapping, as per above.
2328+ */
23102329 if ((dirtied || page_mkwrite) && mapping) {
2311− /*
2312− * Some device drivers do not set page.mapping
2313− * but still dirty their pages
2314− */
2330+ struct file *fpin;
2331+
2332+ fpin = maybe_unlock_mmap_for_io(vmf, NULL);
23152333 balance_dirty_pages_ratelimited(mapping);
2334+ if (fpin) {
2335+ fput(fpin);
2336+ return VM_FAULT_RETRY;
2337+ }
23162338 }
23172339
2318− if (!page_mkwrite)
2319− file_update_time(vma->vm_file);
2340+ return 0;
23202341 }
23212342
23222343 /*
@@ -2571,6 +2592,7 @@ static vm_fault_t wp_page_shared(struct vm_fault *vmf) · 248 unmodified lines
25712592 __releases(vmf->ptl)
25722593 {
25732594 struct vm_area_struct *vma = vmf->vma;
2595+ vm_fault_t ret = VM_FAULT_WRITE;
25742596
25752597 get_page(vmf->page);
25762598
@@ -2594,10 +2616,10 @@ static vm_fault_t wp_page_shared(struct vm_fault *vmf) · 17 unmodified lines
25942616 wp_page_reuse(vmf);
25952617 lock_page(vmf->page);
25962618 }
2597− fault_dirty_shared_page(vma, vmf->page);
2619+ ret |= fault_dirty_shared_page(vmf);
25982620 put_page(vmf->page);
25992621
2600− return VM_FAULT_WRITE;
2622+ return ret;
26012623 }
26022624
26032625 /*
@@ -3083,7 +3105,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) · 479 unmodified lines
30833105
30843106 /*
30853107 * The memory barrier inside __SetPageUptodate makes sure that
3086− * preceeding stores to the page contents become visible before
3108+ * preceding stores to the page contents become visible before
30873109 * the set_pte_at() write.
30883110 */
30893111 __SetPageUptodate(page);
@@ -3641,7 +3663,7 @@ static vm_fault_t do_shared_fault(struct vm_fault *vmf) · 551 unmodified lines
36413663 return ret;
36423664 }
36433665
3644− fault_dirty_shared_page(vma, vmf->page);
3666+ ret |= fault_dirty_shared_page(vmf);
36453667 return ret;
36463668 }
36473669
@@ -3988,6 +4010,7 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, · 340 unmodified lines
39884010 vmf.pud = pud_alloc(mm, p4d, address);
39894011 if (!vmf.pud)
39904012 return VM_FAULT_OOM;
4013+retry_pud:
39914014 if (pud_none(*vmf.pud) && __transparent_hugepage_enabled(vma)) {
39924015 ret = create_huge_pud(&vmf);
39934016 if (!(ret & VM_FAULT_FALLBACK))
@@ -4014,6 +4037,11 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, · 20 unmodified lines
40144037 vmf.pmd = pmd_alloc(mm, vmf.pud, address);
40154038 if (!vmf.pmd)
40164039 return VM_FAULT_OOM;
4040+
4041+ /* Huge pud page fault raced with pmd_alloc? */
4042+ if (pud_trans_unstable(vmf.pud))
4043+ goto retry_pud;
4044+
40174045 if (pmd_none(*vmf.pmd) && __transparent_hugepage_enabled(vma)) {
40184046 ret = create_huge_pmd(&vmf);
40194047 if (!(ret & VM_FAULT_FALLBACK))
mm/memory_hotplug.c+56−30
@@ -49,8 +49,6 @@ · 48 unmodified lines
4949 * and restore_online_page_callback() for generic callback restore.
5050 */
5151
52−static void generic_online_page(struct page *page, unsigned int order);
53−
5452 static online_page_callback_t online_page_callback = generic_online_page;
5553 static DEFINE_MUTEX(online_page_callback_lock);
5654
@@ -278,6 +276,22 @@ static int check_pfn_span(unsigned long pfn, unsigned long nr_pages, · 221 unmodified lines
278276 return 0;
279277 }
280278
279+static int check_hotplug_memory_addressable(unsigned long pfn,
280+ unsigned long nr_pages)
281+{
282+ const u64 max_addr = PFN_PHYS(pfn + nr_pages) - 1;
283+
284+ if (max_addr >> MAX_PHYSMEM_BITS) {
285+ const u64 max_allowed = (1ull << (MAX_PHYSMEM_BITS + 1)) - 1;
286+ WARN(1,
287+ "Hotplugged memory exceeds maximum addressable address, range=%#llx-%#llx, maximum=%#llx\n",
288+ (u64)PFN_PHYS(pfn), max_addr, max_allowed);
289+ return -E2BIG;
290+ }
291+
292+ return 0;
293+}
294+
281295 /*
282296 * Reasonably generic function for adding memory. It is
283297 * expected that archs that support memory hotplug will
@@ -291,6 +305,10 @@ int __ref __add_pages(int nid, unsigned long pfn, unsigned long nr_pages, · 7 unmodified lines
291305 unsigned long nr, start_sec, end_sec;
292306 struct vmem_altmap *altmap = restrictions->altmap;
293307
308+ err = check_hotplug_memory_addressable(pfn, nr_pages);
309+ if (err)
310+ return err;
311+
294312 if (altmap) {
295313 /*
296314 * Validate altmap is within bounds of the total request
@@ -580,24 +598,7 @@ int restore_online_page_callback(online_page_callback_t callback) · 283 unmodified lines
580598 }
581599 EXPORT_SYMBOL_GPL(restore_online_page_callback);
582600
583−void __online_page_set_limits(struct page *page)
584−{
585−}
586−EXPORT_SYMBOL_GPL(__online_page_set_limits);
587−
588−void __online_page_increment_counters(struct page *page)
589−{
590− adjust_managed_page_count(page, 1);
591−}
592−EXPORT_SYMBOL_GPL(__online_page_increment_counters);
593−
594−void __online_page_free(struct page *page)
595−{
596− __free_reserved_page(page);
597−}
598−EXPORT_SYMBOL_GPL(__online_page_free);
599−
600−static void generic_online_page(struct page *page, unsigned int order)
601+void generic_online_page(struct page *page, unsigned int order)
601602 {
602603 kernel_map_pages(page, 1 << order, 1);
603604 __free_pages_core(page, order);
@@ -607,6 +608,7 @@ static void generic_online_page(struct page *page, unsigned int order) · 3 unmodified lines
607608 totalhigh_pages_add(1UL << order);
608609 #endif
609610 }
611+EXPORT_SYMBOL_GPL(generic_online_page);
610612
611613 static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages,
612614 void *arg)
@@ -1180,7 +1182,8 @@ static bool is_pageblock_removable_nolock(unsigned long pfn) · 567 unmodified lines
11801182 if (!zone_spans_pfn(zone, pfn))
11811183 return false;
11821184
1183− return !has_unmovable_pages(zone, page, 0, MIGRATE_MOVABLE, SKIP_HWPOISON);
1185+ return !has_unmovable_pages(zone, page, 0, MIGRATE_MOVABLE,
1186+ MEMORY_OFFLINE);
11841187 }
11851188
11861189 /* Checks if this range of memory is likely to be hot-removable. */
@@ -1377,9 +1380,7 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) · 190 unmodified lines
13771380 return ret;
13781381 }
13791382
1380−/*
1381− * remove from free_area[] and mark all as Reserved.
1382− */
1383+/* Mark all sections offline and remove all free pages from the buddy. */
13831384 static int
13841385 offline_isolated_pages_cb(unsigned long start, unsigned long nr_pages,
13851386 void *data)
@@ -1397,7 +1398,8 @@ static int · 11 unmodified lines
13971398 check_pages_isolated_cb(unsigned long start_pfn, unsigned long nr_pages,
13981399 void *data)
13991400 {
1400− return test_pages_isolated(start_pfn, start_pfn + nr_pages, true);
1401+ return test_pages_isolated(start_pfn, start_pfn + nr_pages,
1402+ MEMORY_OFFLINE);
14011403 }
14021404
14031405 static int __init cmdline_parse_movable_node(char *p)
@@ -1478,10 +1480,19 @@ static void node_states_clear_node(int node, struct memory_notify *arg) · 74 unmodified lines
14781480 node_clear_state(node, N_MEMORY);
14791481 }
14801482
1483+static int count_system_ram_pages_cb(unsigned long start_pfn,
1484+ unsigned long nr_pages, void *data)
1485+{
1486+ unsigned long *nr_system_ram_pages = data;
1487+
1488+ *nr_system_ram_pages += nr_pages;
1489+ return 0;
1490+}
1491+
14811492 static int __ref __offline_pages(unsigned long start_pfn,
14821493 unsigned long end_pfn)
14831494 {
1484− unsigned long pfn, nr_pages;
1495+ unsigned long pfn, nr_pages = 0;
14851496 unsigned long offlined_pages = 0;
14861497 int ret, node, nr_isolate_pageblock;
14871498 unsigned long flags;
@@ -1492,6 +1503,22 @@ static int __ref __offline_pages(unsigned long start_pfn, · 4 unmodified lines
14921503
14931504 mem_hotplug_begin();
14941505
1506+ /*
1507+ * Don't allow to offline memory blocks that contain holes.
1508+ * Consequently, memory blocks with holes can never get onlined
1509+ * via the hotplug path - online_pages() - as hotplugged memory has
1510+ * no holes. This way, we e.g., don't have to worry about marking
1511+ * memory holes PG_reserved, don't need pfn_valid() checks, and can
1512+ * avoid using walk_system_ram_range() later.
1513+ */
1514+ walk_system_ram_range(start_pfn, end_pfn - start_pfn, &nr_pages,
1515+ count_system_ram_pages_cb);
1516+ if (nr_pages != end_pfn - start_pfn) {
1517+ ret = -EINVAL;
1518+ reason = "memory holes";
1519+ goto failed_removal;
1520+ }
1521+
14951522 /* This makes hotplug much easier...and readable.
14961523 we assume this for now. .*/
14971524 if (!test_pages_in_a_zone(start_pfn, end_pfn, &valid_start,
@@ -1503,12 +1530,11 @@ static int __ref __offline_pages(unsigned long start_pfn, · 5 unmodified lines
15031530
15041531 zone = page_zone(pfn_to_page(valid_start));
15051532 node = zone_to_nid(zone);
1506− nr_pages = end_pfn - start_pfn;
15071533
15081534 /* set above range as isolated */
15091535 ret = start_isolate_page_range(start_pfn, end_pfn,
15101536 MIGRATE_MOVABLE,
1511− SKIP_HWPOISON | REPORT_FAILURE);
1537+ MEMORY_OFFLINE | REPORT_FAILURE);
15121538 if (ret < 0) {
15131539 reason = "failure to isolate range";
15141540 goto failed_removal;
@@ -1750,13 +1776,13 @@ static int __ref try_remove_memory(int nid, u64 start, u64 size) · 235 unmodified lines
17501776
17511777 /* remove memmap entry */
17521778 firmware_map_remove(start, start + size, "System RAM");
1753− memblock_free(start, size);
1754− memblock_remove(start, size);
17551779
17561780 /* remove memory block devices before removing memory */
17571781 remove_memory_block_devices(start, size);
17581782
17591783 arch_remove_memory(nid, start, size, NULL);
1784+ memblock_free(start, size);
1785+ memblock_remove(start, size);
17601786 __release_memory_resource(start, size);
17611787
17621788 try_offline_node(nid);
mm/mempolicy.c+31−16
@@ -410,7 +410,9 @@ struct queue_pages { · 409 unmodified lines
410410 struct list_head *pagelist;
411411 unsigned long flags;
412412 nodemask_t *nmask;
413− struct vm_area_struct *prev;
413+ unsigned long start;
414+ unsigned long end;
415+ struct vm_area_struct *first;
414416 };
415417
416418 /*
@@ -618,6 +620,22 @@ static int queue_pages_test_walk(unsigned long start, unsigned long end, · 201 unmodified lines
618620 unsigned long endvma = vma->vm_end;
619621 unsigned long flags = qp->flags;
620622
623+ /* range check first */
624+ VM_BUG_ON((vma->vm_start > start) || (vma->vm_end < end));
625+
626+ if (!qp->first) {
627+ qp->first = vma;
628+ if (!(flags & MPOL_MF_DISCONTIG_OK) &&
629+ (qp->start < vma->vm_start))
630+ /* hole at head side of range */
631+ return -EFAULT;
632+ }
633+ if (!(flags & MPOL_MF_DISCONTIG_OK) &&
634+ ((vma->vm_end < qp->end) &&
635+ (!vma->vm_next || vma->vm_end < vma->vm_next->vm_start)))
636+ /* hole at middle or tail of range */
637+ return -EFAULT;
638+
621639 /*
622640 * Need check MPOL_MF_STRICT to return -EIO if possible
623641 * regardless of vma_migratable
@@ -628,17 +646,6 @@ static int queue_pages_test_walk(unsigned long start, unsigned long end, · 4 unmodified lines
628646
629647 if (endvma > end)
630648 endvma = end;
631− if (vma->vm_start > start)
632− start = vma->vm_start;
633−
634− if (!(flags & MPOL_MF_DISCONTIG_OK)) {
635− if (!vma->vm_next && vma->vm_end < end)
636− return -EFAULT;
637− if (qp->prev && qp->prev->vm_end < vma->vm_start)
638− return -EFAULT;
639− }
640−
641− qp->prev = vma;
642649
643650 if (flags & MPOL_MF_LAZY) {
644651 /* Similar to task_numa_work, skip inaccessible VMAs */
@@ -681,14 +688,23 @@ queue_pages_range(struct mm_struct *mm, unsigned long start, unsigned long end, · 36 unmodified lines
681688 nodemask_t *nodes, unsigned long flags,
682689 struct list_head *pagelist)
683690 {
691+ int err;
684692 struct queue_pages qp = {
685693 .pagelist = pagelist,
686694 .flags = flags,
687695 .nmask = nodes,
688− .prev = NULL,
696+ .start = start,
697+ .end = end,
698+ .first = NULL,
689699 };
690700
691− return walk_page_range(mm, start, end, &queue_pages_walk_ops, &qp);
701+ err = walk_page_range(mm, start, end, &queue_pages_walk_ops, &qp);
702+
703+ if (!qp.first)
704+ /* whole range in hole */
705+ err = -EFAULT;
706+
707+ return err;
692708 }
693709
694710 /*
@@ -740,8 +756,7 @@ static int mbind_range(struct mm_struct *mm, unsigned long start, · 45 unmodified lines
740756 unsigned long vmend;
741757
742758 vma = find_vma(mm, start);
743− if (!vma || vma->vm_start > start)
744− return -EFAULT;
759+ VM_BUG_ON(!vma);
745760
746761 prev = vma->vm_prev;
747762 if (start > vma->vm_start)
mm/migrate.c+6−10
@@ -1168,15 +1168,11 @@ static ICE_noinline int unmap_and_move(new_page_t get_new_page, · 1167 unmodified lines
11681168 enum migrate_reason reason)
11691169 {
11701170 int rc = MIGRATEPAGE_SUCCESS;
1171− struct page *newpage;
1171+ struct page *newpage = NULL;
11721172
11731173 if (!thp_migration_supported() && PageTransHuge(page))
11741174 return -ENOMEM;
11751175
1176− newpage = get_new_page(page, private);
1177− if (!newpage)
1178− return -ENOMEM;
1179−
11801176 if (page_count(page) == 1) {
11811177 /* page was freed from under us. So we are done. */
11821178 ClearPageActive(page);
@@ -1187,13 +1183,13 @@ static ICE_noinline int unmap_and_move(new_page_t get_new_page, · 4 unmodified lines
11871183 __ClearPageIsolated(page);
11881184 unlock_page(page);
11891185 }
1190− if (put_new_page)
1191− put_new_page(newpage, private);
1192− else
1193− put_page(newpage);
11941186 goto out;
11951187 }
11961188
1189+ newpage = get_new_page(page, private);
1190+ if (!newpage)
1191+ return -ENOMEM;
1192+
11971193 rc = __unmap_and_move(page, newpage, force, mode);
11981194 if (rc == MIGRATEPAGE_SUCCESS)
11991195 set_page_owner_migrate_reason(newpage, reason);
@@ -1863,7 +1859,7 @@ static bool migrate_balanced_pgdat(struct pglist_data *pgdat, · 663 unmodified lines
18631859 if (!zone_watermark_ok(zone, 0,
18641860 high_wmark_pages(zone) +
18651861 nr_migrate_pages,
1866− 0, 0))
1862+ ZONE_MOVABLE, 0))
18671863 continue;
18681864 return true;
18691865 }
mm/mmap.c+21−42
@@ -641,7 +641,7 @@ __vma_link(struct mm_struct *mm, struct vm_area_struct *vma, · 640 unmodified lines
641641 struct vm_area_struct *prev, struct rb_node **rb_link,
642642 struct rb_node *rb_parent)
643643 {
644− __vma_link_list(mm, vma, prev, rb_parent);
644+ __vma_link_list(mm, vma, prev);
645645 __vma_link_rb(mm, vma, rb_link, rb_parent);
646646 }
647647
@@ -684,37 +684,14 @@ static void __insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) · 36 unmodified lines
684684
685685 static __always_inline void __vma_unlink_common(struct mm_struct *mm,
686686 struct vm_area_struct *vma,
687− struct vm_area_struct *prev,
688− bool has_prev,
689687 struct vm_area_struct *ignore)
690688 {
691− struct vm_area_struct *next;
692−
693689 vma_rb_erase_ignore(vma, &mm->mm_rb, ignore);
694− next = vma->vm_next;
695− if (has_prev)
696− prev->vm_next = next;
697− else {
698− prev = vma->vm_prev;
699− if (prev)
700− prev->vm_next = next;
701− else
702− mm->mmap = next;
703− }
704− if (next)
705− next->vm_prev = prev;
706−
690+ __vma_unlink_list(mm, vma);
707691 /* Kill the cache */
708692 vmacache_invalidate(mm);
709693 }
710694
711−static inline void __vma_unlink_prev(struct mm_struct *mm,
712− struct vm_area_struct *vma,
713− struct vm_area_struct *prev)
714−{
715− __vma_unlink_common(mm, vma, prev, true, vma);
716−}
717−
718695 /*
719696 * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that
720697 * is already present in an i_mmap tree without adjusting the tree.
@@ -769,8 +746,6 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, · 48 unmodified lines
769746 remove_next = 1 + (end > next->vm_end);
770747 VM_WARN_ON(remove_next == 2 &&
771748 end != next->vm_next->vm_end);
772− VM_WARN_ON(remove_next == 1 &&
773− end != next->vm_end);
774749 /* trim end to next, for case 6 first pass */
775750 end = next->vm_end;
776751 }
@@ -889,7 +864,7 @@ again: · 112 unmodified lines
889864 * us to remove next before dropping the locks.
890865 */
891866 if (remove_next != 3)
892− __vma_unlink_prev(mm, next, vma);
867+ __vma_unlink_common(mm, next, next);
893868 else
894869 /*
895870 * vma is not before next if they've been
@@ -900,7 +875,7 @@ again: · 4 unmodified lines
900875 * "next" (which is stored in post-swap()
901876 * "vma").
902877 */
903− __vma_unlink_common(mm, next, NULL, false, vma);
878+ __vma_unlink_common(mm, next, vma);
904879 if (file)
905880 __remove_shared_vm_struct(next, file, mapping);
906881 } else if (insert) {
@@ -1116,15 +1091,18 @@ can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags, · 209 unmodified lines
11161091 * the area passed down from mprotect_fixup, never extending beyond one
11171092 * vma, PPPPPP is the prev vma specified, and NNNNNN the next vma after:
11181093 *
1119− * AAAA AAAA AAAA AAAA
1120− * PPPPPPNNNNNN PPPPPPNNNNNN PPPPPPNNNNNN PPPPNNNNXXXX
1121− * cannot merge might become might become might become
1122− * PPNNNNNNNNNN PPPPPPPPPPNN PPPPPPPPPPPP 6 or
1123− * mmap, brk or case 4 below case 5 below PPPPPPPPXXXX 7 or
1124− * mremap move: PPPPXXXXXXXX 8
1125− * AAAA
1126− * PPPP NNNN PPPPPPPPPPPP PPPPPPPPNNNN PPPPNNNNNNNN
1127− * might become case 1 below case 2 below case 3 below
1094+ * AAAA AAAA AAAA
1095+ * PPPPPPNNNNNN PPPPPPNNNNNN PPPPPPNNNNNN
1096+ * cannot merge might become might become
1097+ * PPNNNNNNNNNN PPPPPPPPPPNN
1098+ * mmap, brk or case 4 below case 5 below
1099+ * mremap move:
1100+ * AAAA AAAA
1101+ * PPPP NNNN PPPPNNNNXXXX
1102+ * might become might become
1103+ * PPPPPPPPPPPP 1 or PPPPPPPPPPPP 6 or
1104+ * PPPPPPPPNNNN 2 or PPPPPPPPXXXX 7 or
1105+ * PPPPNNNNNNNN 3 PPPPXXXXXXXX 8
11281106 *
11291107 * It is important for case 8 that the vma NNNN overlapping the
11301108 * region AAAA is never going to extended over XXXX. Instead XXXX must
@@ -1442,7 +1420,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, · 311 unmodified lines
14421420 * that it represents a valid section of the address space.
14431421 */
14441422 addr = get_unmapped_area(file, addr, len, pgoff, flags);
1445− if (offset_in_page(addr))
1423+ if (IS_ERR_VALUE(addr))
14461424 return addr;
14471425
14481426 if (flags & MAP_FIXED_NOREPLACE) {
@@ -3006,15 +2984,16 @@ static int do_brk_flags(unsigned long addr, unsigned long len, unsigned long fla · 1557 unmodified lines
30062984 struct rb_node **rb_link, *rb_parent;
30072985 pgoff_t pgoff = addr >> PAGE_SHIFT;
30082986 int error;
2987+ unsigned long mapped_addr;
30092988
30102989 /* Until we need other flags, refuse anything except VM_EXEC. */
30112990 if ((flags & (~VM_EXEC)) != 0)
30122991 return -EINVAL;
30132992 flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
30142993
3015− error = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
3016− if (offset_in_page(error))
3017− return error;
2994+ mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
2995+ if (IS_ERR_VALUE(mapped_addr))
2996+ return mapped_addr;
30182997
30192998 error = mlock_future_check(mm, mm->def_flags, len);
30202999 if (error)
mm/mprotect.c+4−4
@@ -80,6 +80,10 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, · 79 unmodified lines
8080 if (prot_numa) {
8181 struct page *page;
8282
83+ /* Avoid TLB flush if possible */
84+ if (pte_protnone(oldpte))
85+ continue;
86+
8387 page = vm_normal_page(vma, addr, oldpte);
8488 if (!page || PageKsm(page))
8589 continue;
@@ -97,10 +101,6 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, · 11 unmodified lines
97101 if (page_is_file_cache(page) && PageDirty(page))
98102 continue;
99103
100− /* Avoid TLB flush if possible */
101− if (pte_protnone(oldpte))
102− continue;
103−
104104 /*
105105 * Don't mess with PTEs if page is already on the node
106106 * a single-threaded process is running on.
mm/mremap.c+2−2
@@ -558,7 +558,7 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len, · 557 unmodified lines
558558 ret = get_unmapped_area(vma->vm_file, new_addr, new_len, vma->vm_pgoff +
559559 ((addr - vma->vm_start) >> PAGE_SHIFT),
560560 map_flags);
561− if (offset_in_page(ret))
561+ if (IS_ERR_VALUE(ret))
562562 goto out1;
563563
564564 ret = move_vma(vma, addr, old_len, new_len, new_addr, locked, uf,
@@ -706,7 +706,7 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, · 141 unmodified lines
706706 vma->vm_pgoff +
707707 ((addr - vma->vm_start) >> PAGE_SHIFT),
708708 map_flags);
709− if (offset_in_page(new_addr)) {
709+ if (IS_ERR_VALUE(new_addr)) {
710710 ret = new_addr;
711711 goto out;
712712 }
mm/nommu.c+2−8
@@ -648,7 +648,7 @@ static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma) · 647 unmodified lines
648648 if (rb_prev)
649649 prev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
650650
651− __vma_link_list(mm, vma, prev, parent);
651+ __vma_link_list(mm, vma, prev);
652652 }
653653
654654 /*
@@ -684,13 +684,7 @@ static void delete_vma_from_mm(struct vm_area_struct *vma) · 29 unmodified lines
684684 /* remove from the MM's tree and list */
685685 rb_erase(&vma->vm_rb, &mm->mm_rb);
686686
687− if (vma->vm_prev)
688− vma->vm_prev->vm_next = vma->vm_next;
689− else
690− mm->mmap = vma->vm_next;
691−
692− if (vma->vm_next)
693− vma->vm_next->vm_prev = vma->vm_prev;
687+ __vma_unlink_list(mm, vma);
694688 }
695689
696690 /*
mm/page_alloc.c+119−18
@@ -5354,6 +5354,7 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) · 5353 unmodified lines
53545354 " min:%lukB"
53555355 " low:%lukB"
53565356 " high:%lukB"
5357+ " reserved_highatomic:%luKB"
53575358 " active_anon:%lukB"
53585359 " inactive_anon:%lukB"
53595360 " active_file:%lukB"
@@ -5375,6 +5376,7 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) · 15 unmodified lines
53755376 K(min_wmark_pages(zone)),
53765377 K(low_wmark_pages(zone)),
53775378 K(high_wmark_pages(zone)),
5379+ K(zone->nr_reserved_highatomic),
53785380 K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)),
53795381 K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)),
53805382 K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)),
@@ -6711,7 +6713,7 @@ static void __meminit pgdat_init_internals(struct pglist_data *pgdat) · 1330 unmodified lines
67116713
67126714 pgdat_page_ext_init(pgdat);
67136715 spin_lock_init(&pgdat->lru_lock);
6714− lruvec_init(node_lruvec(pgdat));
6716+ lruvec_init(&pgdat->__lruvec);
67156717 }
67166718
67176719 static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid,
@@ -7988,6 +7990,15 @@ int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write, · 1270 unmodified lines
79887990 return 0;
79897991 }
79907992
7993+static void __zone_pcp_update(struct zone *zone)
7994+{
7995+ unsigned int cpu;
7996+
7997+ for_each_possible_cpu(cpu)
7998+ pageset_set_high_and_batch(zone,
7999+ per_cpu_ptr(zone->pageset, cpu));
8000+}
8001+
79918002 /*
79928003 * percpu_pagelist_fraction - changes the pcp->high for each zone on each
79938004 * cpu. It is the fraction of total pages in each zone that a hot per cpu
@@ -8019,13 +8030,8 @@ int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write, · 25 unmodified lines
80198030 if (percpu_pagelist_fraction == old_percpu_pagelist_fraction)
80208031 goto out;
80218032
8022− for_each_populated_zone(zone) {
8023− unsigned int cpu;
8024−
8025− for_each_possible_cpu(cpu)
8026− pageset_set_high_and_batch(zone,
8027− per_cpu_ptr(zone->pageset, cpu));
8028− }
8033+ for_each_populated_zone(zone)
8034+ __zone_pcp_update(zone);
80298035 out:
80308036 mutex_unlock(&pcp_batch_high_lock);
80318037 return ret;
@@ -8261,7 +8267,7 @@ bool has_unmovable_pages(struct zone *zone, struct page *page, int count, · 229 unmodified lines
82618267 * The HWPoisoned page may be not in buddy system, and
82628268 * page_count() is not 0.
82638269 */
8264− if ((flags & SKIP_HWPOISON) && PageHWPoison(page))
8270+ if ((flags & MEMORY_OFFLINE) && PageHWPoison(page))
82658271 continue;
82668272
82678273 if (__PageMovable(page))
@@ -8477,7 +8483,7 @@ int alloc_contig_range(unsigned long start, unsigned long end, · 209 unmodified lines
84778483 }
84788484
84798485 /* Make sure the range is really isolated. */
8480− if (test_pages_isolated(outer_start, end, false)) {
8486+ if (test_pages_isolated(outer_start, end, 0)) {
84818487 pr_info_ratelimited("%s: [%lx, %lx) PFNs busy\n",
84828488 __func__, outer_start, end);
84838489 ret = -EBUSY;
@@ -8502,6 +8508,107 @@ done: · 18 unmodified lines
85028508 pfn_max_align_up(end), migratetype);
85038509 return ret;
85048510 }
8511+
8512+static int __alloc_contig_pages(unsigned long start_pfn,
8513+ unsigned long nr_pages, gfp_t gfp_mask)
8514+{
8515+ unsigned long end_pfn = start_pfn + nr_pages;
8516+
8517+ return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE,
8518+ gfp_mask);
8519+}
8520+
8521+static bool pfn_range_valid_contig(struct zone *z, unsigned long start_pfn,
8522+ unsigned long nr_pages)
8523+{
8524+ unsigned long i, end_pfn = start_pfn + nr_pages;
8525+ struct page *page;
8526+
8527+ for (i = start_pfn; i < end_pfn; i++) {
8528+ page = pfn_to_online_page(i);
8529+ if (!page)
8530+ return false;
8531+
8532+ if (page_zone(page) != z)
8533+ return false;
8534+
8535+ if (PageReserved(page))
8536+ return false;
8537+
8538+ if (page_count(page) > 0)
8539+ return false;
8540+
8541+ if (PageHuge(page))
8542+ return false;
8543+ }
8544+ return true;
8545+}
8546+
8547+static bool zone_spans_last_pfn(const struct zone *zone,
8548+ unsigned long start_pfn, unsigned long nr_pages)
8549+{
8550+ unsigned long last_pfn = start_pfn + nr_pages - 1;
8551+
8552+ return zone_spans_pfn(zone, last_pfn);
8553+}
8554+
8555+/**
8556+ * alloc_contig_pages() -- tries to find and allocate contiguous range of pages
8557+ * @nr_pages: Number of contiguous pages to allocate
8558+ * @gfp_mask: GFP mask to limit search and used during compaction
8559+ * @nid: Target node
8560+ * @nodemask: Mask for other possible nodes
8561+ *
8562+ * This routine is a wrapper around alloc_contig_range(). It scans over zones
8563+ * on an applicable zonelist to find a contiguous pfn range which can then be
8564+ * tried for allocation with alloc_contig_range(). This routine is intended
8565+ * for allocation requests which can not be fulfilled with the buddy allocator.
8566+ *
8567+ * The allocated memory is always aligned to a page boundary. If nr_pages is a
8568+ * power of two then the alignment is guaranteed to be to the given nr_pages
8569+ * (e.g. 1GB request would be aligned to 1GB).
8570+ *
8571+ * Allocated pages can be freed with free_contig_range() or by manually calling
8572+ * __free_page() on each allocated page.
8573+ *
8574+ * Return: pointer to contiguous pages on success, or NULL if not successful.
8575+ */
8576+struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask,
8577+ int nid, nodemask_t *nodemask)
8578+{
8579+ unsigned long ret, pfn, flags;
8580+ struct zonelist *zonelist;
8581+ struct zone *zone;
8582+ struct zoneref *z;
8583+
8584+ zonelist = node_zonelist(nid, gfp_mask);
8585+ for_each_zone_zonelist_nodemask(zone, z, zonelist,
8586+ gfp_zone(gfp_mask), nodemask) {
8587+ spin_lock_irqsave(&zone->lock, flags);
8588+
8589+ pfn = ALIGN(zone->zone_start_pfn, nr_pages);
8590+ while (zone_spans_last_pfn(zone, pfn, nr_pages)) {
8591+ if (pfn_range_valid_contig(zone, pfn, nr_pages)) {
8592+ /*
8593+ * We release the zone lock here because
8594+ * alloc_contig_range() will also lock the zone
8595+ * at some point. If there's an allocation
8596+ * spinning on this lock, it may win the race
8597+ * and cause alloc_contig_range() to fail...
8598+ */
8599+ spin_unlock_irqrestore(&zone->lock, flags);
8600+ ret = __alloc_contig_pages(pfn, nr_pages,
8601+ gfp_mask);
8602+ if (!ret)
8603+ return pfn_to_page(pfn);
8604+ spin_lock_irqsave(&zone->lock, flags);
8605+ }
8606+ pfn += nr_pages;
8607+ }
8608+ spin_unlock_irqrestore(&zone->lock, flags);
8609+ }
8610+ return NULL;
8611+}
85058612 #endif /* CONFIG_CONTIG_ALLOC */
85068613
85078614 void free_contig_range(unsigned long pfn, unsigned int nr_pages)
@@ -8523,11 +8630,8 @@ void free_contig_range(unsigned long pfn, unsigned int nr_pages) · 15 unmodified lines
85238630 */
85248631 void __meminit zone_pcp_update(struct zone *zone)
85258632 {
8526− unsigned cpu;
85278633 mutex_lock(&pcp_batch_high_lock);
8528− for_each_possible_cpu(cpu)
8529− pageset_set_high_and_batch(zone,
8530− per_cpu_ptr(zone->pageset, cpu));
8634+ __zone_pcp_update(zone);
85318635 mutex_unlock(&pcp_batch_high_lock);
85328636 }
85338637
@@ -8560,7 +8664,7 @@ __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) · 26 unmodified lines
85608664 {
85618665 struct page *page;
85628666 struct zone *zone;
8563− unsigned int order, i;
8667+ unsigned int order;
85648668 unsigned long pfn;
85658669 unsigned long flags;
85668670 unsigned long offlined_pages = 0;
@@ -8588,7 +8692,6 @@ __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) · 21 unmodified lines
85888692 */
85898693 if (unlikely(!PageBuddy(page) && PageHWPoison(page))) {
85908694 pfn++;
8591− SetPageReserved(page);
85928695 offlined_pages++;
85938696 continue;
85948697 }
@@ -8602,8 +8705,6 @@ __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) · 7 unmodified lines
86028705 pfn, 1 << order, end_pfn);
86038706 #endif
86048707 del_page_from_free_area(page, &zone->free_area[order]);
8605− for (i = 0; i < (1 << order); i++)
8606− SetPageReserved((page+i));
86078708 pfn += (1 << order);
86088709 }
86098710 spin_unlock_irqrestore(&zone->lock, flags);
mm/page_io.c+13−2
@@ -22,6 +22,7 @@ · 21 unmodified lines
2222 #include <linux/writeback.h>
2323 #include <linux/frontswap.h>
2424 #include <linux/blkdev.h>
25+#include <linux/psi.h>
2526 #include <linux/uio.h>
2627 #include <linux/sched/task.h>
2728 #include <asm/pgtable.h>
@@ -354,10 +355,19 @@ int swap_readpage(struct page *page, bool synchronous) · 326 unmodified lines
354355 struct swap_info_struct *sis = page_swap_info(page);
355356 blk_qc_t qc;
356357 struct gendisk *disk;
358+ unsigned long pflags;
357359
358360 VM_BUG_ON_PAGE(!PageSwapCache(page) && !synchronous, page);
359361 VM_BUG_ON_PAGE(!PageLocked(page), page);
360362 VM_BUG_ON_PAGE(PageUptodate(page), page);
363+
364+ /*
365+ * Count submission time as memory stall. When the device is congested,
366+ * or the submitting cgroup IO-throttled, submission can be a
367+ * significant part of overall IO time.
368+ */
369+ psi_memstall_enter(&pflags);
370+
361371 if (frontswap_load(page) == 0) {
362372 SetPageUptodate(page);
363373 unlock_page(page);
@@ -371,7 +381,7 @@ int swap_readpage(struct page *page, bool synchronous) · 7 unmodified lines
371381 ret = mapping->a_ops->readpage(swap_file, page);
372382 if (!ret)
373383 count_vm_event(PSWPIN);
374− return ret;
384+ goto out;
375385 }
376386
377387 ret = bdev_read_page(sis->bdev, swap_page_sector(page), page);
@@ -382,7 +392,7 @@ int swap_readpage(struct page *page, bool synchronous) · 4 unmodified lines
382392 }
383393
384394 count_vm_event(PSWPIN);
385− return 0;
395+ goto out;
386396 }
387397
388398 ret = 0;
@@ -418,6 +428,7 @@ int swap_readpage(struct page *page, bool synchronous) · 29 unmodified lines
418428 bio_put(bio);
419429
420430 out:
431+ psi_memstall_leave(&pflags);
421432 return ret;
422433 }
423434
mm/page_isolation.c+6−6
@@ -168,7 +168,8 @@ __first_valid_page(unsigned long pfn, unsigned long nr_pages) · 167 unmodified lines
168168 * @migratetype: Migrate type to set in error recovery.
169169 * @flags: The following flags are allowed (they can be combined in
170170 * a bit mask)
171− * SKIP_HWPOISON - ignore hwpoison pages
171+ * MEMORY_OFFLINE - isolate to offline (!allocate) memory
172+ * e.g., skip over PageHWPoison() pages
172173 * REPORT_FAILURE - report details about the failure to
173174 * isolate the range
174175 *
@@ -257,7 +258,7 @@ void undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, · 82 unmodified lines
257258 */
258259 static unsigned long
259260 __test_page_isolated_in_pageblock(unsigned long pfn, unsigned long end_pfn,
260− bool skip_hwpoisoned_pages)
261+ int flags)
261262 {
262263 struct page *page;
263264
@@ -274,7 +275,7 @@ __test_page_isolated_in_pageblock(unsigned long pfn, unsigned long end_pfn, · 10 unmodified lines
274275 * simple way to verify that as VM_BUG_ON(), though.
275276 */
276277 pfn += 1 << page_order(page);
277− else if (skip_hwpoisoned_pages && PageHWPoison(page))
278+ else if ((flags & MEMORY_OFFLINE) && PageHWPoison(page))
278279 /* A HWPoisoned page cannot be also PageBuddy */
279280 pfn++;
280281 else
@@ -286,7 +287,7 @@ __test_page_isolated_in_pageblock(unsigned long pfn, unsigned long end_pfn, · 5 unmodified lines
286287
287288 /* Caller should ensure that requested range is in a single zone */
288289 int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn,
289− bool skip_hwpoisoned_pages)
290+ int isol_flags)
290291 {
291292 unsigned long pfn, flags;
292293 struct page *page;
@@ -308,8 +309,7 @@ int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn, · 15 unmodified lines
308309 /* Check all pages are free or marked as ISOLATED */
309310 zone = page_zone(page);
310311 spin_lock_irqsave(&zone->lock, flags);
311− pfn = __test_page_isolated_in_pageblock(start_pfn, end_pfn,
312− skip_hwpoisoned_pages);
312+ pfn = __test_page_isolated_in_pageblock(start_pfn, end_pfn, isol_flags);
313313 spin_unlock_irqrestore(&zone->lock, flags);
314314
315315 trace_test_pages_isolated(start_pfn, end_pfn, pfn);
mm/pgtable-generic.c+9−0
@@ -24,18 +24,27 @@ void pgd_clear_bad(pgd_t *pgd) · 23 unmodified lines
2424 pgd_clear(pgd);
2525 }
2626
27+#ifndef __PAGETABLE_P4D_FOLDED
2728 void p4d_clear_bad(p4d_t *p4d)
2829 {
2930 p4d_ERROR(*p4d);
3031 p4d_clear(p4d);
3132 }
33+#endif
3234
35+#ifndef __PAGETABLE_PUD_FOLDED
3336 void pud_clear_bad(pud_t *pud)
3437 {
3538 pud_ERROR(*pud);
3639 pud_clear(pud);
3740 }
41+#endif
3842
43+/*
44+ * Note that the pmd variant below can't be stub'ed out just as for p4d/pud
45+ * above. pmd folding is special and typically pmd_* macros refer to upper
46+ * level even when folded
47+ */
3948 void pmd_clear_bad(pmd_t *pmd)
4049 {
4150 pmd_ERROR(*pmd);
mm/rmap.c+45−20
@@ -251,18 +251,37 @@ static inline void unlock_anon_vma_root(struct anon_vma *root) · 250 unmodified lines
251251 * Attach the anon_vmas from src to dst.
252252 * Returns 0 on success, -ENOMEM on failure.
253253 *
254− * If dst->anon_vma is NULL this function tries to find and reuse existing
255− * anon_vma which has no vmas and only one child anon_vma. This prevents
256− * degradation of anon_vma hierarchy to endless linear chain in case of
257− * constantly forking task. On the other hand, an anon_vma with more than one
258− * child isn't reused even if there was no alive vma, thus rmap walker has a
259− * good chance of avoiding scanning the whole hierarchy when it searches where
260− * page is mapped.
254+ * anon_vma_clone() is called by __vma_split(), __split_vma(), copy_vma() and
255+ * anon_vma_fork(). The first three want an exact copy of src, while the last
256+ * one, anon_vma_fork(), may try to reuse an existing anon_vma to prevent
257+ * endless growth of anon_vma. Since dst->anon_vma is set to NULL before call,
258+ * we can identify this case by checking (!dst->anon_vma && src->anon_vma).
259+ *
260+ * If (!dst->anon_vma && src->anon_vma) is true, this function tries to find
261+ * and reuse existing anon_vma which has no vmas and only one child anon_vma.
262+ * This prevents degradation of anon_vma hierarchy to endless linear chain in
263+ * case of constantly forking task. On the other hand, an anon_vma with more
264+ * than one child isn't reused even if there was no alive vma, thus rmap
265+ * walker has a good chance of avoiding scanning the whole hierarchy when it
266+ * searches where page is mapped.
261267 */
262268 int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src)
263269 {
264270 struct anon_vma_chain *avc, *pavc;
265271 struct anon_vma *root = NULL;
272+ struct vm_area_struct *prev = dst->vm_prev, *pprev = src->vm_prev;
273+
274+ /*
275+ * If parent share anon_vma with its vm_prev, keep this sharing in in
276+ * child.
277+ *
278+ * 1. Parent has vm_prev, which implies we have vm_prev.
279+ * 2. Parent and its vm_prev have the same anon_vma.
280+ */
281+ if (!dst->anon_vma && src->anon_vma &&
282+ pprev && pprev->anon_vma == src->anon_vma)
283+ dst->anon_vma = prev->anon_vma;
284+
266285
267286 list_for_each_entry_reverse(pavc, &src->anon_vma_chain, same_vma) {
268287 struct anon_vma *anon_vma;
@@ -287,8 +306,8 @@ int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src) · 18 unmodified lines
287306 * will always reuse it. Root anon_vma is never reused:
288307 * it has self-parent reference and at least one child.
289308 */
290− if (!dst->anon_vma && anon_vma != src->anon_vma &&
291− anon_vma->degree < 2)
309+ if (!dst->anon_vma && src->anon_vma &&
310+ anon_vma != src->anon_vma && anon_vma->degree < 2)
292311 dst->anon_vma = anon_vma;
293312 }
294313 if (dst->anon_vma)
@@ -458,9 +477,10 @@ void __init anon_vma_init(void) · 163 unmodified lines
458477 * chain and verify that the page in question is indeed mapped in it
459478 * [ something equivalent to page_mapped_in_vma() ].
460479 *
461− * Since anon_vma's slab is DESTROY_BY_RCU and we know from page_remove_rmap()
462− * that the anon_vma pointer from page->mapping is valid if there is a
463− * mapcount, we can dereference the anon_vma after observing those.
480+ * Since anon_vma's slab is SLAB_TYPESAFE_BY_RCU and we know from
481+ * page_remove_rmap() that the anon_vma pointer from page->mapping is valid
482+ * if there is a mapcount, we can dereference the anon_vma after observing
483+ * those.
464484 */
465485 struct anon_vma *page_get_anon_vma(struct page *page)
466486 {
@@ -1055,7 +1075,6 @@ static void __page_set_anon_rmap(struct page *page, · 588 unmodified lines
10551075 static void __page_check_anon_rmap(struct page *page,
10561076 struct vm_area_struct *vma, unsigned long address)
10571077 {
1058−#ifdef CONFIG_DEBUG_VM
10591078 /*
10601079 * The page's anon-rmap details (mapping and index) are guaranteed to
10611080 * be set up correctly at this point.
@@ -1068,9 +1087,9 @@ static void __page_check_anon_rmap(struct page *page, · 6 unmodified lines
10681087 * are initially only visible via the pagetables, and the pte is locked
10691088 * over the call to page_add_new_anon_rmap.
10701089 */
1071− BUG_ON(page_anon_vma(page)->root != vma->anon_vma->root);
1072− BUG_ON(page_to_pgoff(page) != linear_page_index(vma, address));
1073−#endif
1090+ VM_BUG_ON_PAGE(page_anon_vma(page)->root != vma->anon_vma->root, page);
1091+ VM_BUG_ON_PAGE(page_to_pgoff(page) != linear_page_index(vma, address),
1092+ page);
10741093 }
10751094
10761095 /**
@@ -1273,12 +1292,20 @@ static void page_remove_anon_compound_rmap(struct page *page) · 196 unmodified lines
12731292 if (TestClearPageDoubleMap(page)) {
12741293 /*
12751294 * Subpages can be mapped with PTEs too. Check how many of
1276− * themi are still mapped.
1295+ * them are still mapped.
12771296 */
12781297 for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) {
12791298 if (atomic_add_negative(-1, &page[i]._mapcount))
12801299 nr++;
12811300 }
1301+
1302+ /*
1303+ * Queue the page for deferred split if at least one small
1304+ * page of the compound page is unmapped, but at least one
1305+ * small page is still mapped.
1306+ */
1307+ if (nr && nr < HPAGE_PMD_NR)
1308+ deferred_split_huge_page(page);
12821309 } else {
12831310 nr = HPAGE_PMD_NR;
12841311 }
@@ -1286,10 +1313,8 @@ static void page_remove_anon_compound_rmap(struct page *page) · 1 unmodified line
12861313 if (unlikely(PageMlocked(page)))
12871314 clear_page_mlock(page);
12881315
1289− if (nr) {
1316+ if (nr)
12901317 __mod_node_page_state(page_pgdat(page), NR_ANON_MAPPED, -nr);
1291− deferred_split_huge_page(page);
1292− }
12931318 }
12941319
12951320 /**
mm/shmem.c+17−12
@@ -1369,7 +1369,8 @@ static int shmem_writepage(struct page *page, struct writeback_control *wbc) · 1368 unmodified lines
13691369 if (list_empty(&info->swaplist))
13701370 list_add(&info->swaplist, &shmem_swaplist);
13711371
1372− if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
1372+ if (add_to_swap_cache(page, swap,
1373+ __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN) == 0) {
13731374 spin_lock_irq(&info->lock);
13741375 shmem_recalc_inode(inode);
13751376 info->swapped++;
@@ -2022,16 +2023,14 @@ static vm_fault_t shmem_fault(struct vm_fault *vmf) · 646 unmodified lines
20222023 shmem_falloc->waitq &&
20232024 vmf->pgoff >= shmem_falloc->start &&
20242025 vmf->pgoff < shmem_falloc->next) {
2026+ struct file *fpin;
20252027 wait_queue_head_t *shmem_falloc_waitq;
20262028 DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function);
20272029
20282030 ret = VM_FAULT_NOPAGE;
2029− if ((vmf->flags & FAULT_FLAG_ALLOW_RETRY) &&
2030− !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) {
2031− /* It's polite to up mmap_sem if we can */
2032− up_read(&vma->vm_mm->mmap_sem);
2031+ fpin = maybe_unlock_mmap_for_io(vmf, NULL);
2032+ if (fpin)
20332033 ret = VM_FAULT_RETRY;
2034− }
20352034
20362035 shmem_falloc_waitq = shmem_falloc->waitq;
20372036 prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait,
@@ -2049,6 +2048,9 @@ static vm_fault_t shmem_fault(struct vm_fault *vmf) · 11 unmodified lines
20492048 spin_lock(&inode->i_lock);
20502049 finish_wait(shmem_falloc_waitq, &shmem_fault_wait);
20512050 spin_unlock(&inode->i_lock);
2051+
2052+ if (fpin)
2053+ fput(fpin);
20522054 return ret;
20532055 }
20542056 spin_unlock(&inode->i_lock);
@@ -2213,11 +2215,14 @@ static int shmem_mmap(struct file *file, struct vm_area_struct *vma) · 158 unmodified lines
22132215 return -EPERM;
22142216
22152217 /*
2216− * Since the F_SEAL_FUTURE_WRITE seals allow for a MAP_SHARED
2217− * read-only mapping, take care to not allow mprotect to revert
2218− * protections.
2218+ * Since an F_SEAL_FUTURE_WRITE sealed memfd can be mapped as
2219+ * MAP_SHARED and read-only, take care to not allow mprotect to
2220+ * revert protections on such mappings. Do this only for shared
2221+ * mappings. For private mappings, don't need to mask
2222+ * VM_MAYWRITE as we still want them to be COW-writable.
22192223 */
2220− vma->vm_flags &= ~(VM_MAYWRITE);
2224+ if (vma->vm_flags & VM_SHARED)
2225+ vma->vm_flags &= ~(VM_MAYWRITE);
22212226 }
22222227
22232228 file_accessed(file);
@@ -2742,7 +2747,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, · 518 unmodified lines
27422747 }
27432748
27442749 shmem_falloc.waitq = &shmem_falloc_waitq;
2745− shmem_falloc.start = unmap_start >> PAGE_SHIFT;
2750+ shmem_falloc.start = (u64)unmap_start >> PAGE_SHIFT;
27462751 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT;
27472752 spin_lock(&inode->i_lock);
27482753 inode->i_private = &shmem_falloc;
@@ -3928,7 +3933,7 @@ out2: · 1179 unmodified lines
39283933 static ssize_t shmem_enabled_show(struct kobject *kobj,
39293934 struct kobj_attribute *attr, char *buf)
39303935 {
3931− int values[] = {
3936+ static const int values[] = {
39323937 SHMEM_HUGE_ALWAYS,
39333938 SHMEM_HUGE_WITHIN_SIZE,
39343939 SHMEM_HUGE_ADVISE,
mm/slab.c+4−3
@@ -1247,9 +1247,10 @@ void __init kmem_cache_init(void) · 1246 unmodified lines
12471247 * structures first. Without this, further allocations will bug.
12481248 */
12491249 kmalloc_caches[KMALLOC_NORMAL][INDEX_NODE] = create_kmalloc_cache(
1250− kmalloc_info[INDEX_NODE].name,
1251− kmalloc_size(INDEX_NODE), ARCH_KMALLOC_FLAGS,
1252− 0, kmalloc_size(INDEX_NODE));
1250+ kmalloc_info[INDEX_NODE].name[KMALLOC_NORMAL],
1251+ kmalloc_info[INDEX_NODE].size,
1252+ ARCH_KMALLOC_FLAGS, 0,
1253+ kmalloc_info[INDEX_NODE].size);
12531254 slab_state = PARTIAL_NODE;
12541255 setup_kmalloc_cache_index_table();
12551256
mm/slab.h+3−3
@@ -139,7 +139,7 @@ extern struct kmem_cache *kmem_cache; · 138 unmodified lines
139139
140140 /* A table of kmalloc cache names and sizes */
141141 extern const struct kmalloc_info_struct {
142− const char *name;
142+ const char *name[NR_KMALLOC_TYPES];
143143 unsigned int size;
144144 } kmalloc_info[];
145145
@@ -369,7 +369,7 @@ static __always_inline int memcg_charge_slab(struct page *page, · 223 unmodified lines
369369 if (ret)
370370 goto out;
371371
372− lruvec = mem_cgroup_lruvec(page_pgdat(page), memcg);
372+ lruvec = mem_cgroup_lruvec(memcg, page_pgdat(page));
373373 mod_lruvec_state(lruvec, cache_vmstat_idx(s), 1 << order);
374374
375375 /* transer try_charge() page references to kmem_cache */
@@ -393,7 +393,7 @@ static __always_inline void memcg_uncharge_slab(struct page *page, int order, · 17 unmodified lines
393393 rcu_read_lock();
394394 memcg = READ_ONCE(s->memcg_params.memcg);
395395 if (likely(!mem_cgroup_is_root(memcg))) {
396− lruvec = mem_cgroup_lruvec(page_pgdat(page), memcg);
396+ lruvec = mem_cgroup_lruvec(memcg, page_pgdat(page));
397397 mod_lruvec_state(lruvec, cache_vmstat_idx(s), -(1 << order));
398398 memcg_kmem_uncharge_memcg(page, order, memcg);
399399 } else {
mm/slab_common.c+53−46
@@ -1139,26 +1139,56 @@ struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags) · 1138 unmodified lines
11391139 return kmalloc_caches[kmalloc_type(flags)][index];
11401140 }
11411141
1142+#ifdef CONFIG_ZONE_DMA
1143+#define INIT_KMALLOC_INFO(__size, __short_size) \
1144+{ \
1145+ .name[KMALLOC_NORMAL] = "kmalloc-" #__short_size, \
1146+ .name[KMALLOC_RECLAIM] = "kmalloc-rcl-" #__short_size, \
1147+ .name[KMALLOC_DMA] = "dma-kmalloc-" #__short_size, \
1148+ .size = __size, \
1149+}
1150+#else
1151+#define INIT_KMALLOC_INFO(__size, __short_size) \
1152+{ \
1153+ .name[KMALLOC_NORMAL] = "kmalloc-" #__short_size, \
1154+ .name[KMALLOC_RECLAIM] = "kmalloc-rcl-" #__short_size, \
1155+ .size = __size, \
1156+}
1157+#endif
1158+
11421159 /*
11431160 * kmalloc_info[] is to make slub_debug=,kmalloc-xx option work at boot time.
11441161 * kmalloc_index() supports up to 2^26=64MB, so the final entry of the table is
11451162 * kmalloc-67108864.
11461163 */
11471164 const struct kmalloc_info_struct kmalloc_info[] __initconst = {
1148− {NULL, 0}, {"kmalloc-96", 96},
1149− {"kmalloc-192", 192}, {"kmalloc-8", 8},
1150− {"kmalloc-16", 16}, {"kmalloc-32", 32},
1151− {"kmalloc-64", 64}, {"kmalloc-128", 128},
1152− {"kmalloc-256", 256}, {"kmalloc-512", 512},
1153− {"kmalloc-1k", 1024}, {"kmalloc-2k", 2048},
1154− {"kmalloc-4k", 4096}, {"kmalloc-8k", 8192},
1155− {"kmalloc-16k", 16384}, {"kmalloc-32k", 32768},
1156− {"kmalloc-64k", 65536}, {"kmalloc-128k", 131072},
1157− {"kmalloc-256k", 262144}, {"kmalloc-512k", 524288},
1158− {"kmalloc-1M", 1048576}, {"kmalloc-2M", 2097152},
1159− {"kmalloc-4M", 4194304}, {"kmalloc-8M", 8388608},
1160− {"kmalloc-16M", 16777216}, {"kmalloc-32M", 33554432},
1161− {"kmalloc-64M", 67108864}
1165+ INIT_KMALLOC_INFO(0, 0),
1166+ INIT_KMALLOC_INFO(96, 96),
1167+ INIT_KMALLOC_INFO(192, 192),
1168+ INIT_KMALLOC_INFO(8, 8),
1169+ INIT_KMALLOC_INFO(16, 16),
1170+ INIT_KMALLOC_INFO(32, 32),
1171+ INIT_KMALLOC_INFO(64, 64),
1172+ INIT_KMALLOC_INFO(128, 128),
1173+ INIT_KMALLOC_INFO(256, 256),
1174+ INIT_KMALLOC_INFO(512, 512),
1175+ INIT_KMALLOC_INFO(1024, 1k),
1176+ INIT_KMALLOC_INFO(2048, 2k),
1177+ INIT_KMALLOC_INFO(4096, 4k),
1178+ INIT_KMALLOC_INFO(8192, 8k),
1179+ INIT_KMALLOC_INFO(16384, 16k),
1180+ INIT_KMALLOC_INFO(32768, 32k),
1181+ INIT_KMALLOC_INFO(65536, 64k),
1182+ INIT_KMALLOC_INFO(131072, 128k),
1183+ INIT_KMALLOC_INFO(262144, 256k),
1184+ INIT_KMALLOC_INFO(524288, 512k),
1185+ INIT_KMALLOC_INFO(1048576, 1M),
1186+ INIT_KMALLOC_INFO(2097152, 2M),
1187+ INIT_KMALLOC_INFO(4194304, 4M),
1188+ INIT_KMALLOC_INFO(8388608, 8M),
1189+ INIT_KMALLOC_INFO(16777216, 16M),
1190+ INIT_KMALLOC_INFO(33554432, 32M),
1191+ INIT_KMALLOC_INFO(67108864, 64M)
11621192 };
11631193
11641194 /*
@@ -1208,36 +1238,14 @@ void __init setup_kmalloc_cache_index_table(void) · 43 unmodified lines
12081238 }
12091239 }
12101240
1211−static const char *
1212−kmalloc_cache_name(const char *prefix, unsigned int size)
1213−{
1214−
1215− static const char units[3] = "\0kM";
1216− int idx = 0;
1217−
1218− while (size >= 1024 && (size % 1024 == 0)) {
1219− size /= 1024;
1220− idx++;
1221− }
1222−
1223− return kasprintf(GFP_NOWAIT, "%s-%u%c", prefix, size, units[idx]);
1224−}
1225−
12261241 static void __init
1227−new_kmalloc_cache(int idx, int type, slab_flags_t flags)
1242+new_kmalloc_cache(int idx, enum kmalloc_cache_type type, slab_flags_t flags)
12281243 {
1229− const char *name;
1230−
1231− if (type == KMALLOC_RECLAIM) {
1244+ if (type == KMALLOC_RECLAIM)
12321245 flags |= SLAB_RECLAIM_ACCOUNT;
1233− name = kmalloc_cache_name("kmalloc-rcl",
1234− kmalloc_info[idx].size);
1235− BUG_ON(!name);
1236− } else {
1237− name = kmalloc_info[idx].name;
1238− }
12391246
1240− kmalloc_caches[type][idx] = create_kmalloc_cache(name,
1247+ kmalloc_caches[type][idx] = create_kmalloc_cache(
1248+ kmalloc_info[idx].name[type],
12411249 kmalloc_info[idx].size, flags, 0,
12421250 kmalloc_info[idx].size);
12431251 }
@@ -1249,7 +1257,8 @@ new_kmalloc_cache(int idx, int type, slab_flags_t flags) · 5 unmodified lines
12491257 */
12501258 void __init create_kmalloc_caches(slab_flags_t flags)
12511259 {
1252− int i, type;
1260+ int i;
1261+ enum kmalloc_cache_type type;
12531262
12541263 for (type = KMALLOC_NORMAL; type <= KMALLOC_RECLAIM; type++) {
12551264 for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
@@ -1278,12 +1287,10 @@ void __init create_kmalloc_caches(slab_flags_t flags) · 22 unmodified lines
12781287 struct kmem_cache *s = kmalloc_caches[KMALLOC_NORMAL][i];
12791288
12801289 if (s) {
1281− unsigned int size = kmalloc_size(i);
1282− const char *n = kmalloc_cache_name("dma-kmalloc", size);
1283−
1284− BUG_ON(!n);
12851290 kmalloc_caches[KMALLOC_DMA][i] = create_kmalloc_cache(
1286− n, size, SLAB_CACHE_DMA | flags, 0, 0);
1291+ kmalloc_info[i].name[KMALLOC_DMA],
1292+ kmalloc_info[i].size,
1293+ SLAB_CACHE_DMA | flags, 0, 0);
12871294 }
12881295 }
12891296 #endif
mm/slub.c+16−20
@@ -93,9 +93,7 @@ · 92 unmodified lines
9393 * minimal so we rely on the page allocators per cpu caches for
9494 * fast frees and allocs.
9595 *
96− * Overloading of page flags that are otherwise used for LRU management.
97− *
98− * PageActive The slab is frozen and exempt from list processing.
96+ * page->frozen The slab is frozen and exempt from list processing.
9997 * This means that the slab is dedicated to a purpose
10098 * such as satisfying allocations for a specific
10199 * processor. Objects may be freed in the slab while
@@ -111,7 +109,7 @@ · 9 unmodified lines
111109 * free objects in addition to the regular freelist
112110 * that requires the slab lock.
113111 *
114− * PageError Slab requires special handling due to debug
112+ * SLAB_DEBUG_FLAGS Slab requires special handling due to debug
115113 * options set. This moves slab handling out of
116114 * the fast path and disables lockless freelists.
117115 */
@@ -736,6 +734,7 @@ static int check_bytes_and_report(struct kmem_cache *s, struct page *page, · 618 unmodified lines
736734 {
737735 u8 *fault;
738736 u8 *end;
737+ u8 *addr = page_address(page);
739738
740739 metadata_access_enable();
741740 fault = memchr_inv(start, value, bytes);
@@ -748,8 +747,9 @@ static int check_bytes_and_report(struct kmem_cache *s, struct page *page, · 6 unmodified lines
748747 end--;
749748
750749 slab_bug(s, "%s overwritten", what);
751− pr_err("INFO: 0x%p-0x%p. First byte 0x%x instead of 0x%x\n",
752− fault, end - 1, fault[0], value);
750+ pr_err("INFO: 0x%p-0x%p @offset=%tu. First byte 0x%x instead of 0x%x\n",
751+ fault, end - 1, fault - addr,
752+ fault[0], value);
753753 print_trailer(s, page, object);
754754
755755 restore_bytes(s, what, value, fault, end);
@@ -844,7 +844,8 @@ static int slab_pad_check(struct kmem_cache *s, struct page *page) · 88 unmodified lines
844844 while (end > fault && end[-1] == POISON_INUSE)
845845 end--;
846846
847− slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1);
847+ slab_err(s, page, "Padding overwritten. 0x%p-0x%p @offset=%tu",
848+ fault, end - 1, fault - start);
848849 print_section(KERN_ERR, "Padding ", pad, remainder);
849850
850851 restore_bytes(s, "slab padding", POISON_INUSE, fault, end);
@@ -4383,31 +4384,26 @@ static int count_total(struct page *page) · 3532 unmodified lines
43834384 #endif
43844385
43854386 #ifdef CONFIG_SLUB_DEBUG
4386−static int validate_slab(struct kmem_cache *s, struct page *page,
4387+static void validate_slab(struct kmem_cache *s, struct page *page,
43874388 unsigned long *map)
43884389 {
43894390 void *p;
43904391 void *addr = page_address(page);
43914392
4392− if (!check_slab(s, page) ||
4393− !on_freelist(s, page, NULL))
4394− return 0;
4393+ if (!check_slab(s, page) || !on_freelist(s, page, NULL))
4394+ return;
43954395
43964396 /* Now we know that a valid freelist exists */
43974397 bitmap_zero(map, page->objects);
43984398
43994399 get_map(s, page, map);
44004400 for_each_object(p, s, addr, page->objects) {
4401− if (test_bit(slab_index(p, s, addr), map))
4402− if (!check_object(s, page, p, SLUB_RED_INACTIVE))
4403− return 0;
4404− }
4401+ u8 val = test_bit(slab_index(p, s, addr), map) ?
4402+ SLUB_RED_INACTIVE : SLUB_RED_ACTIVE;
44054403
4406− for_each_object(p, s, addr, page->objects)
4407− if (!test_bit(slab_index(p, s, addr), map))
4408− if (!check_object(s, page, p, SLUB_RED_ACTIVE))
4409− return 0;
4410− return 1;
4404+ if (!check_object(s, page, p, val))
4405+ break;
4406+ }
44114407 }
44124408
44134409 static void validate_slab_slab(struct kmem_cache *s, struct page *page,
mm/sparse.c+10−8
@@ -458,8 +458,7 @@ struct page __init *__populate_section_memmap(unsigned long pfn, · 457 unmodified lines
458458 if (map)
459459 return map;
460460
461− map = memblock_alloc_try_nid(size,
462− PAGE_SIZE, addr,
461+ map = memblock_alloc_try_nid_raw(size, size, addr,
463462 MEMBLOCK_ALLOC_ACCESSIBLE, nid);
464463 if (!map)
465464 panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa\n",
@@ -482,10 +481,13 @@ static void __init sparse_buffer_init(unsigned long size, int nid) · 16 unmodified lines
482481 {
483482 phys_addr_t addr = __pa(MAX_DMA_ADDRESS);
484483 WARN_ON(sparsemap_buf); /* forgot to call sparse_buffer_fini()? */
485− sparsemap_buf =
486− memblock_alloc_try_nid_raw(size, PAGE_SIZE,
487− addr,
488− MEMBLOCK_ALLOC_ACCESSIBLE, nid);
484+ /*
485+ * Pre-allocated buffer is mainly used by __populate_section_memmap
486+ * and we want it to be properly aligned to the section size - this is
487+ * especially the case for VMEMMAP which maps memmap to PMDs
488+ */
489+ sparsemap_buf = memblock_alloc_exact_nid_raw(size, section_map_size(),
490+ addr, MEMBLOCK_ALLOC_ACCESSIBLE, nid);
489491 sparsemap_buf_end = sparsemap_buf + size;
490492 }
491493
@@ -647,7 +649,7 @@ void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn) · 155 unmodified lines
647649 #endif
648650
649651 #ifdef CONFIG_SPARSEMEM_VMEMMAP
650−static struct page *populate_section_memmap(unsigned long pfn,
652+static struct page * __meminit populate_section_memmap(unsigned long pfn,
651653 unsigned long nr_pages, int nid, struct vmem_altmap *altmap)
652654 {
653655 return __populate_section_memmap(pfn, nr_pages, nid, altmap);
@@ -669,7 +671,7 @@ static void free_map_bootmem(struct page *memmap) · 15 unmodified lines
669671 vmemmap_free(start, end, NULL);
670672 }
671673 #else
672−struct page *populate_section_memmap(unsigned long pfn,
674+struct page * __meminit populate_section_memmap(unsigned long pfn,
673675 unsigned long nr_pages, int nid, struct vmem_altmap *altmap)
674676 {
675677 struct page *page, *ret;
mm/swap.c+24−5
@@ -373,9 +373,16 @@ static void __lru_cache_activate_page(struct page *page) · 372 unmodified lines
373373 void mark_page_accessed(struct page *page)
374374 {
375375 page = compound_head(page);
376− if (!PageActive(page) && !PageUnevictable(page) &&
377− PageReferenced(page)) {
378376
377+ if (!PageReferenced(page)) {
378+ SetPageReferenced(page);
379+ } else if (PageUnevictable(page)) {
380+ /*
381+ * Unevictable pages are on the "LRU_UNEVICTABLE" list. But,
382+ * this list is never rotated or maintained, so marking an
383+ * evictable page accessed has no effect.
384+ */
385+ } else if (!PageActive(page)) {
379386 /*
380387 * If the page is on the LRU, queue it for activation via
381388 * activate_page_pvecs. Otherwise, assume the page is on a
@@ -389,8 +396,6 @@ void mark_page_accessed(struct page *page) · 7 unmodified lines
389396 ClearPageReferenced(page);
390397 if (page_is_file_cache(page))
391398 workingset_activation(page);
392− } else if (!PageReferenced(page)) {
393− SetPageReferenced(page);
394399 }
395400 if (page_is_idle(page))
396401 clear_page_idle(page);
@@ -708,9 +713,10 @@ static void lru_add_drain_per_cpu(struct work_struct *dummy) · 311 unmodified lines
708713 */
709714 void lru_add_drain_all(void)
710715 {
716+ static seqcount_t seqcount = SEQCNT_ZERO(seqcount);
711717 static DEFINE_MUTEX(lock);
712718 static struct cpumask has_work;
713− int cpu;
719+ int cpu, seq;
714720
715721 /*
716722 * Make sure nobody triggers this path before mm_percpu_wq is fully
@@ -719,7 +725,19 @@ void lru_add_drain_all(void) · 2 unmodified lines
719725 if (WARN_ON(!mm_percpu_wq))
720726 return;
721727
728+ seq = raw_read_seqcount_latch(&seqcount);
729+
722730 mutex_lock(&lock);
731+
732+ /*
733+ * Piggyback on drain started and finished while we waited for lock:
734+ * all pages pended at the time of our enter were drained from vectors.
735+ */
736+ if (__read_seqcount_retry(&seqcount, seq))
737+ goto done;
738+
739+ raw_write_seqcount_latch(&seqcount);
740+
723741 cpumask_clear(&has_work);
724742
725743 for_each_online_cpu(cpu) {
@@ -740,6 +758,7 @@ void lru_add_drain_all(void) · 14 unmodified lines
740758 for_each_cpu(cpu, &has_work)
741759 flush_work(&per_cpu(lru_add_drain_work, cpu));
742760
761+done:
743762 mutex_unlock(&lock);
744763 }
745764 #else
mm/swapfile.c+7−0
@@ -2887,6 +2887,13 @@ static int claim_swapfile(struct swap_info_struct *p, struct inode *inode) · 2886 unmodified lines
28872887 error = set_blocksize(p->bdev, PAGE_SIZE);
28882888 if (error < 0)
28892889 return error;
2890+ /*
2891+ * Zoned block devices contain zones that have a sequential
2892+ * write only restriction. Hence zoned block devices are not
2893+ * suitable for swapping. Disallow them here.
2894+ */
2895+ if (blk_queue_is_zoned(p->bdev->bd_queue))
2896+ return -EINVAL;
28902897 p->flags |= SWP_BLKDEV;
28912898 } else if (S_ISREG(inode->i_mode)) {
28922899 p->bdev = inode->i_sb->s_bdev;
mm/userfaultfd.c+37−36
@@ -18,6 +18,36 @@ · 17 unmodified lines
1818 #include <asm/tlbflush.h>
1919 #include "internal.h"
2020
21+static __always_inline
22+struct vm_area_struct *find_dst_vma(struct mm_struct *dst_mm,
23+ unsigned long dst_start,
24+ unsigned long len)
25+{
26+ /*
27+ * Make sure that the dst range is both valid and fully within a
28+ * single existing vma.
29+ */
30+ struct vm_area_struct *dst_vma;
31+
32+ dst_vma = find_vma(dst_mm, dst_start);
33+ if (!dst_vma)
34+ return NULL;
35+
36+ if (dst_start < dst_vma->vm_start ||
37+ dst_start + len > dst_vma->vm_end)
38+ return NULL;
39+
40+ /*
41+ * Check the vma is registered in uffd, this is required to
42+ * enforce the VM_MAYWRITE check done at uffd registration
43+ * time.
44+ */
45+ if (!dst_vma->vm_userfaultfd_ctx.ctx)
46+ return NULL;
47+
48+ return dst_vma;
49+}
50+
2151 static int mcopy_atomic_pte(struct mm_struct *dst_mm,
2252 pmd_t *dst_pmd,
2353 struct vm_area_struct *dst_vma,
@@ -60,7 +90,7 @@ static int mcopy_atomic_pte(struct mm_struct *dst_mm, · 36 unmodified lines
6090
6191 /*
6292 * The memory barrier inside __SetPageUptodate makes sure that
63− * preceeding stores to the page contents become visible before
93+ * preceding stores to the page contents become visible before
6494 * the set_pte_at() write.
6595 */
6696 __SetPageUptodate(page);
@@ -184,7 +214,6 @@ static __always_inline ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm, · 117 unmodified lines
184214 unsigned long src_addr, dst_addr;
185215 long copied;
186216 struct page *page;
187− struct hstate *h;
188217 unsigned long vma_hpagesize;
189218 pgoff_t idx;
190219 u32 hash;
@@ -221,20 +250,9 @@ retry: · 30 unmodified lines
221250 */
222251 if (!dst_vma) {
223252 err = -ENOENT;
224− dst_vma = find_vma(dst_mm, dst_start);
253+ dst_vma = find_dst_vma(dst_mm, dst_start, len);
225254 if (!dst_vma || !is_vm_hugetlb_page(dst_vma))
226255 goto out_unlock;
227− /*
228− * Check the vma is registered in uffd, this is
229− * required to enforce the VM_MAYWRITE check done at
230− * uffd registration time.
231− */
232− if (!dst_vma->vm_userfaultfd_ctx.ctx)
233− goto out_unlock;
234−
235− if (dst_start < dst_vma->vm_start ||
236− dst_start + len > dst_vma->vm_end)
237− goto out_unlock;
238256
239257 err = -EINVAL;
240258 if (vma_hpagesize != vma_kernel_pagesize(dst_vma))
@@ -243,10 +261,6 @@ retry: · 2 unmodified lines
243261 vm_shared = dst_vma->vm_flags & VM_SHARED;
244262 }
245263
246− if (WARN_ON(dst_addr & (vma_hpagesize - 1) ||
247− (len - copied) & (vma_hpagesize - 1)))
248− goto out_unlock;
249−
250264 /*
251265 * If not shared, ensure the dst_vma has a anon_vma.
252266 */
@@ -256,24 +270,21 @@ retry: · 3 unmodified lines
256270 goto out_unlock;
257271 }
258272
259− h = hstate_vma(dst_vma);
260−
261273 while (src_addr < src_start + len) {
262274 pte_t dst_pteval;
263275
264276 BUG_ON(dst_addr >= dst_start + len);
265− VM_BUG_ON(dst_addr & ~huge_page_mask(h));
266277
267278 /*
268279 * Serialize via hugetlb_fault_mutex
269280 */
270281 idx = linear_page_index(dst_vma, dst_addr);
271282 mapping = dst_vma->vm_file->f_mapping;
272− hash = hugetlb_fault_mutex_hash(h, mapping, idx, dst_addr);
283+ hash = hugetlb_fault_mutex_hash(mapping, idx);
273284 mutex_lock(&hugetlb_fault_mutex_table[hash]);
274285
275286 err = -ENOMEM;
276− dst_pte = huge_pte_alloc(dst_mm, dst_addr, huge_page_size(h));
287+ dst_pte = huge_pte_alloc(dst_mm, dst_addr, vma_hpagesize);
277288 if (!dst_pte) {
278289 mutex_unlock(&hugetlb_fault_mutex_table[hash]);
279290 goto out_unlock;
@@ -300,7 +311,8 @@ retry: · 20 unmodified lines
300311
301312 err = copy_huge_page_from_user(page,
302313 (const void __user *)src_addr,
303− pages_per_huge_page(h), true);
314+ vma_hpagesize / PAGE_SIZE,
315+ true);
304316 if (unlikely(err)) {
305317 err = -EFAULT;
306318 goto out;
@@ -475,20 +487,9 @@ retry: · 168 unmodified lines
475487 * both valid and fully within a single existing vma.
476488 */
477489 err = -ENOENT;
478− dst_vma = find_vma(dst_mm, dst_start);
490+ dst_vma = find_dst_vma(dst_mm, dst_start, len);
479491 if (!dst_vma)
480492 goto out_unlock;
481− /*
482− * Check the vma is registered in uffd, this is required to
483− * enforce the VM_MAYWRITE check done at uffd registration
484− * time.
485− */
486− if (!dst_vma->vm_userfaultfd_ctx.ctx)
487− goto out_unlock;
488−
489− if (dst_start < dst_vma->vm_start ||
490− dst_start + len > dst_vma->vm_end)
491− goto out_unlock;
492493
493494 err = -EINVAL;
494495 /*
mm/util.c+16−6
@@ -271,7 +271,7 @@ void *memdup_user_nul(const void __user *src, size_t len) · 270 unmodified lines
271271 EXPORT_SYMBOL(memdup_user_nul);
272272
273273 void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
274− struct vm_area_struct *prev, struct rb_node *rb_parent)
274+ struct vm_area_struct *prev)
275275 {
276276 struct vm_area_struct *next;
277277
@@ -280,18 +280,28 @@ void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma, · 2 unmodified lines
280280 next = prev->vm_next;
281281 prev->vm_next = vma;
282282 } else {
283+ next = mm->mmap;
283284 mm->mmap = vma;
284− if (rb_parent)
285− next = rb_entry(rb_parent,
286− struct vm_area_struct, vm_rb);
287− else
288− next = NULL;
289285 }
290286 vma->vm_next = next;
291287 if (next)
292288 next->vm_prev = vma;
293289 }
294290
291+void __vma_unlink_list(struct mm_struct *mm, struct vm_area_struct *vma)
292+{
293+ struct vm_area_struct *prev, *next;
294+
295+ next = vma->vm_next;
296+ prev = vma->vm_prev;
297+ if (prev)
298+ prev->vm_next = next;
299+ else
300+ mm->mmap = next;
301+ if (next)
302+ next->vm_prev = prev;
303+}
304+
295305 /* Check if the vma is being used as a stack by this task */
296306 int vma_is_stack_for_current(struct vm_area_struct *vma)
297307 {
mm/vmalloc.c+134−58
@@ -331,6 +331,7 @@ EXPORT_SYMBOL(vmalloc_to_pfn); · 330 unmodified lines
331331
332332
333333 static DEFINE_SPINLOCK(vmap_area_lock);
334+static DEFINE_SPINLOCK(free_vmap_area_lock);
334335 /* Export for kexec only */
335336 LIST_HEAD(vmap_area_list);
336337 static LLIST_HEAD(vmap_purge_list);
@@ -682,7 +683,7 @@ insert_vmap_area_augment(struct vmap_area *va, · 345 unmodified lines
682683 * free area is inserted. If VA has been merged, it is
683684 * freed.
684685 */
685−static __always_inline void
686+static __always_inline struct vmap_area *
686687 merge_or_add_vmap_area(struct vmap_area *va,
687688 struct rb_root *root, struct list_head *head)
688689 {
@@ -749,7 +750,10 @@ merge_or_add_vmap_area(struct vmap_area *va, · 60 unmodified lines
749750
750751 /* Free vmap_area object. */
751752 kmem_cache_free(vmap_area_cachep, va);
752− return;
753+
754+ /* Point to the new merged area. */
755+ va = sibling;
756+ merged = true;
753757 }
754758 }
755759
@@ -758,6 +762,8 @@ insert: · 2 unmodified lines
758762 link_va(va, root, parent, link, head);
759763 augment_tree_propagate_from(va);
760764 }
765+
766+ return va;
761767 }
762768
763769 static __always_inline bool
@@ -968,6 +974,19 @@ adjust_va_to_fit_type(struct vmap_area *va, · 204 unmodified lines
968974 * There are a few exceptions though, as an example it is
969975 * a first allocation (early boot up) when we have "one"
970976 * big free space that has to be split.
977+ *
978+ * Also we can hit this path in case of regular "vmap"
979+ * allocations, if "this" current CPU was not preloaded.
980+ * See the comment in alloc_vmap_area() why. If so, then
981+ * GFP_NOWAIT is used instead to get an extra object for
982+ * split purpose. That is rare and most time does not
983+ * occur.
984+ *
985+ * What happens if an allocation gets failed. Basically,
986+ * an "overflow" path is triggered to purge lazily freed
987+ * areas to free some memory, then, the "retry" path is
988+ * triggered to repeat one more time. See more details
989+ * in alloc_vmap_area() function.
971990 */
972991 lva = kmem_cache_alloc(vmap_area_cachep, GFP_NOWAIT);
973992 if (!lva)
@@ -1063,9 +1082,9 @@ static struct vmap_area *alloc_vmap_area(unsigned long size, · 89 unmodified lines
10631082 return ERR_PTR(-EBUSY);
10641083
10651084 might_sleep();
1085+ gfp_mask = gfp_mask & GFP_RECLAIM_MASK;
10661086
1067− va = kmem_cache_alloc_node(vmap_area_cachep,
1068− gfp_mask & GFP_RECLAIM_MASK, node);
1087+ va = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node);
10691088 if (unlikely(!va))
10701089 return ERR_PTR(-ENOMEM);
10711090
@@ -1073,49 +1092,55 @@ static struct vmap_area *alloc_vmap_area(unsigned long size, · 1 unmodified line
10731092 * Only scan the relevant parts containing pointers to other objects
10741093 * to avoid false negatives.
10751094 */
1076− kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask & GFP_RECLAIM_MASK);
1095+ kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask);
10771096
10781097 retry:
10791098 /*
1080− * Preload this CPU with one extra vmap_area object to ensure
1081− * that we have it available when fit type of free area is
1082− * NE_FIT_TYPE.
1099+ * Preload this CPU with one extra vmap_area object. It is used
1100+ * when fit type of free area is NE_FIT_TYPE. Please note, it
1101+ * does not guarantee that an allocation occurs on a CPU that
1102+ * is preloaded, instead we minimize the case when it is not.
1103+ * It can happen because of cpu migration, because there is a
1104+ * race until the below spinlock is taken.
10831105 *
10841106 * The preload is done in non-atomic context, thus it allows us
10851107 * to use more permissive allocation masks to be more stable under
1086− * low memory condition and high memory pressure.
1108+ * low memory condition and high memory pressure. In rare case,
1109+ * if not preloaded, GFP_NOWAIT is used.
10871110 *
1088− * Even if it fails we do not really care about that. Just proceed
1089− * as it is. "overflow" path will refill the cache we allocate from.
1111+ * Set "pva" to NULL here, because of "retry" path.
10901112 */
1091− preempt_disable();
1092− if (!__this_cpu_read(ne_fit_preload_node)) {
1093− preempt_enable();
1094− pva = kmem_cache_alloc_node(vmap_area_cachep, GFP_KERNEL, node);
1095− preempt_disable();
1096−
1097− if (__this_cpu_cmpxchg(ne_fit_preload_node, NULL, pva)) {
1098− if (pva)
1099− kmem_cache_free(vmap_area_cachep, pva);
1100− }
1101− }
1113+ pva = NULL;
11021114
1103− spin_lock(&vmap_area_lock);
1104− preempt_enable();
1115+ if (!this_cpu_read(ne_fit_preload_node))
1116+ /*
1117+ * Even if it fails we do not really care about that.
1118+ * Just proceed as it is. If needed "overflow" path
1119+ * will refill the cache we allocate from.
1120+ */
1121+ pva = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node);
1122+
1123+ spin_lock(&free_vmap_area_lock);
1124+
1125+ if (pva && __this_cpu_cmpxchg(ne_fit_preload_node, NULL, pva))
1126+ kmem_cache_free(vmap_area_cachep, pva);
11051127
11061128 /*
11071129 * If an allocation fails, the "vend" address is
11081130 * returned. Therefore trigger the overflow path.
11091131 */
11101132 addr = __alloc_vmap_area(size, align, vstart, vend);
1133+ spin_unlock(&free_vmap_area_lock);
1134+
11111135 if (unlikely(addr == vend))
11121136 goto overflow;
11131137
11141138 va->va_start = addr;
11151139 va->va_end = addr + size;
11161140 va->vm = NULL;
1117− insert_vmap_area(va, &vmap_area_root, &vmap_area_list);
11181141
1142+ spin_lock(&vmap_area_lock);
1143+ insert_vmap_area(va, &vmap_area_root, &vmap_area_list);
11191144 spin_unlock(&vmap_area_lock);
11201145
11211146 BUG_ON(!IS_ALIGNED(va->va_start, align));
@@ -1125,7 +1150,6 @@ retry: · 3 unmodified lines
11251150 return va;
11261151
11271152 overflow:
1128− spin_unlock(&vmap_area_lock);
11291153 if (!purged) {
11301154 purge_vmap_area_lazy();
11311155 purged = 1;
@@ -1161,28 +1185,24 @@ int unregister_vmap_purge_notifier(struct notifier_block *nb) · 29 unmodified lines
11611185 }
11621186 EXPORT_SYMBOL_GPL(unregister_vmap_purge_notifier);
11631187
1164−static void __free_vmap_area(struct vmap_area *va)
1188+/*
1189+ * Free a region of KVA allocated by alloc_vmap_area
1190+ */
1191+static void free_vmap_area(struct vmap_area *va)
11651192 {
11661193 /*
11671194 * Remove from the busy tree/list.
11681195 */
1196+ spin_lock(&vmap_area_lock);
11691197 unlink_va(va, &vmap_area_root);
1198+ spin_unlock(&vmap_area_lock);
11701199
11711200 /*
1172− * Merge VA with its neighbors, otherwise just add it.
1201+ * Insert/Merge it back to the free tree/list.
11731202 */
1174− merge_or_add_vmap_area(va,
1175− &free_vmap_area_root, &free_vmap_area_list);
1176−}
1177−
1178−/*
1179− * Free a region of KVA allocated by alloc_vmap_area
1180− */
1181−static void free_vmap_area(struct vmap_area *va)
1182−{
1183− spin_lock(&vmap_area_lock);
1184− __free_vmap_area(va);
1185− spin_unlock(&vmap_area_lock);
1203+ spin_lock(&free_vmap_area_lock);
1204+ merge_or_add_vmap_area(va, &free_vmap_area_root, &free_vmap_area_list);
1205+ spin_unlock(&free_vmap_area_lock);
11861206 }
11871207
11881208 /*
@@ -1275,24 +1295,30 @@ static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end) · 86 unmodified lines
12751295 flush_tlb_kernel_range(start, end);
12761296 resched_threshold = lazy_max_pages() << 1;
12771297
1278− spin_lock(&vmap_area_lock);
1298+ spin_lock(&free_vmap_area_lock);
12791299 llist_for_each_entry_safe(va, n_va, valist, purge_list) {
12801300 unsigned long nr = (va->va_end - va->va_start) >> PAGE_SHIFT;
1301+ unsigned long orig_start = va->va_start;
1302+ unsigned long orig_end = va->va_end;
12811303
12821304 /*
12831305 * Finally insert or merge lazily-freed area. It is
12841306 * detached and there is no need to "unlink" it from
12851307 * anything.
12861308 */
1287− merge_or_add_vmap_area(va,
1288− &free_vmap_area_root, &free_vmap_area_list);
1309+ va = merge_or_add_vmap_area(va, &free_vmap_area_root,
1310+ &free_vmap_area_list);
1311+
1312+ if (is_vmalloc_or_module_addr((void *)orig_start))
1313+ kasan_release_vmalloc(orig_start, orig_end,
1314+ va->va_start, va->va_end);
12891315
12901316 atomic_long_sub(nr, &vmap_lazy_nr);
12911317
12921318 if (atomic_long_read(&vmap_lazy_nr) < resched_threshold)
1293− cond_resched_lock(&vmap_area_lock);
1319+ cond_resched_lock(&free_vmap_area_lock);
12941320 }
1295− spin_unlock(&vmap_area_lock);
1321+ spin_unlock(&free_vmap_area_lock);
12961322 return true;
12971323 }
12981324
@@ -2014,15 +2040,21 @@ int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page **pages) · 715 unmodified lines
20142040 }
20152041 EXPORT_SYMBOL_GPL(map_vm_area);
20162042
2017−static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
2018− unsigned long flags, const void *caller)
2043+static inline void setup_vmalloc_vm_locked(struct vm_struct *vm,
2044+ struct vmap_area *va, unsigned long flags, const void *caller)
20192045 {
2020− spin_lock(&vmap_area_lock);
20212046 vm->flags = flags;
20222047 vm->addr = (void *)va->va_start;
20232048 vm->size = va->va_end - va->va_start;
20242049 vm->caller = caller;
20252050 va->vm = vm;
2051+}
2052+
2053+static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
2054+ unsigned long flags, const void *caller)
2055+{
2056+ spin_lock(&vmap_area_lock);
2057+ setup_vmalloc_vm_locked(vm, va, flags, caller);
20262058 spin_unlock(&vmap_area_lock);
20272059 }
20282060
@@ -2068,6 +2100,22 @@ static struct vm_struct *__get_vm_area_node(unsigned long size, · 39 unmodified lines
20682100
20692101 setup_vmalloc_vm(area, va, flags, caller);
20702102
2103+ /*
2104+ * For KASAN, if we are in vmalloc space, we need to cover the shadow
2105+ * area with real memory. If we come here through VM_ALLOC, this is
2106+ * done by a higher level function that has access to the true size,
2107+ * which might not be a full page.
2108+ *
2109+ * We assume module space comes via VM_ALLOC path.
2110+ */
2111+ if (is_vmalloc_addr(area->addr) && !(area->flags & VM_ALLOC)) {
2112+ if (kasan_populate_vmalloc(area->size, area)) {
2113+ unmap_vmap_area(va);
2114+ kfree(area);
2115+ return NULL;
2116+ }
2117+ }
2118+
20712119 return area;
20722120 }
20732121
@@ -2245,6 +2293,9 @@ static void __vunmap(const void *addr, int deallocate_pages) · 171 unmodified lines
22452293 debug_check_no_locks_freed(area->addr, get_vm_area_size(area));
22462294 debug_check_no_obj_freed(area->addr, get_vm_area_size(area));
22472295
2296+ if (area->flags & VM_KASAN)
2297+ kasan_poison_vmalloc(area->addr, area->size);
2298+
22482299 vm_remove_mappings(area, deallocate_pages);
22492300
22502301 if (deallocate_pages) {
@@ -2440,7 +2491,7 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, · 189 unmodified lines
24402491 goto fail;
24412492 }
24422493 area->pages[i] = page;
2443− if (gfpflags_allow_blocking(gfp_mask|highmem_mask))
2494+ if (gfpflags_allow_blocking(gfp_mask))
24442495 cond_resched();
24452496 }
24462497 atomic_long_add(area->nr_pages, &nr_vmalloc_pages);
@@ -2497,6 +2548,11 @@ void *__vmalloc_node_range(unsigned long size, unsigned long align, · 50 unmodified lines
24972548 if (!addr)
24982549 return NULL;
24992550
2551+ if (is_vmalloc_or_module_addr(area->addr)) {
2552+ if (kasan_populate_vmalloc(real_size, area))
2553+ return NULL;
2554+ }
2555+
25002556 /*
25012557 * In this function, newly allocated vm_struct has VM_UNINITIALIZED
25022558 * flag. It means that vm_struct is not fully initialized.
@@ -3282,7 +3338,7 @@ struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, · 779 unmodified lines
32823338 goto err_free;
32833339 }
32843340 retry:
3285− spin_lock(&vmap_area_lock);
3341+ spin_lock(&free_vmap_area_lock);
32863342
32873343 /* start scanning - we scan from the top, begin with the last area */
32883344 area = term_area = last_area;
@@ -3364,29 +3420,44 @@ retry: · 75 unmodified lines
33643420 va = vas[area];
33653421 va->va_start = start;
33663422 va->va_end = start + size;
3367−
3368− insert_vmap_area(va, &vmap_area_root, &vmap_area_list);
33693423 }
33703424
3371− spin_unlock(&vmap_area_lock);
3425+ spin_unlock(&free_vmap_area_lock);
33723426
33733427 /* insert all vm's */
3374− for (area = 0; area < nr_vms; area++)
3375− setup_vmalloc_vm(vms[area], vas[area], VM_ALLOC,
3428+ spin_lock(&vmap_area_lock);
3429+ for (area = 0; area < nr_vms; area++) {
3430+ insert_vmap_area(vas[area], &vmap_area_root, &vmap_area_list);
3431+
3432+ setup_vmalloc_vm_locked(vms[area], vas[area], VM_ALLOC,
33763433 pcpu_get_vm_areas);
3434+ }
3435+ spin_unlock(&vmap_area_lock);
3436+
3437+ /* populate the shadow space outside of the lock */
3438+ for (area = 0; area < nr_vms; area++) {
3439+ /* assume success here */
3440+ kasan_populate_vmalloc(sizes[area], vms[area]);
3441+ }
33773442
33783443 kfree(vas);
33793444 return vms;
33803445
33813446 recovery:
3382− /* Remove previously inserted areas. */
3447+ /*
3448+ * Remove previously allocated areas. There is no
3449+ * need in removing these areas from the busy tree,
3450+ * because they are inserted only on the final step
3451+ * and when pcpu_get_vm_areas() is success.
3452+ */
33833453 while (area--) {
3384− __free_vmap_area(vas[area]);
3454+ merge_or_add_vmap_area(vas[area], &free_vmap_area_root,
3455+ &free_vmap_area_list);
33853456 vas[area] = NULL;
33863457 }
33873458
33883459 overflow:
3389− spin_unlock(&vmap_area_lock);
3460+ spin_unlock(&free_vmap_area_lock);
33903461 if (!purged) {
33913462 purge_vmap_area_lazy();
33923463 purged = true;
@@ -3437,9 +3508,12 @@ void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms) · 44 unmodified lines
34373508
34383509 #ifdef CONFIG_PROC_FS
34393510 static void *s_start(struct seq_file *m, loff_t *pos)
3511+ __acquires(&vmap_purge_lock)
34403512 __acquires(&vmap_area_lock)
34413513 {
3514+ mutex_lock(&vmap_purge_lock);
34423515 spin_lock(&vmap_area_lock);
3516+
34433517 return seq_list_start(&vmap_area_list, *pos);
34443518 }
34453519
@@ -3449,8 +3523,10 @@ static void *s_next(struct seq_file *m, void *p, loff_t *pos) · 3 unmodified lines
34493523 }
34503524
34513525 static void s_stop(struct seq_file *m, void *p)
3526+ __releases(&vmap_purge_lock)
34523527 __releases(&vmap_area_lock)
34533528 {
3529+ mutex_unlock(&vmap_purge_lock);
34543530 spin_unlock(&vmap_area_lock);
34553531 }
34563532
mm/vmscan.c+327−335
@@ -79,6 +79,13 @@ struct scan_control { · 78 unmodified lines
7979 */
8080 struct mem_cgroup *target_mem_cgroup;
8181
82+ /* Can active pages be deactivated as part of reclaim? */
83+#define DEACTIVATE_ANON 1
84+#define DEACTIVATE_FILE 2
85+ unsigned int may_deactivate:2;
86+ unsigned int force_deactivate:1;
87+ unsigned int skipped_deactivate:1;
88+
8289 /* Writepage batching in laptop mode; RECLAIM_WRITE */
8390 unsigned int may_writepage:1;
8491
@@ -101,6 +108,12 @@ struct scan_control { · 16 unmodified lines
101108 /* One of the zones is ready for compaction */
102109 unsigned int compaction_ready:1;
103110
111+ /* There is easily reclaimable cold cache in the current node */
112+ unsigned int cache_trim_mode:1;
113+
114+ /* The file pages on the current node are dangerously low */
115+ unsigned int file_is_tiny:1;
116+
104117 /* Allocation order */
105118 s8 order;
106119
@@ -239,13 +252,13 @@ static void unregister_memcg_shrinker(struct shrinker *shrinker) · 132 unmodified lines
239252 up_write(&shrinker_rwsem);
240253 }
241254
242−static bool global_reclaim(struct scan_control *sc)
255+static bool cgroup_reclaim(struct scan_control *sc)
243256 {
244− return !sc->target_mem_cgroup;
257+ return sc->target_mem_cgroup;
245258 }
246259
247260 /**
248− * sane_reclaim - is the usual dirty throttling mechanism operational?
261+ * writeback_throttling_sane - is the usual dirty throttling mechanism available?
249262 * @sc: scan_control in question
250263 *
251264 * The normal page dirty throttling mechanism in balance_dirty_pages() is
@@ -257,11 +270,9 @@ static bool global_reclaim(struct scan_control *sc) · 5 unmodified lines
257270 * This function tests whether the vmscan currently in progress can assume
258271 * that the normal dirty throttling mechanism is operational.
259272 */
260−static bool sane_reclaim(struct scan_control *sc)
273+static bool writeback_throttling_sane(struct scan_control *sc)
261274 {
262− struct mem_cgroup *memcg = sc->target_mem_cgroup;
263−
264− if (!memcg)
275+ if (!cgroup_reclaim(sc))
265276 return true;
266277 #ifdef CONFIG_CGROUP_WRITEBACK
267278 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
@@ -269,29 +280,6 @@ static bool sane_reclaim(struct scan_control *sc) · 1 unmodified line
269280 #endif
270281 return false;
271282 }
272−
273−static void set_memcg_congestion(pg_data_t *pgdat,
274− struct mem_cgroup *memcg,
275− bool congested)
276−{
277− struct mem_cgroup_per_node *mn;
278−
279− if (!memcg)
280− return;
281−
282− mn = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
283− WRITE_ONCE(mn->congested, congested);
284−}
285−
286−static bool memcg_congested(pg_data_t *pgdat,
287− struct mem_cgroup *memcg)
288−{
289− struct mem_cgroup_per_node *mn;
290−
291− mn = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
292− return READ_ONCE(mn->congested);
293−
294−}
295283 #else
296284 static int prealloc_memcg_shrinker(struct shrinker *shrinker)
297285 {
@@ -302,27 +290,15 @@ static void unregister_memcg_shrinker(struct shrinker *shrinker) · 4 unmodified lines
302290 {
303291 }
304292
305−static bool global_reclaim(struct scan_control *sc)
293+static bool cgroup_reclaim(struct scan_control *sc)
306294 {
307− return true;
295+ return false;
308296 }
309297
310−static bool sane_reclaim(struct scan_control *sc)
298+static bool writeback_throttling_sane(struct scan_control *sc)
311299 {
312300 return true;
313301 }
314−
315−static inline void set_memcg_congestion(struct pglist_data *pgdat,
316− struct mem_cgroup *memcg, bool congested)
317−{
318−}
319−
320−static inline bool memcg_congested(struct pglist_data *pgdat,
321− struct mem_cgroup *memcg)
322−{
323− return false;
324−
325−}
326302 #endif
327303
328304 /*
@@ -351,32 +327,21 @@ unsigned long zone_reclaimable_pages(struct zone *zone) · 22 unmodified lines
351327 */
352328 unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx)
353329 {
354− unsigned long lru_size = 0;
330+ unsigned long size = 0;
355331 int zid;
356332
357− if (!mem_cgroup_disabled()) {
358− for (zid = 0; zid < MAX_NR_ZONES; zid++)
359− lru_size += mem_cgroup_get_zone_lru_size(lruvec, lru, zid);
360− } else
361− lru_size = node_page_state(lruvec_pgdat(lruvec), NR_LRU_BASE + lru);
362−
363− for (zid = zone_idx + 1; zid < MAX_NR_ZONES; zid++) {
333+ for (zid = 0; zid <= zone_idx && zid < MAX_NR_ZONES; zid++) {
364334 struct zone *zone = &lruvec_pgdat(lruvec)->node_zones[zid];
365− unsigned long size;
366335
367336 if (!managed_zone(zone))
368337 continue;
369338
370339 if (!mem_cgroup_disabled())
371− size = mem_cgroup_get_zone_lru_size(lruvec, lru, zid);
340+ size += mem_cgroup_get_zone_lru_size(lruvec, lru, zid);
372341 else
373− size = zone_page_state(&lruvec_pgdat(lruvec)->node_zones[zid],
374− NR_ZONE_LRU_BASE + lru);
375− lru_size -= min(size, lru_size);
342+ size += zone_page_state(zone, NR_ZONE_LRU_BASE + lru);
376343 }
377−
378− return lru_size;
379−
344+ return size;
380345 }
381346
382347 /*
@@ -775,7 +740,7 @@ static inline int is_page_cache_freeable(struct page *page) · 392 unmodified lines
775740 return page_count(page) - page_has_private(page) == 1 + page_cache_pins;
776741 }
777742
778−static int may_write_to_inode(struct inode *inode, struct scan_control *sc)
743+static int may_write_to_inode(struct inode *inode)
779744 {
780745 if (current->flags & PF_SWAPWRITE)
781746 return 1;
@@ -823,8 +788,7 @@ typedef enum { · 41 unmodified lines
823788 * pageout is called by shrink_page_list() for each dirty page.
824789 * Calls ->writepage().
825790 */
826−static pageout_t pageout(struct page *page, struct address_space *mapping,
827− struct scan_control *sc)
791+static pageout_t pageout(struct page *page, struct address_space *mapping)
828792 {
829793 /*
830794 * If the page is dirty, only perform writeback if that write
@@ -860,7 +824,7 @@ static pageout_t pageout(struct page *page, struct address_space *mapping, · 29 unmodified lines
860824 }
861825 if (mapping->a_ops->writepage == NULL)
862826 return PAGE_ACTIVATE;
863− if (!may_write_to_inode(mapping->host, sc))
827+ if (!may_write_to_inode(mapping->host))
864828 return PAGE_KEEP;
865829
866830 if (clear_page_dirty_for_io(page)) {
@@ -899,7 +863,7 @@ static pageout_t pageout(struct page *page, struct address_space *mapping, · 32 unmodified lines
899863 * gets returned with a refcount of 0.
900864 */
901865 static int __remove_mapping(struct address_space *mapping, struct page *page,
902− bool reclaimed)
866+ bool reclaimed, struct mem_cgroup *target_memcg)
903867 {
904868 unsigned long flags;
905869 int refcount;
@@ -971,7 +935,7 @@ static int __remove_mapping(struct address_space *mapping, struct page *page, · 65 unmodified lines
971935 */
972936 if (reclaimed && page_is_file_cache(page) &&
973937 !mapping_exiting(mapping) && !dax_mapping(mapping))
974− shadow = workingset_eviction(page);
938+ shadow = workingset_eviction(page, target_memcg);
975939 __delete_from_page_cache(page, shadow);
976940 xa_unlock_irqrestore(&mapping->i_pages, flags);
977941
@@ -994,7 +958,7 @@ cannot_free: · 16 unmodified lines
994958 */
995959 int remove_mapping(struct address_space *mapping, struct page *page)
996960 {
997− if (__remove_mapping(mapping, page, false)) {
961+ if (__remove_mapping(mapping, page, false, NULL)) {
998962 /*
999963 * Unfreezing the refcount with 1 rather than 2 effectively
1000964 * drops the pagecache ref for us without requiring another
@@ -1239,7 +1203,7 @@ static unsigned long shrink_page_list(struct list_head *page_list, · 238 unmodified lines
12391203 goto activate_locked;
12401204
12411205 /* Case 2 above */
1242− } else if (sane_reclaim(sc) ||
1206+ } else if (writeback_throttling_sane(sc) ||
12431207 !PageReclaim(page) || !may_enter_fs) {
12441208 /*
12451209 * This is slightly racy - end_page_writeback()
@@ -1394,7 +1358,7 @@ static unsigned long shrink_page_list(struct list_head *page_list, · 148 unmodified lines
13941358 * starts and then write it out here.
13951359 */
13961360 try_to_unmap_flush_dirty();
1397− switch (pageout(page, mapping, sc)) {
1361+ switch (pageout(page, mapping)) {
13981362 case PAGE_KEEP:
13991363 goto keep_locked;
14001364 case PAGE_ACTIVATE:
@@ -1472,7 +1436,8 @@ static unsigned long shrink_page_list(struct list_head *page_list, · 71 unmodified lines
14721436
14731437 count_vm_event(PGLAZYFREED);
14741438 count_memcg_page_event(page, PGLAZYFREED);
1475− } else if (!mapping || !__remove_mapping(mapping, page, true))
1439+ } else if (!mapping || !__remove_mapping(mapping, page, true,
1440+ sc->target_mem_cgroup))
14761441 goto keep_locked;
14771442
14781443 unlock_page(page);
@@ -1820,7 +1785,7 @@ int isolate_lru_page(struct page *page) · 341 unmodified lines
18201785
18211786 /*
18221787 * A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and
1823− * then get resheduled. When there are massive number of tasks doing page
1788+ * then get rescheduled. When there are massive number of tasks doing page
18241789 * allocation, such sleeping direct reclaimers may keep piling up on each CPU,
18251790 * the LRU list will go small and be scanned faster than necessary, leading to
18261791 * unnecessary swapping, thrashing and OOM.
@@ -1833,7 +1798,7 @@ static int too_many_isolated(struct pglist_data *pgdat, int file, · 6 unmodified lines
18331798 if (current_is_kswapd())
18341799 return 0;
18351800
1836− if (!sane_reclaim(sc))
1801+ if (!writeback_throttling_sane(sc))
18371802 return 0;
18381803
18391804 if (file) {
@@ -1983,7 +1948,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, · 143 unmodified lines
19831948 reclaim_stat->recent_scanned[file] += nr_taken;
19841949
19851950 item = current_is_kswapd() ? PGSCAN_KSWAPD : PGSCAN_DIRECT;
1986− if (global_reclaim(sc))
1951+ if (!cgroup_reclaim(sc))
19871952 __count_vm_events(item, nr_scanned);
19881953 __count_memcg_events(lruvec_memcg(lruvec), item, nr_scanned);
19891954 spin_unlock_irq(&pgdat->lru_lock);
@@ -1997,7 +1962,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, · 7 unmodified lines
19971962 spin_lock_irq(&pgdat->lru_lock);
19981963
19991964 item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT;
2000− if (global_reclaim(sc))
1965+ if (!cgroup_reclaim(sc))
20011966 __count_vm_events(item, nr_reclaimed);
20021967 __count_memcg_events(lruvec_memcg(lruvec), item, nr_reclaimed);
20031968 reclaim_stat->recent_rotated[0] += stat.nr_activate[0];
@@ -2199,6 +2164,20 @@ unsigned long reclaim_pages(struct list_head *page_list) · 195 unmodified lines
21992164 return nr_reclaimed;
22002165 }
22012166
2167+static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
2168+ struct lruvec *lruvec, struct scan_control *sc)
2169+{
2170+ if (is_active_lru(lru)) {
2171+ if (sc->may_deactivate & (1 << is_file_lru(lru)))
2172+ shrink_active_list(nr_to_scan, lruvec, sc, lru);
2173+ else
2174+ sc->skipped_deactivate = 1;
2175+ return 0;
2176+ }
2177+
2178+ return shrink_inactive_list(nr_to_scan, lruvec, sc, lru);
2179+}
2180+
22022181 /*
22032182 * The inactive anon list should be small enough that the VM never has
22042183 * to do too much work.
@@ -2227,64 +2206,25 @@ unsigned long reclaim_pages(struct list_head *page_list) · 22 unmodified lines
22272206 * 1TB 101 10GB
22282207 * 10TB 320 32GB
22292208 */
2230−static bool inactive_list_is_low(struct lruvec *lruvec, bool file,
2231− struct scan_control *sc, bool trace)
2209+static bool inactive_is_low(struct lruvec *lruvec, enum lru_list inactive_lru)
22322210 {
2233− enum lru_list active_lru = file * LRU_FILE + LRU_ACTIVE;
2234− struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2235− enum lru_list inactive_lru = file * LRU_FILE;
2211+ enum lru_list active_lru = inactive_lru + LRU_ACTIVE;
22362212 unsigned long inactive, active;
22372213 unsigned long inactive_ratio;
2238− unsigned long refaults;
22392214 unsigned long gb;
22402215
2241− /*
2242− * If we don't have swap space, anonymous page deactivation
2243− * is pointless.
2244− */
2245− if (!file && !total_swap_pages)
2246− return false;
2247−
2248− inactive = lruvec_lru_size(lruvec, inactive_lru, sc->reclaim_idx);
2249− active = lruvec_lru_size(lruvec, active_lru, sc->reclaim_idx);
2250−
2251− /*
2252− * When refaults are being observed, it means a new workingset
2253− * is being established. Disable active list protection to get
2254− * rid of the stale workingset quickly.
2255− */
2256− refaults = lruvec_page_state_local(lruvec, WORKINGSET_ACTIVATE);
2257− if (file && lruvec->refaults != refaults) {
2258− inactive_ratio = 0;
2259− } else {
2260− gb = (inactive + active) >> (30 - PAGE_SHIFT);
2261− if (gb)
2262− inactive_ratio = int_sqrt(10 * gb);
2263− else
2264− inactive_ratio = 1;
2265− }
2216+ inactive = lruvec_page_state(lruvec, NR_LRU_BASE + inactive_lru);
2217+ active = lruvec_page_state(lruvec, NR_LRU_BASE + active_lru);
22662218
2267− if (trace)
2268− trace_mm_vmscan_inactive_list_is_low(pgdat->node_id, sc->reclaim_idx,
2269− lruvec_lru_size(lruvec, inactive_lru, MAX_NR_ZONES), inactive,
2270− lruvec_lru_size(lruvec, active_lru, MAX_NR_ZONES), active,
2271− inactive_ratio, file);
2219+ gb = (inactive + active) >> (30 - PAGE_SHIFT);
2220+ if (gb)
2221+ inactive_ratio = int_sqrt(10 * gb);
2222+ else
2223+ inactive_ratio = 1;
22722224
22732225 return inactive * inactive_ratio < active;
22742226 }
22752227
2276−static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
2277− struct lruvec *lruvec, struct scan_control *sc)
2278−{
2279− if (is_active_lru(lru)) {
2280− if (inactive_list_is_low(lruvec, is_file_lru(lru), sc, true))
2281− shrink_active_list(nr_to_scan, lruvec, sc, lru);
2282− return 0;
2283− }
2284−
2285− return shrink_inactive_list(nr_to_scan, lruvec, sc, lru);
2286−}
2287−
22882228 enum scan_balance {
22892229 SCAN_EQUAL,
22902230 SCAN_FRACT,
@@ -2301,10 +2241,10 @@ enum scan_balance { · 10 unmodified lines
23012241 * nr[0] = anon inactive pages to scan; nr[1] = anon active pages to scan
23022242 * nr[2] = file inactive pages to scan; nr[3] = file active pages to scan
23032243 */
2304−static void get_scan_count(struct lruvec *lruvec, struct mem_cgroup *memcg,
2305− struct scan_control *sc, unsigned long *nr,
2306− unsigned long *lru_pages)
2244+static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc,
2245+ unsigned long *nr)
23072246 {
2247+ struct mem_cgroup *memcg = lruvec_memcg(lruvec);
23082248 int swappiness = mem_cgroup_swappiness(memcg);
23092249 struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
23102250 u64 fraction[2];
@@ -2329,7 +2269,7 @@ static void get_scan_count(struct lruvec *lruvec, struct mem_cgroup *memcg, · 18 unmodified lines
23292269 * using the memory controller's swap limit feature would be
23302270 * too expensive.
23312271 */
2332− if (!global_reclaim(sc) && !swappiness) {
2272+ if (cgroup_reclaim(sc) && !swappiness) {
23332273 scan_balance = SCAN_FILE;
23342274 goto out;
23352275 }
@@ -2345,58 +2285,18 @@ static void get_scan_count(struct lruvec *lruvec, struct mem_cgroup *memcg, · 9 unmodified lines
23452285 }
23462286
23472287 /*
2348− * Prevent the reclaimer from falling into the cache trap: as
2349− * cache pages start out inactive, every cache fault will tip
2350− * the scan balance towards the file LRU. And as the file LRU
2351− * shrinks, so does the window for rotation from references.
2352− * This means we have a runaway feedback loop where a tiny
2353− * thrashing file LRU becomes infinitely more attractive than
2354− * anon pages. Try to detect this based on file LRU size.
2288+ * If the system is almost out of file pages, force-scan anon.
23552289 */
2356− if (global_reclaim(sc)) {
2357− unsigned long pgdatfile;
2358− unsigned long pgdatfree;
2359− int z;
2360− unsigned long total_high_wmark = 0;
2361−
2362− pgdatfree = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES);
2363− pgdatfile = node_page_state(pgdat, NR_ACTIVE_FILE) +
2364− node_page_state(pgdat, NR_INACTIVE_FILE);
2365−
2366− for (z = 0; z < MAX_NR_ZONES; z++) {
2367− struct zone *zone = &pgdat->node_zones[z];
2368− if (!managed_zone(zone))
2369− continue;
2370−
2371− total_high_wmark += high_wmark_pages(zone);
2372− }
2373−
2374− if (unlikely(pgdatfile + pgdatfree <= total_high_wmark)) {
2375− /*
2376− * Force SCAN_ANON if there are enough inactive
2377− * anonymous pages on the LRU in eligible zones.
2378− * Otherwise, the small LRU gets thrashed.
2379− */
2380− if (!inactive_list_is_low(lruvec, false, sc, false) &&
2381− lruvec_lru_size(lruvec, LRU_INACTIVE_ANON, sc->reclaim_idx)
2382− >> sc->priority) {
2383− scan_balance = SCAN_ANON;
2384− goto out;
2385− }
2386− }
2290+ if (sc->file_is_tiny) {
2291+ scan_balance = SCAN_ANON;
2292+ goto out;
23872293 }
23882294
23892295 /*
2390− * If there is enough inactive page cache, i.e. if the size of the
2391− * inactive list is greater than that of the active list *and* the
2392− * inactive list actually has some pages to scan on this priority, we
2393− * do not reclaim anything from the anonymous working set right now.
2394− * Without the second condition we could end up never scanning an
2395− * lruvec even if it has plenty of old anonymous pages unless the
2396− * system is under heavy pressure.
2296+ * If there is enough inactive page cache, we do not reclaim
2297+ * anything from the anonymous working right now.
23972298 */
2398− if (!inactive_list_is_low(lruvec, true, sc, false) &&
2399− lruvec_lru_size(lruvec, LRU_INACTIVE_FILE, sc->reclaim_idx) >> sc->priority) {
2299+ if (sc->cache_trim_mode) {
24002300 scan_balance = SCAN_FILE;
24012301 goto out;
24022302 }
@@ -2454,7 +2354,6 @@ static void get_scan_count(struct lruvec *lruvec, struct mem_cgroup *memcg, · 51 unmodified lines
24542354 fraction[1] = fp;
24552355 denominator = ap + fp + 1;
24562356 out:
2457− *lru_pages = 0;
24582357 for_each_evictable_lru(lru) {
24592358 int file = is_file_lru(lru);
24602359 unsigned long lruvec_size;
@@ -2549,18 +2448,12 @@ out: · 88 unmodified lines
25492448 BUG();
25502449 }
25512450
2552− *lru_pages += lruvec_size;
25532451 nr[lru] = scan;
25542452 }
25552453 }
25562454
2557−/*
2558− * This is a basic per-node page freer. Used by both kswapd and direct reclaim.
2559− */
2560−static void shrink_node_memcg(struct pglist_data *pgdat, struct mem_cgroup *memcg,
2561− struct scan_control *sc, unsigned long *lru_pages)
2455+static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
25622456 {
2563− struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg);
25642457 unsigned long nr[NR_LRU_LISTS];
25652458 unsigned long targets[NR_LRU_LISTS];
25662459 unsigned long nr_to_scan;
@@ -2570,7 +2463,7 @@ static void shrink_node_memcg(struct pglist_data *pgdat, struct mem_cgroup *memc · 3 unmodified lines
25702463 struct blk_plug plug;
25712464 bool scan_adjusted;
25722465
2573− get_scan_count(lruvec, memcg, sc, nr, lru_pages);
2466+ get_scan_count(lruvec, sc, nr);
25742467
25752468 /* Record the original scan target for proportional adjustments later */
25762469 memcpy(targets, nr, sizeof(nr));
@@ -2586,7 +2479,7 @@ static void shrink_node_memcg(struct pglist_data *pgdat, struct mem_cgroup *memc · 9 unmodified lines
25862479 * abort proportional reclaim if either the file or anon lru has already
25872480 * dropped to zero at the first pass.
25882481 */
2589− scan_adjusted = (global_reclaim(sc) && !current_is_kswapd() &&
2482+ scan_adjusted = (!cgroup_reclaim(sc) && !current_is_kswapd() &&
25902483 sc->priority == DEF_PRIORITY);
25912484
25922485 blk_start_plug(&plug);
@@ -2668,7 +2561,7 @@ static void shrink_node_memcg(struct pglist_data *pgdat, struct mem_cgroup *memc · 75 unmodified lines
26682561 * Even if we did not try to evict anon pages at all, we want to
26692562 * rebalance the anon lru active/inactive ratio.
26702563 */
2671− if (inactive_list_is_low(lruvec, false, sc, true))
2564+ if (total_swap_pages && inactive_is_low(lruvec, LRU_INACTIVE_ANON))
26722565 shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
26732566 sc, LRU_ACTIVE_ANON);
26742567 }
@@ -2744,156 +2637,234 @@ static inline bool should_continue_reclaim(struct pglist_data *pgdat, · 69 unmodified lines
27442637 return inactive_lru_pages > pages_for_compaction;
27452638 }
27462639
2747−static bool pgdat_memcg_congested(pg_data_t *pgdat, struct mem_cgroup *memcg)
2640+static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc)
27482641 {
2749− return test_bit(PGDAT_CONGESTED, &pgdat->flags) ||
2750− (memcg && memcg_congested(pgdat, memcg));
2642+ struct mem_cgroup *target_memcg = sc->target_mem_cgroup;
2643+ struct mem_cgroup *memcg;
2644+
2645+ memcg = mem_cgroup_iter(target_memcg, NULL, NULL);
2646+ do {
2647+ struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
2648+ unsigned long reclaimed;
2649+ unsigned long scanned;
2650+
2651+ switch (mem_cgroup_protected(target_memcg, memcg)) {
2652+ case MEMCG_PROT_MIN:
2653+ /*
2654+ * Hard protection.
2655+ * If there is no reclaimable memory, OOM.
2656+ */
2657+ continue;
2658+ case MEMCG_PROT_LOW:
2659+ /*
2660+ * Soft protection.
2661+ * Respect the protection only as long as
2662+ * there is an unprotected supply
2663+ * of reclaimable memory from other cgroups.
2664+ */
2665+ if (!sc->memcg_low_reclaim) {
2666+ sc->memcg_low_skipped = 1;
2667+ continue;
2668+ }
2669+ memcg_memory_event(memcg, MEMCG_LOW);
2670+ break;
2671+ case MEMCG_PROT_NONE:
2672+ /*
2673+ * All protection thresholds breached. We may
2674+ * still choose to vary the scan pressure
2675+ * applied based on by how much the cgroup in
2676+ * question has exceeded its protection
2677+ * thresholds (see get_scan_count).
2678+ */
2679+ break;
2680+ }
2681+
2682+ reclaimed = sc->nr_reclaimed;
2683+ scanned = sc->nr_scanned;
2684+
2685+ shrink_lruvec(lruvec, sc);
2686+
2687+ shrink_slab(sc->gfp_mask, pgdat->node_id, memcg,
2688+ sc->priority);
2689+
2690+ /* Record the group's reclaim efficiency */
2691+ vmpressure(sc->gfp_mask, memcg, false,
2692+ sc->nr_scanned - scanned,
2693+ sc->nr_reclaimed - reclaimed);
2694+
2695+ } while ((memcg = mem_cgroup_iter(target_memcg, memcg, NULL)));
27512696 }
27522697
27532698 static bool shrink_node(pg_data_t *pgdat, struct scan_control *sc)
27542699 {
27552700 struct reclaim_state *reclaim_state = current->reclaim_state;
27562701 unsigned long nr_reclaimed, nr_scanned;
2702+ struct lruvec *target_lruvec;
27572703 bool reclaimable = false;
2704+ unsigned long file;
27582705
2759− do {
2760− struct mem_cgroup *root = sc->target_mem_cgroup;
2761− unsigned long node_lru_pages = 0;
2762− struct mem_cgroup *memcg;
2706+ target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat);
27632707
2764− memset(&sc->nr, 0, sizeof(sc->nr));
2708+again:
2709+ memset(&sc->nr, 0, sizeof(sc->nr));
27652710
2766− nr_reclaimed = sc->nr_reclaimed;
2767− nr_scanned = sc->nr_scanned;
2711+ nr_reclaimed = sc->nr_reclaimed;
2712+ nr_scanned = sc->nr_scanned;
27682713
2769− memcg = mem_cgroup_iter(root, NULL, NULL);
2770− do {
2771− unsigned long lru_pages;
2772− unsigned long reclaimed;
2773− unsigned long scanned;
2714+ /*
2715+ * Target desirable inactive:active list ratios for the anon
2716+ * and file LRU lists.
2717+ */
2718+ if (!sc->force_deactivate) {
2719+ unsigned long refaults;
27742720
2775− switch (mem_cgroup_protected(root, memcg)) {
2776− case MEMCG_PROT_MIN:
2777− /*
2778− * Hard protection.
2779− * If there is no reclaimable memory, OOM.
2780− */
2781− continue;
2782− case MEMCG_PROT_LOW:
2783− /*
2784− * Soft protection.
2785− * Respect the protection only as long as
2786− * there is an unprotected supply
2787− * of reclaimable memory from other cgroups.
2788− */
2789− if (!sc->memcg_low_reclaim) {
2790− sc->memcg_low_skipped = 1;
2791− continue;
2792− }
2793− memcg_memory_event(memcg, MEMCG_LOW);
2794− break;
2795− case MEMCG_PROT_NONE:
2796− /*
2797− * All protection thresholds breached. We may
2798− * still choose to vary the scan pressure
2799− * applied based on by how much the cgroup in
2800− * question has exceeded its protection
2801− * thresholds (see get_scan_count).
2802− */
2803− break;
2804− }
2721+ if (inactive_is_low(target_lruvec, LRU_INACTIVE_ANON))
2722+ sc->may_deactivate |= DEACTIVATE_ANON;
2723+ else
2724+ sc->may_deactivate &= ~DEACTIVATE_ANON;
28052725
2806− reclaimed = sc->nr_reclaimed;
2807− scanned = sc->nr_scanned;
2808− shrink_node_memcg(pgdat, memcg, sc, &lru_pages);
2809− node_lru_pages += lru_pages;
2726+ /*
2727+ * When refaults are being observed, it means a new
2728+ * workingset is being established. Deactivate to get
2729+ * rid of any stale active pages quickly.
2730+ */
2731+ refaults = lruvec_page_state(target_lruvec,
2732+ WORKINGSET_ACTIVATE);
2733+ if (refaults != target_lruvec->refaults ||
2734+ inactive_is_low(target_lruvec, LRU_INACTIVE_FILE))
2735+ sc->may_deactivate |= DEACTIVATE_FILE;
2736+ else
2737+ sc->may_deactivate &= ~DEACTIVATE_FILE;
2738+ } else
2739+ sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE;
28102740
2811− shrink_slab(sc->gfp_mask, pgdat->node_id, memcg,
2812− sc->priority);
2741+ /*
2742+ * If we have plenty of inactive file pages that aren't
2743+ * thrashing, try to reclaim those first before touching
2744+ * anonymous pages.
2745+ */
2746+ file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE);
2747+ if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE))
2748+ sc->cache_trim_mode = 1;
2749+ else
2750+ sc->cache_trim_mode = 0;
2751+
2752+ /*
2753+ * Prevent the reclaimer from falling into the cache trap: as
2754+ * cache pages start out inactive, every cache fault will tip
2755+ * the scan balance towards the file LRU. And as the file LRU
2756+ * shrinks, so does the window for rotation from references.
2757+ * This means we have a runaway feedback loop where a tiny
2758+ * thrashing file LRU becomes infinitely more attractive than
2759+ * anon pages. Try to detect this based on file LRU size.
2760+ */
2761+ if (!cgroup_reclaim(sc)) {
2762+ unsigned long total_high_wmark = 0;
2763+ unsigned long free, anon;
2764+ int z;
28132765
2814− /* Record the group's reclaim efficiency */
2815− vmpressure(sc->gfp_mask, memcg, false,
2816− sc->nr_scanned - scanned,
2817− sc->nr_reclaimed - reclaimed);
2766+ free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES);
2767+ file = node_page_state(pgdat, NR_ACTIVE_FILE) +
2768+ node_page_state(pgdat, NR_INACTIVE_FILE);
28182769
2819− } while ((memcg = mem_cgroup_iter(root, memcg, NULL)));
2770+ for (z = 0; z < MAX_NR_ZONES; z++) {
2771+ struct zone *zone = &pgdat->node_zones[z];
2772+ if (!managed_zone(zone))
2773+ continue;
28202774
2821− if (reclaim_state) {
2822− sc->nr_reclaimed += reclaim_state->reclaimed_slab;
2823− reclaim_state->reclaimed_slab = 0;
2775+ total_high_wmark += high_wmark_pages(zone);
28242776 }
28252777
2826− /* Record the subtree's reclaim efficiency */
2827− vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true,
2828− sc->nr_scanned - nr_scanned,
2829− sc->nr_reclaimed - nr_reclaimed);
2778+ /*
2779+ * Consider anon: if that's low too, this isn't a
2780+ * runaway file reclaim problem, but rather just
2781+ * extreme pressure. Reclaim as per usual then.
2782+ */
2783+ anon = node_page_state(pgdat, NR_INACTIVE_ANON);
28302784
2831− if (sc->nr_reclaimed - nr_reclaimed)
2832− reclaimable = true;
2785+ sc->file_is_tiny =
2786+ file + free <= total_high_wmark &&
2787+ !(sc->may_deactivate & DEACTIVATE_ANON) &&
2788+ anon >> sc->priority;
2789+ }
28332790
2834− if (current_is_kswapd()) {
2835− /*
2836− * If reclaim is isolating dirty pages under writeback,
2837− * it implies that the long-lived page allocation rate
2838− * is exceeding the page laundering rate. Either the
2839− * global limits are not being effective at throttling
2840− * processes due to the page distribution throughout
2841− * zones or there is heavy usage of a slow backing
2842− * device. The only option is to throttle from reclaim
2843− * context which is not ideal as there is no guarantee
2844− * the dirtying process is throttled in the same way
2845− * balance_dirty_pages() manages.
2846− *
2847− * Once a node is flagged PGDAT_WRITEBACK, kswapd will
2848− * count the number of pages under pages flagged for
2849− * immediate reclaim and stall if any are encountered
2850− * in the nr_immediate check below.
2851− */
2852− if (sc->nr.writeback && sc->nr.writeback == sc->nr.taken)
2853− set_bit(PGDAT_WRITEBACK, &pgdat->flags);
2791+ shrink_node_memcgs(pgdat, sc);
28542792
2855− /*
2856− * Tag a node as congested if all the dirty pages
2857− * scanned were backed by a congested BDI and
2858− * wait_iff_congested will stall.
2859− */
2860− if (sc->nr.dirty && sc->nr.dirty == sc->nr.congested)
2861− set_bit(PGDAT_CONGESTED, &pgdat->flags);
2793+ if (reclaim_state) {
2794+ sc->nr_reclaimed += reclaim_state->reclaimed_slab;
2795+ reclaim_state->reclaimed_slab = 0;
2796+ }
28622797
2863− /* Allow kswapd to start writing pages during reclaim.*/
2864− if (sc->nr.unqueued_dirty == sc->nr.file_taken)
2865− set_bit(PGDAT_DIRTY, &pgdat->flags);
2798+ /* Record the subtree's reclaim efficiency */
2799+ vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true,
2800+ sc->nr_scanned - nr_scanned,
2801+ sc->nr_reclaimed - nr_reclaimed);
28662802
2867− /*
2868− * If kswapd scans pages marked marked for immediate
2869− * reclaim and under writeback (nr_immediate), it
2870− * implies that pages are cycling through the LRU
2871− * faster than they are written so also forcibly stall.
2872− */
2873− if (sc->nr.immediate)
2874− congestion_wait(BLK_RW_ASYNC, HZ/10);
2875− }
2803+ if (sc->nr_reclaimed - nr_reclaimed)
2804+ reclaimable = true;
28762805
2806+ if (current_is_kswapd()) {
28772807 /*
2878− * Legacy memcg will stall in page writeback so avoid forcibly
2879− * stalling in wait_iff_congested().
2808+ * If reclaim is isolating dirty pages under writeback,
2809+ * it implies that the long-lived page allocation rate
2810+ * is exceeding the page laundering rate. Either the
2811+ * global limits are not being effective at throttling
2812+ * processes due to the page distribution throughout
2813+ * zones or there is heavy usage of a slow backing
2814+ * device. The only option is to throttle from reclaim
2815+ * context which is not ideal as there is no guarantee
2816+ * the dirtying process is throttled in the same way
2817+ * balance_dirty_pages() manages.
2818+ *
2819+ * Once a node is flagged PGDAT_WRITEBACK, kswapd will
2820+ * count the number of pages under pages flagged for
2821+ * immediate reclaim and stall if any are encountered
2822+ * in the nr_immediate check below.
28802823 */
2881− if (!global_reclaim(sc) && sane_reclaim(sc) &&
2882− sc->nr.dirty && sc->nr.dirty == sc->nr.congested)
2883− set_memcg_congestion(pgdat, root, true);
2824+ if (sc->nr.writeback && sc->nr.writeback == sc->nr.taken)
2825+ set_bit(PGDAT_WRITEBACK, &pgdat->flags);
2826+
2827+ /* Allow kswapd to start writing pages during reclaim.*/
2828+ if (sc->nr.unqueued_dirty == sc->nr.file_taken)
2829+ set_bit(PGDAT_DIRTY, &pgdat->flags);
28842830
28852831 /*
2886− * Stall direct reclaim for IO completions if underlying BDIs
2887− * and node is congested. Allow kswapd to continue until it
2888− * starts encountering unqueued dirty pages or cycling through
2889− * the LRU too quickly.
2832+ * If kswapd scans pages marked marked for immediate
2833+ * reclaim and under writeback (nr_immediate), it
2834+ * implies that pages are cycling through the LRU
2835+ * faster than they are written so also forcibly stall.
28902836 */
2891− if (!sc->hibernation_mode && !current_is_kswapd() &&
2892− current_may_throttle() && pgdat_memcg_congested(pgdat, root))
2893− wait_iff_congested(BLK_RW_ASYNC, HZ/10);
2837+ if (sc->nr.immediate)
2838+ congestion_wait(BLK_RW_ASYNC, HZ/10);
2839+ }
28942840
2895− } while (should_continue_reclaim(pgdat, sc->nr_reclaimed - nr_reclaimed,
2896− sc));
2841+ /*
2842+ * Tag a node/memcg as congested if all the dirty pages
2843+ * scanned were backed by a congested BDI and
2844+ * wait_iff_congested will stall.
2845+ *
2846+ * Legacy memcg will stall in page writeback so avoid forcibly
2847+ * stalling in wait_iff_congested().
2848+ */
2849+ if ((current_is_kswapd() ||
2850+ (cgroup_reclaim(sc) && writeback_throttling_sane(sc))) &&
2851+ sc->nr.dirty && sc->nr.dirty == sc->nr.congested)
2852+ set_bit(LRUVEC_CONGESTED, &target_lruvec->flags);
2853+
2854+ /*
2855+ * Stall direct reclaim for IO completions if underlying BDIs
2856+ * and node is congested. Allow kswapd to continue until it
2857+ * starts encountering unqueued dirty pages or cycling through
2858+ * the LRU too quickly.
2859+ */
2860+ if (!current_is_kswapd() && current_may_throttle() &&
2861+ !sc->hibernation_mode &&
2862+ test_bit(LRUVEC_CONGESTED, &target_lruvec->flags))
2863+ wait_iff_congested(BLK_RW_ASYNC, HZ/10);
2864+
2865+ if (should_continue_reclaim(pgdat, sc->nr_reclaimed - nr_reclaimed,
2866+ sc))
2867+ goto again;
28972868
28982869 /*
28992870 * Kswapd gives up on balancing particular nodes after too
@@ -2973,7 +2944,7 @@ static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc) · 73 unmodified lines
29732944 * Take care memory controller reclaiming has small influence
29742945 * to global LRU.
29752946 */
2976− if (global_reclaim(sc)) {
2947+ if (!cgroup_reclaim(sc)) {
29772948 if (!cpuset_zone_allowed(zone,
29782949 GFP_KERNEL | __GFP_HARDWALL))
29792950 continue;
@@ -3032,19 +3003,14 @@ static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc) · 52 unmodified lines
30323003 sc->gfp_mask = orig_mask;
30333004 }
30343005
3035−static void snapshot_refaults(struct mem_cgroup *root_memcg, pg_data_t *pgdat)
3006+static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat)
30363007 {
3037− struct mem_cgroup *memcg;
3038−
3039− memcg = mem_cgroup_iter(root_memcg, NULL, NULL);
3040− do {
3041− unsigned long refaults;
3042− struct lruvec *lruvec;
3008+ struct lruvec *target_lruvec;
3009+ unsigned long refaults;
30433010
3044− lruvec = mem_cgroup_lruvec(pgdat, memcg);
3045− refaults = lruvec_page_state_local(lruvec, WORKINGSET_ACTIVATE);
3046− lruvec->refaults = refaults;
3047− } while ((memcg = mem_cgroup_iter(root_memcg, memcg, NULL)));
3011+ target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat);
3012+ refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE);
3013+ target_lruvec->refaults = refaults;
30483014 }
30493015
30503016 /*
@@ -3073,7 +3039,7 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist, · 22 unmodified lines
30733039 retry:
30743040 delayacct_freepages_start();
30753041
3076− if (global_reclaim(sc))
3042+ if (!cgroup_reclaim(sc))
30773043 __count_zid_vm_events(ALLOCSTALL, sc->reclaim_idx, 1);
30783044
30793045 do {
@@ -3102,8 +3068,16 @@ retry: · 22 unmodified lines
31023068 if (zone->zone_pgdat == last_pgdat)
31033069 continue;
31043070 last_pgdat = zone->zone_pgdat;
3071+
31053072 snapshot_refaults(sc->target_mem_cgroup, zone->zone_pgdat);
3106− set_memcg_congestion(last_pgdat, sc->target_mem_cgroup, false);
3073+
3074+ if (cgroup_reclaim(sc)) {
3075+ struct lruvec *lruvec;
3076+
3077+ lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup,
3078+ zone->zone_pgdat);
3079+ clear_bit(LRUVEC_CONGESTED, &lruvec->flags);
3080+ }
31073081 }
31083082
31093083 delayacct_freepages_end();
@@ -3115,9 +3089,27 @@ retry: · 5 unmodified lines
31153089 if (sc->compaction_ready)
31163090 return 1;
31173091
3092+ /*
3093+ * We make inactive:active ratio decisions based on the node's
3094+ * composition of memory, but a restrictive reclaim_idx or a
3095+ * memory.low cgroup setting can exempt large amounts of
3096+ * memory from reclaim. Neither of which are very common, so
3097+ * instead of doing costly eligibility calculations of the
3098+ * entire cgroup subtree up front, we assume the estimates are
3099+ * good, and retry with forcible deactivation if that fails.
3100+ */
3101+ if (sc->skipped_deactivate) {
3102+ sc->priority = initial_priority;
3103+ sc->force_deactivate = 1;
3104+ sc->skipped_deactivate = 0;
3105+ goto retry;
3106+ }
3107+
31183108 /* Untapped cgroup reserves? Don't OOM, retry. */
31193109 if (sc->memcg_low_skipped) {
31203110 sc->priority = initial_priority;
3111+ sc->force_deactivate = 0;
3112+ sc->skipped_deactivate = 0;
31213113 sc->memcg_low_reclaim = 1;
31223114 sc->memcg_low_skipped = 0;
31233115 goto retry;
@@ -3309,6 +3301,7 @@ unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, · 185 unmodified lines
33093301 pg_data_t *pgdat,
33103302 unsigned long *nr_scanned)
33113303 {
3304+ struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
33123305 struct scan_control sc = {
33133306 .nr_to_reclaim = SWAP_CLUSTER_MAX,
33143307 .target_mem_cgroup = memcg,
@@ -3317,7 +3310,6 @@ unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, · 2 unmodified lines
33173310 .reclaim_idx = MAX_NR_ZONES - 1,
33183311 .may_swap = !noswap,
33193312 };
3320− unsigned long lru_pages;
33213313
33223314 WARN_ON_ONCE(!current->reclaim_state);
33233315
@@ -3334,7 +3326,7 @@ unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, · 10 unmodified lines
33343326 * will pick up pages from other mem cgroup's as well. We hack
33353327 * the priority and make it zero.
33363328 */
3337− shrink_node_memcg(pgdat, memcg, &sc, &lru_pages);
3329+ shrink_lruvec(lruvec, &sc);
33383330
33393331 trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed);
33403332
@@ -3348,10 +3340,8 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, · 7 unmodified lines
33483340 gfp_t gfp_mask,
33493341 bool may_swap)
33503342 {
3351− struct zonelist *zonelist;
33523343 unsigned long nr_reclaimed;
33533344 unsigned long pflags;
3354− int nid;
33553345 unsigned int noreclaim_flag;
33563346 struct scan_control sc = {
33573347 .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
@@ -3364,16 +3354,14 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, · 6 unmodified lines
33643354 .may_unmap = 1,
33653355 .may_swap = may_swap,
33663356 };
3367−
3368− set_task_reclaim_state(current, &sc.reclaim_state);
33693357 /*
3370− * Unlike direct reclaim via alloc_pages(), memcg's reclaim doesn't
3371− * take care of from where we get pages. So the node where we start the
3372− * scan does not need to be the current node.
3358+ * Traverse the ZONELIST_FALLBACK zonelist of the current node to put
3359+ * equal pressure on all the nodes. This is based on the assumption that
3360+ * the reclaim does not bail out early.
33733361 */
3374− nid = mem_cgroup_select_victim_node(memcg);
3362+ struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
33753363
3376− zonelist = &NODE_DATA(nid)->node_zonelists[ZONELIST_FALLBACK];
3364+ set_task_reclaim_state(current, &sc.reclaim_state);
33773365
33783366 trace_mm_vmscan_memcg_reclaim_begin(0, sc.gfp_mask);
33793367
@@ -3396,18 +3384,20 @@ static void age_active_anon(struct pglist_data *pgdat, · 16 unmodified lines
33963384 struct scan_control *sc)
33973385 {
33983386 struct mem_cgroup *memcg;
3387+ struct lruvec *lruvec;
33993388
34003389 if (!total_swap_pages)
34013390 return;
34023391
3392+ lruvec = mem_cgroup_lruvec(NULL, pgdat);
3393+ if (!inactive_is_low(lruvec, LRU_INACTIVE_ANON))
3394+ return;
3395+
34033396 memcg = mem_cgroup_iter(NULL, NULL, NULL);
34043397 do {
3405− struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg);
3406−
3407− if (inactive_list_is_low(lruvec, false, sc, true))
3408− shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
3409− sc, LRU_ACTIVE_ANON);
3410−
3398+ lruvec = mem_cgroup_lruvec(memcg, pgdat);
3399+ shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
3400+ sc, LRU_ACTIVE_ANON);
34113401 memcg = mem_cgroup_iter(NULL, memcg, NULL);
34123402 } while (memcg);
34133403 }
@@ -3475,7 +3465,9 @@ static bool pgdat_balanced(pg_data_t *pgdat, int order, int classzone_idx) · 61 unmodified lines
34753465 /* Clear pgdat state for congested, dirty or under writeback. */
34763466 static void clear_pgdat_congested(pg_data_t *pgdat)
34773467 {
3478− clear_bit(PGDAT_CONGESTED, &pgdat->flags);
3468+ struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat);
3469+
3470+ clear_bit(LRUVEC_CONGESTED, &lruvec->flags);
34793471 clear_bit(PGDAT_DIRTY, &pgdat->flags);
34803472 clear_bit(PGDAT_WRITEBACK, &pgdat->flags);
34813473 }
mm/workingset.c+53−16
@@ -213,28 +213,53 @@ static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat, · 212 unmodified lines
213213 *workingsetp = workingset;
214214 }
215215
216+static void advance_inactive_age(struct mem_cgroup *memcg, pg_data_t *pgdat)
217+{
218+ /*
219+ * Reclaiming a cgroup means reclaiming all its children in a
220+ * round-robin fashion. That means that each cgroup has an LRU
221+ * order that is composed of the LRU orders of its child
222+ * cgroups; and every page has an LRU position not just in the
223+ * cgroup that owns it, but in all of that group's ancestors.
224+ *
225+ * So when the physical inactive list of a leaf cgroup ages,
226+ * the virtual inactive lists of all its parents, including
227+ * the root cgroup's, age as well.
228+ */
229+ do {
230+ struct lruvec *lruvec;
231+
232+ lruvec = mem_cgroup_lruvec(memcg, pgdat);
233+ atomic_long_inc(&lruvec->inactive_age);
234+ } while (memcg && (memcg = parent_mem_cgroup(memcg)));
235+}
236+
216237 /**
217238 * workingset_eviction - note the eviction of a page from memory
239+ * @target_memcg: the cgroup that is causing the reclaim
218240 * @page: the page being evicted
219241 *
220242 * Returns a shadow entry to be stored in @page->mapping->i_pages in place
221243 * of the evicted @page so that a later refault can be detected.
222244 */
223−void *workingset_eviction(struct page *page)
245+void *workingset_eviction(struct page *page, struct mem_cgroup *target_memcg)
224246 {
225247 struct pglist_data *pgdat = page_pgdat(page);
226− struct mem_cgroup *memcg = page_memcg(page);
227− int memcgid = mem_cgroup_id(memcg);
228248 unsigned long eviction;
229249 struct lruvec *lruvec;
250+ int memcgid;
230251
231252 /* Page is fully exclusive and pins page->mem_cgroup */
232253 VM_BUG_ON_PAGE(PageLRU(page), page);
233254 VM_BUG_ON_PAGE(page_count(page), page);
234255 VM_BUG_ON_PAGE(!PageLocked(page), page);
235256
236− lruvec = mem_cgroup_lruvec(pgdat, memcg);
237− eviction = atomic_long_inc_return(&lruvec->inactive_age);
257+ advance_inactive_age(page_memcg(page), pgdat);
258+
259+ lruvec = mem_cgroup_lruvec(target_memcg, pgdat);
260+ /* XXX: target_memcg can be NULL, go through lruvec */
261+ memcgid = mem_cgroup_id(lruvec_memcg(lruvec));
262+ eviction = atomic_long_read(&lruvec->inactive_age);
238263 return pack_shadow(memcgid, pgdat, eviction, PageWorkingset(page));
239264 }
240265
@@ -244,10 +269,13 @@ void *workingset_eviction(struct page *page) · 3 unmodified lines
244269 * @shadow: shadow entry of the evicted page
245270 *
246271 * Calculates and evaluates the refault distance of the previously
247− * evicted page in the context of the node it was allocated in.
272+ * evicted page in the context of the node and the memcg whose memory
273+ * pressure caused the eviction.
248274 */
249275 void workingset_refault(struct page *page, void *shadow)
250276 {
277+ struct mem_cgroup *eviction_memcg;
278+ struct lruvec *eviction_lruvec;
251279 unsigned long refault_distance;
252280 struct pglist_data *pgdat;
253281 unsigned long active_file;
@@ -277,12 +305,12 @@ void workingset_refault(struct page *page, void *shadow) · 23 unmodified lines
277305 * would be better if the root_mem_cgroup existed in all
278306 * configurations instead.
279307 */
280− memcg = mem_cgroup_from_id(memcgid);
281− if (!mem_cgroup_disabled() && !memcg)
308+ eviction_memcg = mem_cgroup_from_id(memcgid);
309+ if (!mem_cgroup_disabled() && !eviction_memcg)
282310 goto out;
283− lruvec = mem_cgroup_lruvec(pgdat, memcg);
284− refault = atomic_long_read(&lruvec->inactive_age);
285− active_file = lruvec_lru_size(lruvec, LRU_ACTIVE_FILE, MAX_NR_ZONES);
311+ eviction_lruvec = mem_cgroup_lruvec(eviction_memcg, pgdat);
312+ refault = atomic_long_read(&eviction_lruvec->inactive_age);
313+ active_file = lruvec_page_state(eviction_lruvec, NR_ACTIVE_FILE);
286314
287315 /*
288316 * Calculate the refault distance
@@ -302,6 +330,17 @@ void workingset_refault(struct page *page, void *shadow) · 13 unmodified lines
302330 */
303331 refault_distance = (refault - eviction) & EVICTION_MASK;
304332
333+ /*
334+ * The activation decision for this page is made at the level
335+ * where the eviction occurred, as that is where the LRU order
336+ * during page reclaim is being determined.
337+ *
338+ * However, the cgroup that will own the page is the one that
339+ * is actually experiencing the refault event.
340+ */
341+ memcg = page_memcg(page);
342+ lruvec = mem_cgroup_lruvec(memcg, pgdat);
343+
305344 inc_lruvec_state(lruvec, WORKINGSET_REFAULT);
306345
307346 /*
@@ -313,7 +352,7 @@ void workingset_refault(struct page *page, void *shadow) · 5 unmodified lines
313352 goto out;
314353
315354 SetPageActive(page);
316− atomic_long_inc(&lruvec->inactive_age);
355+ advance_inactive_age(memcg, pgdat);
317356 inc_lruvec_state(lruvec, WORKINGSET_ACTIVATE);
318357
319358 /* Page was active prior to eviction */
@@ -332,7 +371,6 @@ out: · 12 unmodified lines
332371 void workingset_activation(struct page *page)
333372 {
334373 struct mem_cgroup *memcg;
335− struct lruvec *lruvec;
336374
337375 rcu_read_lock();
338376 /*
@@ -345,8 +383,7 @@ void workingset_activation(struct page *page) · 6 unmodified lines
345383 memcg = page_memcg_rcu(page);
346384 if (!mem_cgroup_disabled() && !memcg)
347385 goto out;
348− lruvec = mem_cgroup_lruvec(page_pgdat(page), memcg);
349− atomic_long_inc(&lruvec->inactive_age);
386+ advance_inactive_age(memcg, page_pgdat(page));
350387 out:
351388 rcu_read_unlock();
352389 }
@@ -426,7 +463,7 @@ static unsigned long count_shadow_nodes(struct shrinker *shrinker, · 73 unmodified lines
426463 struct lruvec *lruvec;
427464 int i;
428465
429− lruvec = mem_cgroup_lruvec(NODE_DATA(sc->nid), sc->memcg);
466+ lruvec = mem_cgroup_lruvec(sc->memcg, NODE_DATA(sc->nid));
430467 for (pages = 0, i = 0; i < NR_LRU_LISTS; i++)
431468 pages += lruvec_page_state_local(lruvec,
432469 NR_LRU_BASE + i);
mm/z3fold.c+303−72
@@ -41,6 +41,7 @@ · 40 unmodified lines
4141 #include <linux/workqueue.h>
4242 #include <linux/slab.h>
4343 #include <linux/spinlock.h>
44+#include <linux/rwlock.h>
4445 #include <linux/zpool.h>
4546 #include <linux/magic.h>
4647
@@ -90,6 +91,7 @@ struct z3fold_buddy_slots { · 43 unmodified lines
9091 */
9192 unsigned long slot[BUDDY_MASK + 1];
9293 unsigned long pool; /* back link + flags */
94+ rwlock_t lock;
9395 };
9496 #define HANDLE_FLAG_MASK (0x03)
9597
@@ -124,6 +126,7 @@ struct z3fold_header { · 28 unmodified lines
124126 unsigned short start_middle;
125127 unsigned short first_num:2;
126128 unsigned short mapped_count:2;
129+ unsigned short foreign_handles:2;
127130 };
128131
129132 /**
@@ -178,6 +181,19 @@ enum z3fold_page_flags { · 48 unmodified lines
178181 PAGE_CLAIMED, /* by either reclaim or free */
179182 };
180183
184+/*
185+ * handle flags, go under HANDLE_FLAG_MASK
186+ */
187+enum z3fold_handle_flags {
188+ HANDLES_ORPHANED = 0,
189+};
190+
191+/*
192+ * Forward declarations
193+ */
194+static struct z3fold_header *__z3fold_alloc(struct z3fold_pool *, size_t, bool);
195+static void compact_page_work(struct work_struct *w);
196+
181197 /*****************
182198 * Helpers
183199 *****************/
@@ -191,8 +207,6 @@ static int size_to_chunks(size_t size) · 7 unmodified lines
191207 #define for_each_unbuddied_list(_iter, _begin) \
192208 for ((_iter) = (_begin); (_iter) < NCHUNKS; (_iter)++)
193209
194−static void compact_page_work(struct work_struct *w);
195−
196210 static inline struct z3fold_buddy_slots *alloc_slots(struct z3fold_pool *pool,
197211 gfp_t gfp)
198212 {
@@ -204,6 +218,7 @@ static inline struct z3fold_buddy_slots *alloc_slots(struct z3fold_pool *pool, · 5 unmodified lines
204218 if (slots) {
205219 memset(slots->slot, 0, sizeof(slots->slot));
206220 slots->pool = (unsigned long)pool;
221+ rwlock_init(&slots->lock);
207222 }
208223
209224 return slots;
@@ -219,25 +234,110 @@ static inline struct z3fold_buddy_slots *handle_to_slots(unsigned long handle) · 9 unmodified lines
219234 return (struct z3fold_buddy_slots *)(handle & ~(SLOTS_ALIGN - 1));
220235 }
221236
237+/* Lock a z3fold page */
238+static inline void z3fold_page_lock(struct z3fold_header *zhdr)
239+{
240+ spin_lock(&zhdr->page_lock);
241+}
242+
243+/* Try to lock a z3fold page */
244+static inline int z3fold_page_trylock(struct z3fold_header *zhdr)
245+{
246+ return spin_trylock(&zhdr->page_lock);
247+}
248+
249+/* Unlock a z3fold page */
250+static inline void z3fold_page_unlock(struct z3fold_header *zhdr)
251+{
252+ spin_unlock(&zhdr->page_lock);
253+}
254+
255+
256+static inline struct z3fold_header *__get_z3fold_header(unsigned long handle,
257+ bool lock)
258+{
259+ struct z3fold_buddy_slots *slots;
260+ struct z3fold_header *zhdr;
261+ int locked = 0;
262+
263+ if (!(handle & (1 << PAGE_HEADLESS))) {
264+ slots = handle_to_slots(handle);
265+ do {
266+ unsigned long addr;
267+
268+ read_lock(&slots->lock);
269+ addr = *(unsigned long *)handle;
270+ zhdr = (struct z3fold_header *)(addr & PAGE_MASK);
271+ if (lock)
272+ locked = z3fold_page_trylock(zhdr);
273+ read_unlock(&slots->lock);
274+ if (locked)
275+ break;
276+ cpu_relax();
277+ } while (lock);
278+ } else {
279+ zhdr = (struct z3fold_header *)(handle & PAGE_MASK);
280+ }
281+
282+ return zhdr;
283+}
284+
285+/* Returns the z3fold page where a given handle is stored */
286+static inline struct z3fold_header *handle_to_z3fold_header(unsigned long h)
287+{
288+ return __get_z3fold_header(h, false);
289+}
290+
291+/* return locked z3fold page if it's not headless */
292+static inline struct z3fold_header *get_z3fold_header(unsigned long h)
293+{
294+ return __get_z3fold_header(h, true);
295+}
296+
297+static inline void put_z3fold_header(struct z3fold_header *zhdr)
298+{
299+ struct page *page = virt_to_page(zhdr);
300+
301+ if (!test_bit(PAGE_HEADLESS, &page->private))
302+ z3fold_page_unlock(zhdr);
303+}
304+
222305 static inline void free_handle(unsigned long handle)
223306 {
224307 struct z3fold_buddy_slots *slots;
308+ struct z3fold_header *zhdr;
225309 int i;
226310 bool is_free;
227311
228312 if (handle & (1 << PAGE_HEADLESS))
229313 return;
230314
231− WARN_ON(*(unsigned long *)handle == 0);
232− *(unsigned long *)handle = 0;
315+ if (WARN_ON(*(unsigned long *)handle == 0))
316+ return;
317+
318+ zhdr = handle_to_z3fold_header(handle);
233319 slots = handle_to_slots(handle);
320+ write_lock(&slots->lock);
321+ *(unsigned long *)handle = 0;
322+ write_unlock(&slots->lock);
323+ if (zhdr->slots == slots)
324+ return; /* simple case, nothing else to do */
325+
326+ /* we are freeing a foreign handle if we are here */
327+ zhdr->foreign_handles--;
234328 is_free = true;
329+ read_lock(&slots->lock);
330+ if (!test_bit(HANDLES_ORPHANED, &slots->pool)) {
331+ read_unlock(&slots->lock);
332+ return;
333+ }
235334 for (i = 0; i <= BUDDY_MASK; i++) {
236335 if (slots->slot[i]) {
237336 is_free = false;
238337 break;
239338 }
240339 }
340+ read_unlock(&slots->lock);
241341
242342 if (is_free) {
243343 struct z3fold_pool *pool = slots_to_pool(slots);
@@ -322,6 +422,7 @@ static struct z3fold_header *init_z3fold_page(struct page *page, bool headless, · 78 unmodified lines
322422 zhdr->first_num = 0;
323423 zhdr->start_middle = 0;
324424 zhdr->cpu = -1;
425+ zhdr->foreign_handles = 0;
325426 zhdr->slots = slots;
326427 zhdr->pool = pool;
327428 INIT_LIST_HEAD(&zhdr->buddy);
@@ -341,24 +442,6 @@ static void free_z3fold_page(struct page *page, bool headless) · 13 unmodified lines
341442 __free_page(page);
342443 }
343444
344−/* Lock a z3fold page */
345−static inline void z3fold_page_lock(struct z3fold_header *zhdr)
346−{
347− spin_lock(&zhdr->page_lock);
348−}
349−
350−/* Try to lock a z3fold page */
351−static inline int z3fold_page_trylock(struct z3fold_header *zhdr)
352−{
353− return spin_trylock(&zhdr->page_lock);
354−}
355−
356−/* Unlock a z3fold page */
357−static inline void z3fold_page_unlock(struct z3fold_header *zhdr)
358−{
359− spin_unlock(&zhdr->page_lock);
360−}
361−
362445 /* Helper function to build the index */
363446 static inline int __idx(struct z3fold_header *zhdr, enum buddy bud)
364447 {
@@ -389,7 +472,9 @@ static unsigned long __encode_handle(struct z3fold_header *zhdr, · 24 unmodified lines
389472 if (bud == LAST)
390473 h |= (zhdr->last_chunks << BUDDY_SHIFT);
391474
475+ write_lock(&slots->lock);
392476 slots->slot[idx] = h;
477+ write_unlock(&slots->lock);
393478 return (unsigned long)&slots->slot[idx];
394479 }
395480
@@ -398,22 +483,15 @@ static unsigned long encode_handle(struct z3fold_header *zhdr, enum buddy bud) · 2 unmodified lines
398483 return __encode_handle(zhdr, zhdr->slots, bud);
399484 }
400485
401−/* Returns the z3fold page where a given handle is stored */
402−static inline struct z3fold_header *handle_to_z3fold_header(unsigned long h)
403−{
404− unsigned long addr = h;
405−
406− if (!(addr & (1 << PAGE_HEADLESS)))
407− addr = *(unsigned long *)h;
408−
409− return (struct z3fold_header *)(addr & PAGE_MASK);
410−}
411−
412486 /* only for LAST bud, returns zero otherwise */
413487 static unsigned short handle_to_chunks(unsigned long handle)
414488 {
415− unsigned long addr = *(unsigned long *)handle;
489+ struct z3fold_buddy_slots *slots = handle_to_slots(handle);
490+ unsigned long addr;
416491
492+ read_lock(&slots->lock);
493+ addr = *(unsigned long *)handle;
494+ read_unlock(&slots->lock);
417495 return (addr & ~PAGE_MASK) >> BUDDY_SHIFT;
418496 }
419497
@@ -425,10 +503,13 @@ static unsigned short handle_to_chunks(unsigned long handle) · 5 unmodified lines
425503 static enum buddy handle_to_buddy(unsigned long handle)
426504 {
427505 struct z3fold_header *zhdr;
506+ struct z3fold_buddy_slots *slots = handle_to_slots(handle);
428507 unsigned long addr;
429508
509+ read_lock(&slots->lock);
430510 WARN_ON(handle & (1 << PAGE_HEADLESS));
431511 addr = *(unsigned long *)handle;
512+ read_unlock(&slots->lock);
432513 zhdr = (struct z3fold_header *)(addr & PAGE_MASK);
433514 return (addr - zhdr->first_num) & BUDDY_MASK;
434515 }
@@ -442,6 +523,8 @@ static void __release_z3fold_page(struct z3fold_header *zhdr, bool locked) · 7 unmodified lines
442523 {
443524 struct page *page = virt_to_page(zhdr);
444525 struct z3fold_pool *pool = zhdr_to_pool(zhdr);
526+ bool is_free = true;
527+ int i;
445528
446529 WARN_ON(!list_empty(&zhdr->buddy));
447530 set_bit(PAGE_STALE, &page->private);
@@ -450,8 +533,25 @@ static void __release_z3fold_page(struct z3fold_header *zhdr, bool locked) · 2 unmodified lines
450533 if (!list_empty(&page->lru))
451534 list_del_init(&page->lru);
452535 spin_unlock(&pool->lock);
536+
537+ /* If there are no foreign handles, free the handles array */
538+ read_lock(&zhdr->slots->lock);
539+ for (i = 0; i <= BUDDY_MASK; i++) {
540+ if (zhdr->slots->slot[i]) {
541+ is_free = false;
542+ break;
543+ }
544+ }
545+ if (!is_free)
546+ set_bit(HANDLES_ORPHANED, &zhdr->slots->pool);
547+ read_unlock(&zhdr->slots->lock);
548+
549+ if (is_free)
550+ kmem_cache_free(pool->c_handle, zhdr->slots);
551+
453552 if (locked)
454553 z3fold_page_unlock(zhdr);
554+
455555 spin_lock(&pool->stale_lock);
456556 list_add(&zhdr->buddy, &pool->stale);
457557 queue_work(pool->release_wq, &pool->work);
@@ -479,6 +579,7 @@ static void release_z3fold_page_locked_list(struct kref *ref) · 21 unmodified lines
479579 struct z3fold_header *zhdr = container_of(ref, struct z3fold_header,
480580 refcount);
481581 struct z3fold_pool *pool = zhdr_to_pool(zhdr);
582+
482583 spin_lock(&pool->lock);
483584 list_del_init(&zhdr->buddy);
484585 spin_unlock(&pool->lock);
@@ -559,6 +660,119 @@ static inline void *mchunk_memmove(struct z3fold_header *zhdr, · 74 unmodified lines
559660 zhdr->middle_chunks << CHUNK_SHIFT);
560661 }
561662
663+static inline bool buddy_single(struct z3fold_header *zhdr)
664+{
665+ return !((zhdr->first_chunks && zhdr->middle_chunks) ||
666+ (zhdr->first_chunks && zhdr->last_chunks) ||
667+ (zhdr->middle_chunks && zhdr->last_chunks));
668+}
669+
670+static struct z3fold_header *compact_single_buddy(struct z3fold_header *zhdr)
671+{
672+ struct z3fold_pool *pool = zhdr_to_pool(zhdr);
673+ void *p = zhdr;
674+ unsigned long old_handle = 0;
675+ size_t sz = 0;
676+ struct z3fold_header *new_zhdr = NULL;
677+ int first_idx = __idx(zhdr, FIRST);
678+ int middle_idx = __idx(zhdr, MIDDLE);
679+ int last_idx = __idx(zhdr, LAST);
680+ unsigned short *moved_chunks = NULL;
681+
682+ /*
683+ * No need to protect slots here -- all the slots are "local" and
684+ * the page lock is already taken
685+ */
686+ if (zhdr->first_chunks && zhdr->slots->slot[first_idx]) {
687+ p += ZHDR_SIZE_ALIGNED;
688+ sz = zhdr->first_chunks << CHUNK_SHIFT;
689+ old_handle = (unsigned long)&zhdr->slots->slot[first_idx];
690+ moved_chunks = &zhdr->first_chunks;
691+ } else if (zhdr->middle_chunks && zhdr->slots->slot[middle_idx]) {
692+ p += zhdr->start_middle << CHUNK_SHIFT;
693+ sz = zhdr->middle_chunks << CHUNK_SHIFT;
694+ old_handle = (unsigned long)&zhdr->slots->slot[middle_idx];
695+ moved_chunks = &zhdr->middle_chunks;
696+ } else if (zhdr->last_chunks && zhdr->slots->slot[last_idx]) {
697+ p += PAGE_SIZE - (zhdr->last_chunks << CHUNK_SHIFT);
698+ sz = zhdr->last_chunks << CHUNK_SHIFT;
699+ old_handle = (unsigned long)&zhdr->slots->slot[last_idx];
700+ moved_chunks = &zhdr->last_chunks;
701+ }
702+
703+ if (sz > 0) {
704+ enum buddy new_bud = HEADLESS;
705+ short chunks = size_to_chunks(sz);
706+ void *q;
707+
708+ new_zhdr = __z3fold_alloc(pool, sz, false);
709+ if (!new_zhdr)
710+ return NULL;
711+
712+ if (WARN_ON(new_zhdr == zhdr))
713+ goto out_fail;
714+
715+ if (new_zhdr->first_chunks == 0) {
716+ if (new_zhdr->middle_chunks != 0 &&
717+ chunks >= new_zhdr->start_middle) {
718+ new_bud = LAST;
719+ } else {
720+ new_bud = FIRST;
721+ }
722+ } else if (new_zhdr->last_chunks == 0) {
723+ new_bud = LAST;
724+ } else if (new_zhdr->middle_chunks == 0) {
725+ new_bud = MIDDLE;
726+ }
727+ q = new_zhdr;
728+ switch (new_bud) {
729+ case FIRST:
730+ new_zhdr->first_chunks = chunks;
731+ q += ZHDR_SIZE_ALIGNED;
732+ break;
733+ case MIDDLE:
734+ new_zhdr->middle_chunks = chunks;
735+ new_zhdr->start_middle =
736+ new_zhdr->first_chunks + ZHDR_CHUNKS;
737+ q += new_zhdr->start_middle << CHUNK_SHIFT;
738+ break;
739+ case LAST:
740+ new_zhdr->last_chunks = chunks;
741+ q += PAGE_SIZE - (new_zhdr->last_chunks << CHUNK_SHIFT);
742+ break;
743+ default:
744+ goto out_fail;
745+ }
746+ new_zhdr->foreign_handles++;
747+ memcpy(q, p, sz);
748+ write_lock(&zhdr->slots->lock);
749+ *(unsigned long *)old_handle = (unsigned long)new_zhdr +
750+ __idx(new_zhdr, new_bud);
751+ if (new_bud == LAST)
752+ *(unsigned long *)old_handle |=
753+ (new_zhdr->last_chunks << BUDDY_SHIFT);
754+ write_unlock(&zhdr->slots->lock);
755+ add_to_unbuddied(pool, new_zhdr);
756+ z3fold_page_unlock(new_zhdr);
757+
758+ *moved_chunks = 0;
759+ }
760+
761+ return new_zhdr;
762+
763+out_fail:
764+ if (new_zhdr) {
765+ if (kref_put(&new_zhdr->refcount, release_z3fold_page_locked))
766+ atomic64_dec(&pool->pages_nr);
767+ else {
768+ add_to_unbuddied(pool, new_zhdr);
769+ z3fold_page_unlock(new_zhdr);
770+ }
771+ }
772+ return NULL;
773+
774+}
775+
562776 #define BIG_CHUNK_GAP 3
563777 /* Has to be called with lock held */
564778 static int z3fold_compact_page(struct z3fold_header *zhdr)
@@ -638,6 +852,15 @@ static void do_compact_page(struct z3fold_header *zhdr, bool locked) · 73 unmodified lines
638852 return;
639853 }
640854
855+ if (!zhdr->foreign_handles && buddy_single(zhdr) &&
856+ zhdr->mapped_count == 0 && compact_single_buddy(zhdr)) {
857+ if (kref_put(&zhdr->refcount, release_z3fold_page_locked))
858+ atomic64_dec(&pool->pages_nr);
859+ else
860+ z3fold_page_unlock(zhdr);
861+ return;
862+ }
863+
641864 z3fold_compact_page(zhdr);
642865 add_to_unbuddied(pool, zhdr);
643866 z3fold_page_unlock(zhdr);
@@ -690,7 +913,8 @@ lookup: · 46 unmodified lines
690913 spin_unlock(&pool->lock);
691914
692915 page = virt_to_page(zhdr);
693− if (test_bit(NEEDS_COMPACTING, &page->private)) {
916+ if (test_bit(NEEDS_COMPACTING, &page->private) ||
917+ test_bit(PAGE_CLAIMED, &page->private)) {
694918 z3fold_page_unlock(zhdr);
695919 zhdr = NULL;
696920 put_cpu_ptr(pool->unbuddied);
@@ -734,7 +958,8 @@ lookup: · 37 unmodified lines
734958 spin_unlock(&pool->lock);
735959
736960 page = virt_to_page(zhdr);
737− if (test_bit(NEEDS_COMPACTING, &page->private)) {
961+ if (test_bit(NEEDS_COMPACTING, &page->private) ||
962+ test_bit(PAGE_CLAIMED, &page->private)) {
738963 z3fold_page_unlock(zhdr);
739964 zhdr = NULL;
740965 if (can_sleep)
@@ -1000,7 +1225,7 @@ static void z3fold_free(struct z3fold_pool *pool, unsigned long handle) · 259 unmodified lines
10001225 enum buddy bud;
10011226 bool page_claimed;
10021227
1003− zhdr = handle_to_z3fold_header(handle);
1228+ zhdr = get_z3fold_header(handle);
10041229 page = virt_to_page(zhdr);
10051230 page_claimed = test_and_set_bit(PAGE_CLAIMED, &page->private);
10061231
@@ -1014,6 +1239,7 @@ static void z3fold_free(struct z3fold_pool *pool, unsigned long handle) · 7 unmodified lines
10141239 spin_lock(&pool->lock);
10151240 list_del(&page->lru);
10161241 spin_unlock(&pool->lock);
1242+ put_z3fold_header(zhdr);
10171243 free_z3fold_page(page, true);
10181244 atomic64_dec(&pool->pages_nr);
10191245 }
@@ -1021,7 +1247,6 @@ static void z3fold_free(struct z3fold_pool *pool, unsigned long handle) · 1 unmodified line
10211247 }
10221248
10231249 /* Non-headless case */
1024− z3fold_page_lock(zhdr);
10251250 bud = handle_to_buddy(handle);
10261251
10271252 switch (bud) {
@@ -1037,11 +1262,13 @@ static void z3fold_free(struct z3fold_pool *pool, unsigned long handle) · 9 unmodified lines
10371262 default:
10381263 pr_err("%s: unknown bud %d\n", __func__, bud);
10391264 WARN_ON(1);
1040− z3fold_page_unlock(zhdr);
1265+ put_z3fold_header(zhdr);
1266+ clear_bit(PAGE_CLAIMED, &page->private);
10411267 return;
10421268 }
10431269
1044− free_handle(handle);
1270+ if (!page_claimed)
1271+ free_handle(handle);
10451272 if (kref_put(&zhdr->refcount, release_z3fold_page_locked_list)) {
10461273 atomic64_dec(&pool->pages_nr);
10471274 return;
@@ -1053,7 +1280,7 @@ static void z3fold_free(struct z3fold_pool *pool, unsigned long handle) · 5 unmodified lines
10531280 }
10541281 if (unlikely(PageIsolated(page)) ||
10551282 test_and_set_bit(NEEDS_COMPACTING, &page->private)) {
1056− z3fold_page_unlock(zhdr);
1283+ put_z3fold_header(zhdr);
10571284 clear_bit(PAGE_CLAIMED, &page->private);
10581285 return;
10591286 }
@@ -1063,14 +1290,14 @@ static void z3fold_free(struct z3fold_pool *pool, unsigned long handle) · 3 unmodified lines
10631290 spin_unlock(&pool->lock);
10641291 zhdr->cpu = -1;
10651292 kref_get(&zhdr->refcount);
1066− do_compact_page(zhdr, true);
10671293 clear_bit(PAGE_CLAIMED, &page->private);
1294+ do_compact_page(zhdr, true);
10681295 return;
10691296 }
10701297 kref_get(&zhdr->refcount);
1071− queue_work_on(zhdr->cpu, pool->compact_wq, &zhdr->work);
10721298 clear_bit(PAGE_CLAIMED, &page->private);
1073− z3fold_page_unlock(zhdr);
1299+ queue_work_on(zhdr->cpu, pool->compact_wq, &zhdr->work);
1300+ put_z3fold_header(zhdr);
10741301 }
10751302
10761303 /**
@@ -1111,11 +1338,10 @@ static void z3fold_free(struct z3fold_pool *pool, unsigned long handle) · 34 unmodified lines
11111338 */
11121339 static int z3fold_reclaim_page(struct z3fold_pool *pool, unsigned int retries)
11131340 {
1114− int i, ret = 0;
1341+ int i, ret = -1;
11151342 struct z3fold_header *zhdr = NULL;
11161343 struct page *page = NULL;
11171344 struct list_head *pos;
1118− struct z3fold_buddy_slots slots;
11191345 unsigned long first_handle = 0, middle_handle = 0, last_handle = 0;
11201346
11211347 spin_lock(&pool->lock);
@@ -1153,6 +1379,12 @@ static int z3fold_reclaim_page(struct z3fold_pool *pool, unsigned int retries) · 31 unmodified lines
11531379 zhdr = NULL;
11541380 continue; /* can't evict at this point */
11551381 }
1382+ if (zhdr->foreign_handles) {
1383+ clear_bit(PAGE_CLAIMED, &page->private);
1384+ z3fold_page_unlock(zhdr);
1385+ zhdr = NULL;
1386+ continue; /* can't evict such page */
1387+ }
11561388 kref_get(&zhdr->refcount);
11571389 list_del_init(&zhdr->buddy);
11581390 zhdr->cpu = -1;
@@ -1176,39 +1408,38 @@ static int z3fold_reclaim_page(struct z3fold_pool *pool, unsigned int retries) · 17 unmodified lines
11761408 last_handle = 0;
11771409 middle_handle = 0;
11781410 if (zhdr->first_chunks)
1179− first_handle = __encode_handle(zhdr, &slots,
1180− FIRST);
1411+ first_handle = encode_handle(zhdr, FIRST);
11811412 if (zhdr->middle_chunks)
1182− middle_handle = __encode_handle(zhdr, &slots,
1183− MIDDLE);
1413+ middle_handle = encode_handle(zhdr, MIDDLE);
11841414 if (zhdr->last_chunks)
1185− last_handle = __encode_handle(zhdr, &slots,
1186− LAST);
1415+ last_handle = encode_handle(zhdr, LAST);
11871416 /*
11881417 * it's safe to unlock here because we hold a
11891418 * reference to this page
11901419 */
11911420 z3fold_page_unlock(zhdr);
11921421 } else {
1193− first_handle = __encode_handle(zhdr, &slots, HEADLESS);
1422+ first_handle = encode_handle(zhdr, HEADLESS);
11941423 last_handle = middle_handle = 0;
11951424 }
1196−
11971425 /* Issue the eviction callback(s) */
11981426 if (middle_handle) {
11991427 ret = pool->ops->evict(pool, middle_handle);
12001428 if (ret)
12011429 goto next;
1430+ free_handle(middle_handle);
12021431 }
12031432 if (first_handle) {
12041433 ret = pool->ops->evict(pool, first_handle);
12051434 if (ret)
12061435 goto next;
1436+ free_handle(first_handle);
12071437 }
12081438 if (last_handle) {
12091439 ret = pool->ops->evict(pool, last_handle);
12101440 if (ret)
12111441 goto next;
1442+ free_handle(last_handle);
12121443 }
12131444 next:
12141445 if (test_bit(PAGE_HEADLESS, &page->private)) {
@@ -1264,14 +1495,13 @@ static void *z3fold_map(struct z3fold_pool *pool, unsigned long handle) · 49 unmodified lines
12641495 void *addr;
12651496 enum buddy buddy;
12661497
1267− zhdr = handle_to_z3fold_header(handle);
1498+ zhdr = get_z3fold_header(handle);
12681499 addr = zhdr;
12691500 page = virt_to_page(zhdr);
12701501
12711502 if (test_bit(PAGE_HEADLESS, &page->private))
12721503 goto out;
12731504
1274− z3fold_page_lock(zhdr);
12751505 buddy = handle_to_buddy(handle);
12761506 switch (buddy) {
12771507 case FIRST:
@@ -1293,8 +1523,8 @@ static void *z3fold_map(struct z3fold_pool *pool, unsigned long handle) · 15 unmodified lines
12931523
12941524 if (addr)
12951525 zhdr->mapped_count++;
1296− z3fold_page_unlock(zhdr);
12971526 out:
1527+ put_z3fold_header(zhdr);
12981528 return addr;
12991529 }
13001530
@@ -1309,18 +1539,17 @@ static void z3fold_unmap(struct z3fold_pool *pool, unsigned long handle) · 8 unmodified lines
13091539 struct page *page;
13101540 enum buddy buddy;
13111541
1312− zhdr = handle_to_z3fold_header(handle);
1542+ zhdr = get_z3fold_header(handle);
13131543 page = virt_to_page(zhdr);
13141544
13151545 if (test_bit(PAGE_HEADLESS, &page->private))
13161546 return;
13171547
1318− z3fold_page_lock(zhdr);
13191548 buddy = handle_to_buddy(handle);
13201549 if (buddy == MIDDLE)
13211550 clear_bit(MIDDLE_CHUNK_MAPPED, &page->private);
13221551 zhdr->mapped_count--;
1323− z3fold_page_unlock(zhdr);
1552+ put_z3fold_header(zhdr);
13241553 }
13251554
13261555 /**
@@ -1352,19 +1581,21 @@ static bool z3fold_page_isolate(struct page *page, isolate_mode_t mode) · 25 unmodified lines
13521581 test_bit(PAGE_STALE, &page->private))
13531582 goto out;
13541583
1584+ if (zhdr->mapped_count != 0 || zhdr->foreign_handles != 0)
1585+ goto out;
1586+
13551587 pool = zhdr_to_pool(zhdr);
1588+ spin_lock(&pool->lock);
1589+ if (!list_empty(&zhdr->buddy))
1590+ list_del_init(&zhdr->buddy);
1591+ if (!list_empty(&page->lru))
1592+ list_del_init(&page->lru);
1593+ spin_unlock(&pool->lock);
1594+
1595+ kref_get(&zhdr->refcount);
1596+ z3fold_page_unlock(zhdr);
1597+ return true;
13561598
1357− if (zhdr->mapped_count == 0) {
1358− kref_get(&zhdr->refcount);
1359− if (!list_empty(&zhdr->buddy))
1360− list_del_init(&zhdr->buddy);
1361− spin_lock(&pool->lock);
1362− if (!list_empty(&page->lru))
1363− list_del(&page->lru);
1364− spin_unlock(&pool->lock);
1365− z3fold_page_unlock(zhdr);
1366− return true;
1367− }
13681599 out:
13691600 z3fold_page_unlock(zhdr);
13701601 return false;
@@ -1387,7 +1618,7 @@ static int z3fold_page_migrate(struct address_space *mapping, struct page *newpa · 16 unmodified lines
13871618 if (!z3fold_page_trylock(zhdr)) {
13881619 return -EAGAIN;
13891620 }
1390− if (zhdr->mapped_count != 0) {
1621+ if (zhdr->mapped_count != 0 || zhdr->foreign_handles != 0) {
13911622 z3fold_page_unlock(zhdr);
13921623 return -EBUSY;
13931624 }
scripts/spelling.txt+28−0
@@ -87,6 +87,7 @@ algorith||algorithm · 86 unmodified lines
8787 algorithmical||algorithmically
8888 algoritm||algorithm
8989 algoritms||algorithms
90+algorithmn||algorithm
9091 algorrithm||algorithm
9192 algorritm||algorithm
9293 aligment||alignment
@@ -109,6 +110,7 @@ alredy||already · 16 unmodified lines
109110 altough||although
110111 alue||value
111112 ambigious||ambiguous
113+ambigous||ambiguous
112114 amoung||among
113115 amout||amount
114116 amplifer||amplifier
@@ -179,6 +181,7 @@ attepmpt||attempt · 64 unmodified lines
179181 attnetion||attention
180182 attruibutes||attributes
181183 authentification||authentication
184+authenicated||authenticated
182185 automaticaly||automatically
183186 automaticly||automatically
184187 automatize||automate
@@ -286,6 +289,7 @@ claread||cleared · 101 unmodified lines
286289 clared||cleared
287290 closeing||closing
288291 clustred||clustered
292+cnfiguration||configuration
289293 coexistance||coexistence
290294 colescing||coalescing
291295 collapsable||collapsible
@@ -325,9 +329,11 @@ comression||compression · 33 unmodified lines
325329 comunication||communication
326330 conbination||combination
327331 conditionaly||conditionally
332+conditon||condition
328333 conected||connected
329334 conector||connector
330335 connecetd||connected
336+configration||configuration
331337 configuartion||configuration
332338 configuation||configuration
333339 configued||configured
@@ -347,6 +353,7 @@ containts||contains · 13 unmodified lines
347353 contaisn||contains
348354 contant||contact
349355 contence||contents
356+contiguos||contiguous
350357 continious||continuous
351358 continous||continuous
352359 continously||continuously
@@ -380,6 +387,7 @@ cylic||cyclic · 27 unmodified lines
380387 dafault||default
381388 deafult||default
382389 deamon||daemon
390+debouce||debounce
383391 decompres||decompress
384392 decsribed||described
385393 decription||description
@@ -448,6 +456,7 @@ diffrent||different · 62 unmodified lines
448456 differenciate||differentiate
449457 diffrentiate||differentiate
450458 difinition||definition
459+digial||digital
451460 dimention||dimension
452461 dimesions||dimensions
453462 dispalying||displaying
@@ -489,6 +498,7 @@ droput||dropout · 35 unmodified lines
489498 druing||during
490499 dynmaic||dynamic
491500 eanable||enable
501+eanble||enable
492502 easilly||easily
493503 ecspecially||especially
494504 edditable||editable
@@ -502,6 +512,7 @@ elementry||elementary · 7 unmodified lines
502512 eletronic||electronic
503513 embeded||embedded
504514 enabledi||enabled
515+enbale||enable
505516 enble||enable
506517 enchanced||enhanced
507518 encorporating||incorporating
@@ -536,6 +547,7 @@ excellant||excellent · 28 unmodified lines
536547 execeeded||exceeded
537548 execeeds||exceeds
538549 exeed||exceed
550+exeuction||execution
539551 existance||existence
540552 existant||existent
541553 exixt||exist
@@ -601,10 +613,12 @@ frambuffer||framebuffer · 59 unmodified lines
601613 framming||framing
602614 framwork||framework
603615 frequncy||frequency
616+frequancy||frequency
604617 frome||from
605618 fucntion||function
606619 fuction||function
607620 fuctions||functions
621+fullill||fulfill
608622 funcation||function
609623 funcion||function
610624 functionallity||functionality
@@ -642,6 +656,7 @@ happend||happened · 31 unmodified lines
642656 harware||hardware
643657 heirarchically||hierarchically
644658 helpfull||helpful
659+hexdecimal||hexadecimal
645660 hybernate||hibernate
646661 hierachy||hierarchy
647662 hierarchie||hierarchy
@@ -709,12 +724,14 @@ initalize||initialize · 61 unmodified lines
709724 initation||initiation
710725 initators||initiators
711726 initialiazation||initialization
727+initializationg||initialization
712728 initializiation||initialization
713729 initialze||initialize
714730 initialzed||initialized
715731 initialzing||initializing
716732 initilization||initialization
717733 initilize||initialize
734+initliaze||initialize
718735 inofficial||unofficial
719736 inrerface||interface
720737 insititute||institute
@@ -779,6 +796,7 @@ itertation||iteration · 58 unmodified lines
779796 itslef||itself
780797 jave||java
781798 jeffies||jiffies
799+jumpimng||jumping
782800 juse||just
783801 jus||just
784802 kown||known
@@ -839,6 +857,7 @@ messags||messages · 54 unmodified lines
839857 messgaes||messages
840858 messsage||message
841859 messsages||messages
860+metdata||metadata
842861 micropone||microphone
843862 microprocesspr||microprocessor
844863 migrateable||migratable
@@ -857,6 +876,7 @@ mismactch||mismatch · 12 unmodified lines
857876 missign||missing
858877 missmanaged||mismanaged
859878 missmatch||mismatch
879+misssing||missing
860880 miximum||maximum
861881 mmnemonic||mnemonic
862882 mnay||many
@@ -912,6 +932,7 @@ occured||occurred · 49 unmodified lines
912932 occuring||occurring
913933 offser||offset
914934 offet||offset
935+offlaod||offload
915936 offloded||offloaded
916937 offseting||offsetting
917938 omited||omitted
@@ -993,6 +1014,7 @@ poiter||pointer · 75 unmodified lines
9931014 posible||possible
9941015 positon||position
9951016 possibilites||possibilities
1017+potocol||protocol
9961018 powerfull||powerful
9971019 pramater||parameter
9981020 preamle||preamble
@@ -1061,11 +1083,13 @@ psychadelic||psychedelic · 62 unmodified lines
10611083 pwoer||power
10621084 queing||queuing
10631085 quering||querying
1086+queus||queues
10641087 randomally||randomly
10651088 raoming||roaming
10661089 reasearcher||researcher
10671090 reasearchers||researchers
10681091 reasearch||research
1092+receieve||receive
10691093 recepient||recipient
10701094 recevied||received
10711095 receving||receiving
@@ -1166,6 +1190,7 @@ scaleing||scaling · 94 unmodified lines
11661190 scaned||scanned
11671191 scaning||scanning
11681192 scarch||search
1193+schdule||schedule
11691194 seach||search
11701195 searchs||searches
11711196 secquence||sequence
@@ -1308,6 +1333,7 @@ taskelt||tasklet · 136 unmodified lines
13081333 teh||the
13091334 temorary||temporary
13101335 temproarily||temporarily
1336+temperture||temperature
13111337 thead||thread
13121338 therfore||therefore
13131339 thier||their
@@ -1354,6 +1380,7 @@ uknown||unknown · 40 unmodified lines
13541380 usupported||unsupported
13551381 uncommited||uncommitted
13561382 unconditionaly||unconditionally
1383+undeflow||underflow
13571384 underun||underrun
13581385 unecessary||unnecessary
13591386 unexecpted||unexpected
@@ -1414,6 +1441,7 @@ varible||variable · 54 unmodified lines
14141441 varient||variant
14151442 vaule||value
14161443 verbse||verbose
1444+veify||verify
14171445 verisons||versions
14181446 verison||version
14191447 verson||version
tools/testing/selftests/memfd/memfd_test.c+36−0
@@ -290,6 +290,40 @@ static void mfd_assert_read_shared(int fd) · 289 unmodified lines
290290 munmap(p, mfd_def_size);
291291 }
292292
293+static void mfd_assert_fork_private_write(int fd)
294+{
295+ int *p;
296+ pid_t pid;
297+
298+ p = mmap(NULL,
299+ mfd_def_size,
300+ PROT_READ | PROT_WRITE,
301+ MAP_PRIVATE,
302+ fd,
303+ 0);
304+ if (p == MAP_FAILED) {
305+ printf("mmap() failed: %m\n");
306+ abort();
307+ }
308+
309+ p[0] = 22;
310+
311+ pid = fork();
312+ if (pid == 0) {
313+ p[0] = 33;
314+ exit(0);
315+ } else {
316+ waitpid(pid, NULL, 0);
317+
318+ if (p[0] != 22) {
319+ printf("MAP_PRIVATE copy-on-write failed: %m\n");
320+ abort();
321+ }
322+ }
323+
324+ munmap(p, mfd_def_size);
325+}
326+
293327 static void mfd_assert_write(int fd)
294328 {
295329 ssize_t l;
@@ -760,6 +794,8 @@ static void test_seal_future_write(void) · 464 unmodified lines
760794 mfd_assert_read_shared(fd2);
761795 mfd_fail_write(fd2);
762796
797+ mfd_assert_fork_private_write(fd);
798+
763799 munmap(p, mfd_def_size);
764800 close(fd2);
765801 close(fd);
tools/testing/selftests/vm/config+1−0
@@ -1,2 +1,3 @@
11 CONFIG_SYSVIPC=y
22 CONFIG_USERFAULTFD=y
3+CONFIG_TEST_VMALLOC=m

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t