lael-labs/linux

Documentation/admin-guide/cgroup-v2.rst+6−1

 	  inactive_anon, active_anon, inactive_file, active_file, unevictable
 		Amount of memory, swap-backed and filesystem-backed,
 		on the internal memory management lists used by the
-		page reclaim algorithm
+		page reclaim algorithm.
+		As these represent internal list state (eg. shmem pages are on anon
+		memory management lists), inactive_foo + active_foo may not be equal to
+		the value for the foo counter, since the foo counter is type-based, not
+		list-based.
 	  slab_reclaimable
 		Part of "slab" that might be reclaimed, such as

Documentation/dev-tools/kasan.rst+63−0

@@ -218,3 +218,66 @@ brk handler is used to print bug reports. · 217 unmodified lines
218	A potential expansion of this mode is a hardware tag-based mode, which would	218	A potential expansion of this mode is a hardware tag-based mode, which would
219	use hardware memory tagging support instead of compiler instrumentation and	219	use hardware memory tagging support instead of compiler instrumentation and
220	manual shadow memory manipulation.	220	manual shadow memory manipulation.
		221
		222	What memory accesses are sanitised by KASAN?
		223	--------------------------------------------
		224
		225	The kernel maps memory in a number of different parts of the address
		226	space. This poses something of a problem for KASAN, which requires
		227	that all addresses accessed by instrumented code have a valid shadow
		228	region.
		229
		230	The range of kernel virtual addresses is large: there is not enough
		231	real memory to support a real shadow region for every address that
		232	could be accessed by the kernel.
		233
		234	By default
		235	~~~~~~~~~~
		236
		237	By default, architectures only map real memory over the shadow region
		238	for the linear mapping (and potentially other small areas). For all
		239	other areas - such as vmalloc and vmemmap space - a single read-only
		240	page is mapped over the shadow area. This read-only shadow page
		241	declares all memory accesses as permitted.
		242
		243	This presents a problem for modules: they do not live in the linear
		244	mapping, but in a dedicated module space. By hooking in to the module
		245	allocator, KASAN can temporarily map real shadow memory to cover
		246	them. This allows detection of invalid accesses to module globals, for
		247	example.
		248
		249	This also creates an incompatibility with ``VMAP_STACK``: if the stack
		250	lives in vmalloc space, it will be shadowed by the read-only page, and
		251	the kernel will fault when trying to set up the shadow data for stack
		252	variables.
		253
		254	CONFIG_KASAN_VMALLOC
		255	~~~~~~~~~~~~~~~~~~~~
		256
		257	With ``CONFIG_KASAN_VMALLOC``, KASAN can cover vmalloc space at the
		258	cost of greater memory usage. Currently this is only supported on x86.
		259
		260	This works by hooking into vmalloc and vmap, and dynamically
		261	allocating real shadow memory to back the mappings.
		262
		263	Most mappings in vmalloc space are small, requiring less than a full
		264	page of shadow space. Allocating a full shadow page per mapping would
		265	therefore be wasteful. Furthermore, to ensure that different mappings
		266	use different shadow pages, mappings would have to be aligned to
		267	``KASAN_SHADOW_SCALE_SIZE * PAGE_SIZE``.
		268
		269	Instead, we share backing space across multiple mappings. We allocate
		270	a backing page when a mapping in vmalloc space uses a particular page
		271	of the shadow region. This page can be shared by other vmalloc
		272	mappings later on.
		273
		274	We hook in to the vmap infrastructure to lazily clean up unused shadow
		275	memory.
		276
		277	To avoid the difficulties around swapping mappings around, we expect
		278	that the part of the shadow region that covers the vmalloc space will
		279	not be covered by the early shadow page, but will be left
		280	unmapped. This will require changes in arch-specific code.
		281
		282	This allows ``VMAP_STACK`` support on x86, and can simplify support of
		283	architectures that do not have a fixed module region.

arch/Kconfig+5−4

 config VMAP_STACK
 	default y
 	bool "Use a virtually-mapped stack"
-	depends on HAVE_ARCH_VMAP_STACK && !KASAN
+	depends on HAVE_ARCH_VMAP_STACK
+	depends on !KASAN || KASAN_VMALLOC
 	---help---
 	  Enable this if you want the use virtually-mapped kernel stacks
 	  with guard pages.  This causes kernel stack overflows to be
 	  caught immediately rather than causing difficult-to-diagnose
 	  corruption.
-	  This is presently incompatible with KASAN because KASAN expects
+	  To use this with KASAN, the architecture must support backing
-	  the stack to map directly to the KASAN shadow map using a formula
+	  virtual mappings with real shadow memory, and KASAN_VMALLOC must
-	  that is incorrect if the stack is in vmalloc space.
+	  be enabled.
 config ARCH_OPTIONAL_KERNEL_RWX
 	def_bool n

arch/arc/include/asm/pgtable.h+0−1

@@ -33,7 +33,6 @@ · 32 unmodified lines
33	#define _ASM_ARC_PGTABLE_H	33	#define _ASM_ARC_PGTABLE_H
34		34
35	#include <linux/bits.h>	35	#include <linux/bits.h>
36	#define __ARCH_USE_5LEVEL_HACK
37	#include <asm-generic/pgtable-nopmd.h>	36	#include <asm-generic/pgtable-nopmd.h>
38	#include <asm/page.h>	37	#include <asm/page.h>
39	#include <asm/mmu.h> /* to propagate CONFIG_ARC_MMU_VER <n> */	38	#include <asm/mmu.h> /* to propagate CONFIG_ARC_MMU_VER <n> */

arch/arc/mm/fault.c+8−2

 	 * with the 'reference' page table.
 	 */
 	pgd_t *pgd, *pgd_k;
+	p4d_t *p4d, *p4d_k;
 	pud_t *pud, *pud_k;
 	pmd_t *pmd, *pmd_k;
 	if (!pgd_present(*pgd_k))
 		goto bad_area;
-	pud = pud_offset(pgd, address);
+	p4d = p4d_offset(pgd, address);
-	pud_k = pud_offset(pgd_k, address);
+	p4d_k = p4d_offset(pgd_k, address);
+	if (!p4d_present(*p4d_k))
+		goto bad_area;
+	pud = pud_offset(p4d, address);
+	pud_k = pud_offset(p4d_k, address);
 	if (!pud_present(*pud_k))
 		goto bad_area;

arch/arc/mm/highmem.c+3−1

 static noinline pte_t * __init alloc_kmap_pgtable(unsigned long kvaddr)
 {
 	pgd_t *pgd_k;
+	p4d_t *p4d_k;
 	pud_t *pud_k;
 	pmd_t *pmd_k;
 	pte_t *pte_k;
 	pgd_k = pgd_offset_k(kvaddr);
-	pud_k = pud_offset(pgd_k, kvaddr);
+	p4d_k = p4d_offset(pgd_k, kvaddr);
+	pud_k = pud_offset(p4d_k, kvaddr);
 	pmd_k = pmd_offset(pud_k, kvaddr);
 	pte_k = (pte_t *)memblock_alloc_low(PAGE_SIZE, PAGE_SIZE);

arch/powerpc/include/asm/book3s/64/pgtable-4k.h+0−3

@@ -70,9 +70,6 @@ static inline int get_hugepd_cache_index(int index) · 69 unmodified lines
70	/* should not reach */	70	/* should not reach */
71	}	71	}
72		72
73	#else /* !CONFIG_HUGETLB_PAGE */
74	static inline int pmd_huge(pmd_t pmd) { return 0; }
75	static inline int pud_huge(pud_t pud) { return 0; }
76	#endif /* CONFIG_HUGETLB_PAGE */	73	#endif /* CONFIG_HUGETLB_PAGE */
77		74
78	#endif /* __ASSEMBLY__ */	75	#endif /* __ASSEMBLY__ */

arch/powerpc/include/asm/book3s/64/pgtable-64k.h+0−3

@@ -59,9 +59,6 @@ static inline int get_hugepd_cache_index(int index) · 58 unmodified lines
59	BUG();	59	BUG();
60	}	60	}
61		61
62	#else /* !CONFIG_HUGETLB_PAGE */
63	static inline int pmd_huge(pmd_t pmd) { return 0; }
64	static inline int pud_huge(pud_t pud) { return 0; }
65	#endif /* CONFIG_HUGETLB_PAGE */	62	#endif /* CONFIG_HUGETLB_PAGE */
66		63
67	static inline int remap_4k_pfn(struct vm_area_struct *vma, unsigned long addr,	64	static inline int remap_4k_pfn(struct vm_area_struct *vma, unsigned long addr,

arch/powerpc/mm/book3s64/radix_pgtable.c+1−0

@@ -13,6 +13,7 @@ · 12 unmodified lines
13	#include <linux/memblock.h>	13	#include <linux/memblock.h>
14	#include <linux/of_fdt.h>	14	#include <linux/of_fdt.h>
15	#include <linux/mm.h>	15	#include <linux/mm.h>
		16	#include <linux/hugetlb.h>
16	#include <linux/string_helpers.h>	17	#include <linux/string_helpers.h>
17	#include <linux/stop_machine.h>	18	#include <linux/stop_machine.h>
18		19

arch/x86/Kconfig+1−0

@@ -134,6 +134,7 @@ config X86 · 133 unmodified lines
134	select HAVE_ARCH_JUMP_LABEL	134	select HAVE_ARCH_JUMP_LABEL
135	select HAVE_ARCH_JUMP_LABEL_RELATIVE	135	select HAVE_ARCH_JUMP_LABEL_RELATIVE
136	select HAVE_ARCH_KASAN if X86_64	136	select HAVE_ARCH_KASAN if X86_64
		137	select HAVE_ARCH_KASAN_VMALLOC if X86_64
137	select HAVE_ARCH_KGDB	138	select HAVE_ARCH_KGDB
138	select HAVE_ARCH_MMAP_RND_BITS if MMU	139	select HAVE_ARCH_MMAP_RND_BITS if MMU
139	select HAVE_ARCH_MMAP_RND_COMPAT_BITS if MMU && COMPAT	140	select HAVE_ARCH_MMAP_RND_COMPAT_BITS if MMU && COMPAT

arch/x86/mm/kasan_init_64.c+61−0

@@ -245,6 +245,49 @@ static void __init kasan_map_early_shadow(pgd_t *pgd) · 244 unmodified lines
245	} while (pgd++, addr = next, addr != end);	245	} while (pgd++, addr = next, addr != end);
246	}	246	}
247		247
		248	static void __init kasan_shallow_populate_p4ds(pgd_t *pgd,
		249	unsigned long addr,
		250	unsigned long end)
		251	{
		252	p4d_t *p4d;
		253	unsigned long next;
		254	void *p;
		255
		256	p4d = p4d_offset(pgd, addr);
		257	do {
		258	next = p4d_addr_end(addr, end);
		259
		260	if (p4d_none(*p4d)) {
		261	p = early_alloc(PAGE_SIZE, NUMA_NO_NODE, true);
		262	p4d_populate(&init_mm, p4d, p);
		263	}
		264	} while (p4d++, addr = next, addr != end);
		265	}
		266
		267	static void __init kasan_shallow_populate_pgds(void start, void end)
		268	{
		269	unsigned long addr, next;
		270	pgd_t *pgd;
		271	void *p;
		272
		273	addr = (unsigned long)start;
		274	pgd = pgd_offset_k(addr);
		275	do {
		276	next = pgd_addr_end(addr, (unsigned long)end);
		277
		278	if (pgd_none(*pgd)) {
		279	p = early_alloc(PAGE_SIZE, NUMA_NO_NODE, true);
		280	pgd_populate(&init_mm, pgd, p);
		281	}
		282
		283	/*
		284	* we need to populate p4ds to be synced when running in
		285	* four level mode - see sync_global_pgds_l4()
		286	*/
		287	kasan_shallow_populate_p4ds(pgd, addr, next);
		288	} while (pgd++, addr = next, addr != (unsigned long)end);
		289	}
		290
248	#ifdef CONFIG_KASAN_INLINE	291	#ifdef CONFIG_KASAN_INLINE
249	static int kasan_die_handler(struct notifier_block *self,	292	static int kasan_die_handler(struct notifier_block *self,
250	unsigned long val,	293	unsigned long val,
@@ -354,6 +397,24 @@ void __init kasan_init(void) · 103 unmodified lines
354		397
355	kasan_populate_early_shadow(	398	kasan_populate_early_shadow(
356	kasan_mem_to_shadow((void *)PAGE_OFFSET + MAXMEM),	399	kasan_mem_to_shadow((void *)PAGE_OFFSET + MAXMEM),
		400	kasan_mem_to_shadow((void *)VMALLOC_START));
		401
		402	/*
		403	* If we're in full vmalloc mode, don't back vmalloc space with early
		404	* shadow pages. Instead, prepopulate pgds/p4ds so they are synced to
		405	* the global table and we can populate the lower levels on demand.
		406	*/
		407	if (IS_ENABLED(CONFIG_KASAN_VMALLOC))
		408	kasan_shallow_populate_pgds(
		409	kasan_mem_to_shadow((void *)VMALLOC_START),
		410	kasan_mem_to_shadow((void *)VMALLOC_END));
		411	else
		412	kasan_populate_early_shadow(
		413	kasan_mem_to_shadow((void *)VMALLOC_START),
		414	kasan_mem_to_shadow((void *)VMALLOC_END));
		415
		416	kasan_populate_early_shadow(
		417	kasan_mem_to_shadow((void *)VMALLOC_END + 1),
357	shadow_cpu_entry_begin);	418	shadow_cpu_entry_begin);
358		419
359	kasan_populate_shadow((unsigned long)shadow_cpu_entry_begin,	420	kasan_populate_shadow((unsigned long)shadow_cpu_entry_begin,

drivers/base/memory.c+15−25

 #include <linux/memory.h>
 #include <linux/memory_hotplug.h>
 #include <linux/mm.h>
-#include <linux/mutex.h>
 #include <linux/stat.h>
 #include <linux/slab.h>
 #include <linux/atomic.h>
 #include <linux/uaccess.h>
-static DEFINE_MUTEX(mem_sysfs_mutex);
 #define MEMORY_CLASS_NAME	"memory"
 #define to_memory_block(dev) container_of(dev, struct memory_block, dev)
 	if (kstrtoull(buf, 0, &pfn) < 0)
 		return -EINVAL;
 	pfn >>= PAGE_SHIFT;
-	if (!pfn_valid(pfn))
+	ret = soft_offline_page(pfn, 0);
-		return -ENXIO;
-	/* Only online pages can be soft-offlined (esp., not ZONE_DEVICE). */
-	if (!pfn_to_online_page(pfn))
-		return -EIO;
-	ret = soft_offline_page(pfn_to_page(pfn), 0);
 	return ret == 0 ? count : ret;
 }
  * Create memory block devices for the given memory area. Start and size
  * have to be aligned to memory block granularity. Memory block devices
  * will be initialized as offline.
+ *
+ * Called under device_hotplug_lock.
  */
 int create_memory_block_devices(unsigned long start, unsigned long size)
 {
 			 !IS_ALIGNED(size, memory_block_size_bytes())))
 		return -EINVAL;
-	mutex_lock(&mem_sysfs_mutex);
 	for (block_id = start_block_id; block_id != end_block_id; block_id++) {
 		ret = init_memory_block(&mem, block_id, MEM_OFFLINE);
 		if (ret)
 		for (block_id = start_block_id; block_id != end_block_id;
 		     block_id++) {
 			mem = find_memory_block_by_id(block_id);
+			if (WARN_ON_ONCE(!mem))
+				continue;
 			mem->section_count = 0;
 			unregister_memory(mem);
 		}
 	}
-	mutex_unlock(&mem_sysfs_mutex);
 	return ret;
 }
  * Remove memory block devices for the given memory area. Start and size
  * have to be aligned to memory block granularity. Memory block devices
  * have to be offline.
+ *
+ * Called under device_hotplug_lock.
  */
 void remove_memory_block_devices(unsigned long start, unsigned long size)
 {
 			 !IS_ALIGNED(size, memory_block_size_bytes())))
 		return;
-	mutex_lock(&mem_sysfs_mutex);
 	for (block_id = start_block_id; block_id != end_block_id; block_id++) {
 		mem = find_memory_block_by_id(block_id);
 		if (WARN_ON_ONCE(!mem))
 		unregister_memory_block_under_nodes(mem);
 		unregister_memory(mem);
 	}
-	mutex_unlock(&mem_sysfs_mutex);
 }
 /* return true if the memory block is offlined, otherwise, return false */
 };
 /*
- * Initialize the sysfs support for memory devices...
+ * Initialize the sysfs support for memory devices. At the time this function
+ * is called, we cannot have concurrent creation/deletion of memory block
+ * devices, the device_hotplug_lock is not needed.
  */
 void __init memory_dev_init(void)
 {
 	int ret;
-	int err;
 	unsigned long block_sz, nr;
 	/* Validate the configured memory block size */
 	ret = subsys_system_register(&memory_subsys, memory_root_attr_groups);
 	if (ret)
-		goto out;
+		panic("%s() failed to register subsystem: %d\n", __func__, ret);
 	/*
 	 * Create entries for memory sections that were found
 	 * during boot and have been initialized
 	 */
-	mutex_lock(&mem_sysfs_mutex);
 	for (nr = 0; nr <= __highest_present_section_nr;
 	     nr += sections_per_block) {
-		err = add_memory_block(nr);
+		ret = add_memory_block(nr);
-		if (!ret)
+		if (ret)
-			ret = err;
+			panic("%s() failed to add memory block: %d\n", __func__,
+			      ret);
 	}
-	mutex_unlock(&mem_sysfs_mutex);
-out:
-	if (ret)
-		panic("%s() failed: %d\n", __func__, ret);
 }
 /**

drivers/hv/hv_balloon.c+1−3

@@ -682,9 +682,7 @@ static void hv_page_online_one(struct hv_hotadd_state has, struct page pg) · 681 unmodified lines
682	__ClearPageOffline(pg);	682	__ClearPageOffline(pg);
683		683
684	/* This frame is currently backed; online the page. */	684	/* This frame is currently backed; online the page. */
685	__online_page_set_limits(pg);	685	generic_online_page(pg, 0);
686	__online_page_increment_counters(pg);
687	__online_page_free(pg);
688		686
689	lockdep_assert_held(&dm_device.ha_lock);	687	lockdep_assert_held(&dm_device.ha_lock);
690	dm_device.num_pages_onlined++;	688	dm_device.num_pages_onlined++;

drivers/xen/balloon.c+0−1

@@ -374,7 +374,6 @@ static void xen_online_page(struct page *page, unsigned int order) · 373 unmodified lines
374	mutex_lock(&balloon_mutex);	374	mutex_lock(&balloon_mutex);
375	for (i = 0; i < size; i++) {	375	for (i = 0; i < size; i++) {
376	p = pfn_to_page(start_pfn + i);	376	p = pfn_to_page(start_pfn + i);
377	__online_page_set_limits(p);
378	balloon_append(p);	377	balloon_append(p);
379	}	378	}
380	mutex_unlock(&balloon_mutex);	379	mutex_unlock(&balloon_mutex);

fs/buffer.c+4−2

 #include <trace/events/block.h>
 #include <linux/fscrypt.h>
+#include "internal.h"
 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
 static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh,
 			 enum rw_hint hint, struct writeback_control *wbc);
 	for (i = 0; i < BH_LRU_SIZE; i++) {
 		if (b->bhs[i])
-			return 1;
+			return true;
 	}
-	return 0;
+	return false;
 }
 void invalidate_bh_lrus(void)

fs/direct-io.c+0−21

@@ -220,27 +220,6 @@ static inline struct page dio_get_page(struct dio dio, · 219 unmodified lines
220	return dio->pages[sdio->head];	220	return dio->pages[sdio->head];
221	}	221	}
222		222
223	/*
224	* Warn about a page cache invalidation failure during a direct io write.
225	*/
226	void dio_warn_stale_pagecache(struct file *filp)
227	{
228	static DEFINE_RATELIMIT_STATE(_rs, 86400 * HZ, DEFAULT_RATELIMIT_BURST);
229	char pathname[128];
230	struct inode *inode = file_inode(filp);
231	char *path;
232
233	errseq_set(&inode->i_mapping->wb_err, -EIO);
234	if (__ratelimit(&_rs)) {
235	path = file_path(filp, pathname, sizeof(pathname));
236	if (IS_ERR(path))
237	path = "(unknown)";
238	pr_crit("Page cache invalidation failure on direct I/O. Possible data corruption due to collision with buffered I/O!\n");
239	pr_crit("File: %s PID: %d Comm: %.20s\n", path, current->pid,
240	current->comm);
241	}
242	}
243
244	/*	223	/*
245	* dio_complete() - called when all DIO BIO I/O has been completed	224	* dio_complete() - called when all DIO BIO I/O has been completed
246	*	225	*

fs/hugetlbfs/inode.c+48−15

 			u32 hash;
 			index = page->index;
-			hash = hugetlb_fault_mutex_hash(h, mapping, index, 0);
+			hash = hugetlb_fault_mutex_hash(mapping, index);
 			mutex_lock(&hugetlb_fault_mutex_table[hash]);
 			/*
 		addr = index * hpage_size;
 		/* mutex taken here, fault path and hole punch */
-		hash = hugetlb_fault_mutex_hash(h, mapping, index, addr);
+		hash = hugetlb_fault_mutex_hash(mapping, index);
 		mutex_lock(&hugetlb_fault_mutex_table[hash]);
 		/* See if already present in mapping to avoid alloc/free */
 /*
  * File creation. Allocate an inode, and we're done..
  */
-static int hugetlbfs_mknod(struct inode *dir,
+static int do_hugetlbfs_mknod(struct inode *dir,
-			struct dentry *dentry, umode_t mode, dev_t dev)
+			struct dentry *dentry,
+			umode_t mode,
+			dev_t dev,
+			bool tmpfile)
 {
 	struct inode *inode;
 	int error = -ENOSPC;
 	inode = hugetlbfs_get_inode(dir->i_sb, dir, mode, dev);
 	if (inode) {
 		dir->i_ctime = dir->i_mtime = current_time(dir);
-		d_instantiate(dentry, inode);
+		if (tmpfile) {
-		dget(dentry);	/* Extra count - pin the dentry in core */
+			d_tmpfile(dentry, inode);
+		} else {
+			d_instantiate(dentry, inode);
+			dget(dentry);/* Extra count - pin the dentry in core */
+		}
 		error = 0;
 	}
 	return error;
 }
+static int hugetlbfs_mknod(struct inode *dir,
+			struct dentry *dentry, umode_t mode, dev_t dev)
+{
+	return do_hugetlbfs_mknod(dir, dentry, mode, dev, false);
+}
 static int hugetlbfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
 {
 	int retval = hugetlbfs_mknod(dir, dentry, mode | S_IFDIR, 0);
 	return hugetlbfs_mknod(dir, dentry, mode | S_IFREG, 0);
 }
+static int hugetlbfs_tmpfile(struct inode *dir,
+			struct dentry *dentry, umode_t mode)
+{
+	return do_hugetlbfs_mknod(dir, dentry, mode | S_IFREG, 0, true);
+}
 static int hugetlbfs_symlink(struct inode *dir,
 			struct dentry *dentry, const char *symname)
 {
 	.mknod		= hugetlbfs_mknod,
 	.rename		= simple_rename,
 	.setattr	= hugetlbfs_setattr,
+	.tmpfile	= hugetlbfs_tmpfile,
 };
 static const struct inode_operations hugetlbfs_inode_operations = {
 					sizeof(struct hugetlbfs_inode_info),
 					0, SLAB_ACCOUNT, init_once);
 	if (hugetlbfs_inode_cachep == NULL)
-		goto out2;
+		goto out;
 	error = register_filesystem(&hugetlbfs_fs_type);
 	if (error)
-		goto out;
+		goto out_free;
+	/* default hstate mount is required */
+	mnt = mount_one_hugetlbfs(&hstates[default_hstate_idx]);
+	if (IS_ERR(mnt)) {
+		error = PTR_ERR(mnt);
+		goto out_unreg;
+	}
+	hugetlbfs_vfsmount[default_hstate_idx] = mnt;
+	/* other hstates are optional */
 	i = 0;
 	for_each_hstate(h) {
+		if (i == default_hstate_idx)
+			continue;
 		mnt = mount_one_hugetlbfs(h);
-		if (IS_ERR(mnt) && i == 0) {
+		if (IS_ERR(mnt))
-			error = PTR_ERR(mnt);
+			hugetlbfs_vfsmount[i] = NULL;
-			goto out;
+		else
-		}
+			hugetlbfs_vfsmount[i] = mnt;
-		hugetlbfs_vfsmount[i] = mnt;
 		i++;
 	}
 	return 0;
- out:
+ out_unreg:
+	(void)unregister_filesystem(&hugetlbfs_fs_type);
+ out_free:
 	kmem_cache_destroy(hugetlbfs_inode_cachep);
- out2:
+ out:
 	return error;
 }
 fs_initcall(init_hugetlbfs_fs)

fs/ocfs2/acl.c+2−2

@@ -327,8 +327,8 @@ int ocfs2_acl_chmod(struct inode inode, struct buffer_head bh) · 326 unmodified lines
327	down_read(&OCFS2_I(inode)->ip_xattr_sem);	327	down_read(&OCFS2_I(inode)->ip_xattr_sem);
328	acl = ocfs2_get_acl_nolock(inode, ACL_TYPE_ACCESS, bh);	328	acl = ocfs2_get_acl_nolock(inode, ACL_TYPE_ACCESS, bh);
329	up_read(&OCFS2_I(inode)->ip_xattr_sem);	329	up_read(&OCFS2_I(inode)->ip_xattr_sem);
330	if (IS_ERR(acl) \|\| !acl)	330	if (IS_ERR_OR_NULL(acl))
331	return PTR_ERR(acl);	331	return PTR_ERR_OR_ZERO(acl);
332	ret = __posix_acl_chmod(&acl, GFP_KERNEL, inode->i_mode);	332	ret = __posix_acl_chmod(&acl, GFP_KERNEL, inode->i_mode);
333	if (ret)	333	if (ret)
334	return ret;	334	return ret;

fs/userfaultfd.c+13−8

 			start = vma->vm_start;
 		vma_end = min(end, vma->vm_end);
-		new_flags = (vma->vm_flags & ~vm_flags) | vm_flags;
+		new_flags = (vma->vm_flags &
+			     ~(VM_UFFD_MISSING|VM_UFFD_WP)) | vm_flags;
 		prev = vma_merge(mm, prev, start, vma_end, new_flags,
 				 vma->anon_vma, vma->vm_file, vma->vm_pgoff,
 				 vma_policy(vma),
 	if (copy_from_user(&uffdio_api, buf, sizeof(uffdio_api)))
 		goto out;
 	features = uffdio_api.features;
-	if (uffdio_api.api != UFFD_API || (features & ~UFFD_API_FEATURES)) {
+	ret = -EINVAL;
-		memset(&uffdio_api, 0, sizeof(uffdio_api));
+	if (uffdio_api.api != UFFD_API || (features & ~UFFD_API_FEATURES))
-		if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
+		goto err_out;
-			goto out;
+	ret = -EPERM;
-		ret = -EINVAL;
+	if ((features & UFFD_FEATURE_EVENT_FORK) && !capable(CAP_SYS_PTRACE))
-		goto out;
+		goto err_out;
-	}
 	/* report all available features and ioctls to userland */
 	uffdio_api.features = UFFD_API_FEATURES;
 	uffdio_api.ioctls = UFFD_API_IOCTLS;
 	ret = 0;
 out:
 	return ret;
+err_out:
+	memset(&uffdio_api, 0, sizeof(uffdio_api));
+	if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
+		ret = -EFAULT;
+	goto out;
 }
 static long userfaultfd_ioctl(struct file *file, unsigned cmd,

include/asm-generic/4level-fixup.h+0−1

@@ -30,7 +30,6 @@ · 29 unmodified lines
30	#undef pud_free_tlb	30	#undef pud_free_tlb
31	#define pud_free_tlb(tlb, x, addr) do { } while (0)	31	#define pud_free_tlb(tlb, x, addr) do { } while (0)
32	#define pud_free(mm, x) do { } while (0)	32	#define pud_free(mm, x) do { } while (0)
33	#define __pud_free_tlb(tlb, x, addr) do { } while (0)
34		33
35	#undef pud_addr_end	34	#undef pud_addr_end
36	#define pud_addr_end(addr, end) (end)	35	#define pud_addr_end(addr, end) (end)

include/asm-generic/5level-fixup.h+0−1

@@ -51,7 +51,6 @@ static inline int p4d_present(p4d_t p4d) · 50 unmodified lines
51	#undef p4d_free_tlb	51	#undef p4d_free_tlb
52	#define p4d_free_tlb(tlb, x, addr) do { } while (0)	52	#define p4d_free_tlb(tlb, x, addr) do { } while (0)
53	#define p4d_free(mm, x) do { } while (0)	53	#define p4d_free(mm, x) do { } while (0)
54	#define __p4d_free_tlb(tlb, x, addr) do { } while (0)
55		54
56	#undef p4d_addr_end	55	#undef p4d_addr_end
57	#define p4d_addr_end(addr, end) (end)	56	#define p4d_addr_end(addr, end) (end)

include/asm-generic/pgtable-nop4d.h+1−1

@@ -50,7 +50,7 @@ static inline p4d_t p4d_offset(pgd_t pgd, unsigned long address) · 49 unmodified lines
50	*/	50	*/
51	#define p4d_alloc_one(mm, address) NULL	51	#define p4d_alloc_one(mm, address) NULL
52	#define p4d_free(mm, x) do { } while (0)	52	#define p4d_free(mm, x) do { } while (0)
53	#define __p4d_free_tlb(tlb, x, a) do { } while (0)	53	#define p4d_free_tlb(tlb, x, a) do { } while (0)
54		54
55	#undef p4d_addr_end	55	#undef p4d_addr_end
56	#define p4d_addr_end(addr, end) (end)	56	#define p4d_addr_end(addr, end) (end)

include/asm-generic/pgtable-nopmd.h+1−1

@@ -60,7 +60,7 @@ static inline pmd_t * pmd_offset(pud_t * pud, unsigned long address) · 59 unmodified lines
60	static inline void pmd_free(struct mm_struct mm, pmd_t pmd)	60	static inline void pmd_free(struct mm_struct mm, pmd_t pmd)
61	{	61	{
62	}	62	}
63	#define __pmd_free_tlb(tlb, x, a) do { } while (0)	63	#define pmd_free_tlb(tlb, x, a) do { } while (0)
64		64
65	#undef pmd_addr_end	65	#undef pmd_addr_end
66	#define pmd_addr_end(addr, end) (end)	66	#define pmd_addr_end(addr, end) (end)

include/asm-generic/pgtable-nopud.h+1−1

@@ -59,7 +59,7 @@ static inline pud_t pud_offset(p4d_t p4d, unsigned long address) · 58 unmodified lines
59	*/	59	*/
60	#define pud_alloc_one(mm, address) NULL	60	#define pud_alloc_one(mm, address) NULL
61	#define pud_free(mm, x) do { } while (0)	61	#define pud_free(mm, x) do { } while (0)
62	#define __pud_free_tlb(tlb, x, a) do { } while (0)	62	#define pud_free_tlb(tlb, x, a) do { } while (0)
63		63
64	#undef pud_addr_end	64	#undef pud_addr_end
65	#define pud_addr_end(addr, end) (end)	65	#define pud_addr_end(addr, end) (end)

include/asm-generic/pgtable.h+51−0

@@ -558,8 +558,19 @@ static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) · 557 unmodified lines
558	* Do the tests inline, but report and clear the bad entry in mm/memory.c.	558	* Do the tests inline, but report and clear the bad entry in mm/memory.c.
559	*/	559	*/
560	void pgd_clear_bad(pgd_t *);	560	void pgd_clear_bad(pgd_t *);
		561
		562	#ifndef __PAGETABLE_P4D_FOLDED
561	void p4d_clear_bad(p4d_t *);	563	void p4d_clear_bad(p4d_t *);
		564	#else
		565	#define p4d_clear_bad(p4d) do { } while (0)
		566	#endif
		567
		568	#ifndef __PAGETABLE_PUD_FOLDED
562	void pud_clear_bad(pud_t *);	569	void pud_clear_bad(pud_t *);
		570	#else
		571	#define pud_clear_bad(p4d) do { } while (0)
		572	#endif
		573
563	void pmd_clear_bad(pmd_t *);	574	void pmd_clear_bad(pmd_t *);
564		575
565	static inline int pgd_none_or_clear_bad(pgd_t *pgd)	576	static inline int pgd_none_or_clear_bad(pgd_t *pgd)
@@ -903,6 +914,21 @@ static inline int pud_write(pud_t pud) · 337 unmodified lines
903	}	914	}
904	#endif /* pud_write */	915	#endif /* pud_write */
905		916
		917	#if !defined(CONFIG_ARCH_HAS_PTE_DEVMAP) \|\| !defined(CONFIG_TRANSPARENT_HUGEPAGE)
		918	static inline int pmd_devmap(pmd_t pmd)
		919	{
		920	return 0;
		921	}
		922	static inline int pud_devmap(pud_t pud)
		923	{
		924	return 0;
		925	}
		926	static inline int pgd_devmap(pgd_t pgd)
		927	{
		928	return 0;
		929	}
		930	#endif
		931
906	#if !defined(CONFIG_TRANSPARENT_HUGEPAGE) \|\| \	932	#if !defined(CONFIG_TRANSPARENT_HUGEPAGE) \|\| \
907	(defined(CONFIG_TRANSPARENT_HUGEPAGE) && \	933	(defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
908	!defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD))	934	!defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD))
@@ -912,6 +938,31 @@ static inline int pud_trans_huge(pud_t pud) · 3 unmodified lines
912	}	938	}
913	#endif	939	#endif
914		940
		941	/* See pmd_none_or_trans_huge_or_clear_bad for discussion. */
		942	static inline int pud_none_or_trans_huge_or_dev_or_clear_bad(pud_t *pud)
		943	{
		944	pud_t pudval = READ_ONCE(*pud);
		945
		946	if (pud_none(pudval) \|\| pud_trans_huge(pudval) \|\| pud_devmap(pudval))
		947	return 1;
		948	if (unlikely(pud_bad(pudval))) {
		949	pud_clear_bad(pud);
		950	return 1;
		951	}
		952	return 0;
		953	}
		954
		955	/* See pmd_trans_unstable for discussion. */
		956	static inline int pud_trans_unstable(pud_t *pud)
		957	{
		958	#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \
		959	defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
		960	return pud_none_or_trans_huge_or_dev_or_clear_bad(pud);
		961	#else
		962	return 0;
		963	#endif
		964	}
		965
915	#ifndef pmd_read_atomic	966	#ifndef pmd_read_atomic
916	static inline pmd_t pmd_read_atomic(pmd_t *pmdp)	967	static inline pmd_t pmd_read_atomic(pmd_t *pmdp)
917	{	968	{

include/asm-generic/tlb.h+0−4

 	} while (0)
 #endif
-#ifndef __ARCH_HAS_4LEVEL_HACK
 #ifndef pud_free_tlb
 #define pud_free_tlb(tlb, pudp, address)			\
 	do {							\
 		__pud_free_tlb(tlb, pudp, address);		\
 	} while (0)
 #endif
-#endif
-#ifndef __ARCH_HAS_5LEVEL_HACK
 #ifndef p4d_free_tlb
 #define p4d_free_tlb(tlb, pudp, address)			\
 	do {							\
 		__p4d_free_tlb(tlb, pudp, address);		\
 	} while (0)
 #endif
-#endif
 #endif /* CONFIG_MMU */

include/linux/fs.h+5−1

 };
 void dio_end_io(struct bio *bio);
-void dio_warn_stale_pagecache(struct file *filp);
 ssize_t __blockdev_direct_IO(struct kiocb *iocb, struct inode *inode,
 			     struct block_device *bdev, struct iov_iter *iter,
 		wake_up_bit(&inode->i_state, __I_DIO_WAKEUP);
 }
+/*
+ * Warn about a page cache invalidation failure diring a direct I/O write.
+ */
+void dio_warn_stale_pagecache(struct file *filp);
 extern void inode_set_flags(struct inode *inode, unsigned int flags,
 			    unsigned int mask);

include/linux/gfp.h+2−0

@@ -612,6 +612,8 @@ static inline bool pm_suspended_storage(void) · 611 unmodified lines
612	/* The below functions must be run on a range from a single zone. */	612	/* The below functions must be run on a range from a single zone. */
613	extern int alloc_contig_range(unsigned long start, unsigned long end,	613	extern int alloc_contig_range(unsigned long start, unsigned long end,
614	unsigned migratetype, gfp_t gfp_mask);	614	unsigned migratetype, gfp_t gfp_mask);
		615	extern struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask,
		616	int nid, nodemask_t *nodemask);
615	#endif	617	#endif
616	void free_contig_range(unsigned long pfn, unsigned int nr_pages);	618	void free_contig_range(unsigned long pfn, unsigned int nr_pages);
617		619

include/linux/hugetlb.h+116−24

@@ -105,8 +105,7 @@ void move_hugetlb_state(struct page oldpage, struct page newpage, int reason); · 104 unmodified lines
105	void free_huge_page(struct page *page);	105	void free_huge_page(struct page *page);
106	void hugetlb_fix_reserve_counts(struct inode *inode);	106	void hugetlb_fix_reserve_counts(struct inode *inode);
107	extern struct mutex *hugetlb_fault_mutex_table;	107	extern struct mutex *hugetlb_fault_mutex_table;
108	u32 hugetlb_fault_mutex_hash(struct hstate h, struct address_space mapping,	108	u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx);
109	pgoff_t idx, unsigned long address);
110		109
111	pte_t huge_pmd_share(struct mm_struct mm, unsigned long addr, pud_t *pud);	110	pte_t huge_pmd_share(struct mm_struct mm, unsigned long addr, pud_t *pud);
112		111
@@ -164,38 +163,130 @@ static inline void adjust_range_if_pmd_sharing_possible( · 51 unmodified lines
164	{	163	{
165	}	164	}
166		165
167	#define follow_hugetlb_page(m,v,p,vs,a,b,i,w,n) ({ BUG(); 0; })	166	static inline long follow_hugetlb_page(struct mm_struct *mm,
168	#define follow_huge_addr(mm, addr, write) ERR_PTR(-EINVAL)	167	struct vm_area_struct vma, struct page *pages,
169	#define copy_hugetlb_page_range(src, dst, vma) ({ BUG(); 0; })	168	struct vm_area_struct *vmas, unsigned long position,
		169	unsigned long *nr_pages, long i, unsigned int flags,
		170	int *nonblocking)
		171	{
		172	BUG();
		173	return 0;
		174	}
		175
		176	static inline struct page follow_huge_addr(struct mm_struct mm,
		177	unsigned long address, int write)
		178	{
		179	return ERR_PTR(-EINVAL);
		180	}
		181
		182	static inline int copy_hugetlb_page_range(struct mm_struct *dst,
		183	struct mm_struct src, struct vm_area_struct vma)
		184	{
		185	BUG();
		186	return 0;
		187	}
		188
170	static inline void hugetlb_report_meminfo(struct seq_file *m)	189	static inline void hugetlb_report_meminfo(struct seq_file *m)
171	{	190	{
172	}	191	}
173	#define hugetlb_report_node_meminfo(n, buf) 0	192
		193	static inline int hugetlb_report_node_meminfo(int nid, char *buf)
		194	{
		195	return 0;
		196	}
		197
174	static inline void hugetlb_show_meminfo(void)	198	static inline void hugetlb_show_meminfo(void)
175	{	199	{
176	}	200	}
177	#define follow_huge_pd(vma, addr, hpd, flags, pdshift) NULL	201
178	#define follow_huge_pmd(mm, addr, pmd, flags) NULL	202	static inline struct page follow_huge_pd(struct vm_area_struct vma,
179	#define follow_huge_pud(mm, addr, pud, flags) NULL	203	unsigned long address, hugepd_t hpd, int flags,
180	#define follow_huge_pgd(mm, addr, pgd, flags) NULL	204	int pdshift)
181	#define prepare_hugepage_range(file, addr, len) (-EINVAL)	205	{
182	#define pmd_huge(x) 0	206	return NULL;
183	#define pud_huge(x) 0	207	}
184	#define is_hugepage_only_range(mm, addr, len) 0	208
185	#define hugetlb_free_pgd_range(tlb, addr, end, floor, ceiling) ({BUG(); 0; })	209	static inline struct page follow_huge_pmd(struct mm_struct mm,
186	#define hugetlb_mcopy_atomic_pte(dst_mm, dst_pte, dst_vma, dst_addr, \	210	unsigned long address, pmd_t *pmd, int flags)
187	src_addr, pagep) ({ BUG(); 0; })	211	{
188	#define huge_pte_offset(mm, address, sz) 0	212	return NULL;
		213	}
		214
		215	static inline struct page follow_huge_pud(struct mm_struct mm,
		216	unsigned long address, pud_t *pud, int flags)
		217	{
		218	return NULL;
		219	}
		220
		221	static inline struct page follow_huge_pgd(struct mm_struct mm,
		222	unsigned long address, pgd_t *pgd, int flags)
		223	{
		224	return NULL;
		225	}
		226
		227	static inline int prepare_hugepage_range(struct file *file,
		228	unsigned long addr, unsigned long len)
		229	{
		230	return -EINVAL;
		231	}
		232
		233	static inline int pmd_huge(pmd_t pmd)
		234	{
		235	return 0;
		236	}
		237
		238	static inline int pud_huge(pud_t pud)
		239	{
		240	return 0;
		241	}
		242
		243	static inline int is_hugepage_only_range(struct mm_struct *mm,
		244	unsigned long addr, unsigned long len)
		245	{
		246	return 0;
		247	}
		248
		249	static inline void hugetlb_free_pgd_range(struct mmu_gather *tlb,
		250	unsigned long addr, unsigned long end,
		251	unsigned long floor, unsigned long ceiling)
		252	{
		253	BUG();
		254	}
		255
		256	static inline int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm,
		257	pte_t *dst_pte,
		258	struct vm_area_struct *dst_vma,
		259	unsigned long dst_addr,
		260	unsigned long src_addr,
		261	struct page **pagep)
		262	{
		263	BUG();
		264	return 0;
		265	}
		266
		267	static inline pte_t huge_pte_offset(struct mm_struct mm, unsigned long addr,
		268	unsigned long sz)
		269	{
		270	return NULL;
		271	}
189		272
190	static inline bool isolate_huge_page(struct page page, struct list_head list)	273	static inline bool isolate_huge_page(struct page page, struct list_head list)
191	{	274	{
192	return false;	275	return false;
193	}	276	}
194	#define putback_active_hugepage(p) do {} while (0)
195	#define move_hugetlb_state(old, new, reason) do {} while (0)
196		277
197	static inline unsigned long hugetlb_change_protection(struct vm_area_struct *vma,	278	static inline void putback_active_hugepage(struct page *page)
198	unsigned long address, unsigned long end, pgprot_t newprot)	279	{
		280	}
		281
		282	static inline void move_hugetlb_state(struct page *oldpage,
		283	struct page *newpage, int reason)
		284	{
		285	}
		286
		287	static inline unsigned long hugetlb_change_protection(
		288	struct vm_area_struct *vma, unsigned long address,
		289	unsigned long end, pgprot_t newprot)
199	{	290	{
200	return 0;	291	return 0;
201	}	292	}
@@ -213,9 +304,10 @@ static inline void __unmap_hugepage_range(struct mmu_gather *tlb, · 11 unmodified lines
213	{	304	{
214	BUG();	305	BUG();
215	}	306	}
		307
216	static inline vm_fault_t hugetlb_fault(struct mm_struct *mm,	308	static inline vm_fault_t hugetlb_fault(struct mm_struct *mm,
217	struct vm_area_struct *vma, unsigned long address,	309	struct vm_area_struct *vma, unsigned long address,
218	unsigned int flags)	310	unsigned int flags)
219	{	311	{
220	BUG();	312	BUG();
221	return 0;	313	return 0;

include/linux/kasan.h+31−0

@@ -70,8 +70,18 @@ struct kasan_cache { · 69 unmodified lines
70	int free_meta_offset;	70	int free_meta_offset;
71	};	71	};
72		72
		73	/*
		74	* These functions provide a special case to support backing module
		75	* allocations with real shadow memory. With KASAN vmalloc, the special
		76	* case is unnecessary, as the work is handled in the generic case.
		77	*/
		78	#ifndef CONFIG_KASAN_VMALLOC
73	int kasan_module_alloc(void *addr, size_t size);	79	int kasan_module_alloc(void *addr, size_t size);
74	void kasan_free_shadow(const struct vm_struct *vm);	80	void kasan_free_shadow(const struct vm_struct *vm);
		81	#else
		82	static inline int kasan_module_alloc(void *addr, size_t size) { return 0; }
		83	static inline void kasan_free_shadow(const struct vm_struct *vm) {}
		84	#endif
75		85
76	int kasan_add_zero_shadow(void *start, unsigned long size);	86	int kasan_add_zero_shadow(void *start, unsigned long size);
77	void kasan_remove_zero_shadow(void *start, unsigned long size);	87	void kasan_remove_zero_shadow(void *start, unsigned long size);
@@ -194,4 +204,25 @@ static inline void kasan_reset_tag(const void addr) · 116 unmodified lines
194		204
195	#endif /* CONFIG_KASAN_SW_TAGS */	205	#endif /* CONFIG_KASAN_SW_TAGS */
196		206
		207	#ifdef CONFIG_KASAN_VMALLOC
		208	int kasan_populate_vmalloc(unsigned long requested_size,
		209	struct vm_struct *area);
		210	void kasan_poison_vmalloc(void *start, unsigned long size);
		211	void kasan_release_vmalloc(unsigned long start, unsigned long end,
		212	unsigned long free_region_start,
		213	unsigned long free_region_end);
		214	#else
		215	static inline int kasan_populate_vmalloc(unsigned long requested_size,
		216	struct vm_struct *area)
		217	{
		218	return 0;
		219	}
		220
		221	static inline void kasan_poison_vmalloc(void *start, unsigned long size) {}
		222	static inline void kasan_release_vmalloc(unsigned long start,
		223	unsigned long end,
		224	unsigned long free_region_start,
		225	unsigned long free_region_end) {}
		226	#endif
		227
197	#endif /* LINUX_KASAN_H */	228	#endif /* LINUX_KASAN_H */

include/linux/memblock.h+3−0

 					 MEMBLOCK_ALLOC_ACCESSIBLE);
 }
+void *memblock_alloc_exact_nid_raw(phys_addr_t size, phys_addr_t align,
+				 phys_addr_t min_addr, phys_addr_t max_addr,
+				 int nid);
 void *memblock_alloc_try_nid_raw(phys_addr_t size, phys_addr_t align,
 				 phys_addr_t min_addr, phys_addr_t max_addr,
 				 int nid);

include/linux/memcontrol.h+22−27

 struct mem_cgroup_reclaim_cookie {
 	pg_data_t *pgdat;
-	int priority;
 	unsigned int generation;
 };
 enum mem_cgroup_events_target {
 	MEM_CGROUP_TARGET_THRESH,
 	MEM_CGROUP_TARGET_SOFTLIMIT,
-	MEM_CGROUP_TARGET_NUMAINFO,
 	MEM_CGROUP_NTARGETS,
 };
 };
 /*
- * per-zone information in memory controller.
+ * per-node information in memory controller.
  */
 struct mem_cgroup_per_node {
 	struct lruvec		lruvec;
 	unsigned long		lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS];
-	struct mem_cgroup_reclaim_iter	iter[DEF_PRIORITY + 1];
+	struct mem_cgroup_reclaim_iter	iter;
 	struct memcg_shrinker_map __rcu	*shrinker_map;
 	unsigned long		usage_in_excess;/* Set to the value by which */
 						/* the soft limit is exceeded*/
 	bool			on_tree;
-	bool			congested;	/* memcg has many dirty pages */
-						/* backed by a congested BDI */
 	struct mem_cgroup	*memcg;		/* Back pointer, we cannot */
 						/* use container_of	   */
 };
 	struct list_head kmem_caches;
 #endif
-	int last_scanned_node;
-#if MAX_NUMNODES > 1
-	nodemask_t	scan_nodes;
-	atomic_t	numainfo_events;
-	atomic_t	numainfo_updating;
-#endif
 #ifdef CONFIG_CGROUP_WRITEBACK
 	struct list_head cgwb_list;
 	struct wb_domain cgwb_domain;
 }
 /**
- * mem_cgroup_lruvec - get the lru list vector for a node or a memcg zone
+ * mem_cgroup_lruvec - get the lru list vector for a memcg & node
- * @node: node of the wanted lruvec
  * @memcg: memcg of the wanted lruvec
  *
- * Returns the lru list vector holding pages for a given @node or a given
+ * Returns the lru list vector holding pages for a given @memcg &
- * @memcg and @zone. This can be the node lruvec, if the memory controller
+ * @node combination. This can be the node lruvec, if the memory
- * is disabled.
+ * controller is disabled.
  */
-static inline struct lruvec *mem_cgroup_lruvec(struct pglist_data *pgdat,
+static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,
-				struct mem_cgroup *memcg)
+					       struct pglist_data *pgdat)
 {
 	struct mem_cgroup_per_node *mz;
 	struct lruvec *lruvec;
 	if (mem_cgroup_disabled()) {
-		lruvec = node_lruvec(pgdat);
+		lruvec = &pgdat->__lruvec;
 		goto out;
 	}
+	if (!memcg)
+		memcg = root_mem_cgroup;
 	mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
 	lruvec = &mz->lruvec;
 out:
 		return;
 	}
-	lruvec = mem_cgroup_lruvec(pgdat, page->mem_cgroup);
+	lruvec = mem_cgroup_lruvec(page->mem_cgroup, pgdat);
 	__mod_lruvec_state(lruvec, idx, val);
 }
 {
 }
-static inline struct lruvec *mem_cgroup_lruvec(struct pglist_data *pgdat,
+static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,
-				struct mem_cgroup *memcg)
+					       struct pglist_data *pgdat)
 {
-	return node_lruvec(pgdat);
+	return &pgdat->__lruvec;
 }
 static inline struct lruvec *mem_cgroup_page_lruvec(struct page *page,
 						    struct pglist_data *pgdat)
 {
-	return &pgdat->lruvec;
+	return &pgdat->__lruvec;
+}
+static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
+{
+	return NULL;
 }
 static inline bool mm_match_cgroup(struct mm_struct *mm,

include/linux/memory_hotplug.h+4−7

 typedef void (*online_page_callback_t)(struct page *page, unsigned int order);
+extern void generic_online_page(struct page *page, unsigned int order);
 extern int set_online_page_callback(online_page_callback_t callback);
 extern int restore_online_page_callback(online_page_callback_t callback);
-extern void __online_page_set_limits(struct page *page);
-extern void __online_page_increment_counters(struct page *page);
-extern void __online_page_free(struct page *page);
 extern int try_online_node(int nid);
 extern int arch_add_memory(int nid, u64 start, u64 size,
 void mem_hotplug_begin(void);
 void mem_hotplug_done(void);
-extern void set_zone_contiguous(struct zone *zone);
-extern void clear_zone_contiguous(struct zone *zone);
 #else /* ! CONFIG_MEMORY_HOTPLUG */
 #define pfn_to_online_page(pfn)			\
 ({						\
 static inline void __remove_memory(int nid, u64 start, u64 size) {}
 #endif /* CONFIG_MEMORY_HOTREMOVE */
+extern void set_zone_contiguous(struct zone *zone);
+extern void clear_zone_contiguous(struct zone *zone);
 extern void __ref free_area_init_core_hotplug(int nid);
 extern int __add_memory(int nid, u64 start, u64 size);
 extern int add_memory(int nid, u64 start, u64 size);

include/linux/mm.h+12−22

 struct mmu_gather;
 struct inode;
-#if !defined(CONFIG_ARCH_HAS_PTE_DEVMAP) || !defined(CONFIG_TRANSPARENT_HUGEPAGE)
-static inline int pmd_devmap(pmd_t pmd)
-{
-	return 0;
-}
-static inline int pud_devmap(pud_t pud)
-{
-	return 0;
-}
-static inline int pgd_devmap(pgd_t pgd)
-{
-	return 0;
-}
-#endif
 /*
  * FIXME: take this include out, include page-flags.h in
  * files which need it (119 of them)
 	return (unsigned long)val;
 }
+void mm_trace_rss_stat(struct mm_struct *mm, int member, long count);
 static inline void add_mm_counter(struct mm_struct *mm, int member, long value)
 {
-	atomic_long_add(value, &mm->rss_stat.count[member]);
+	long count = atomic_long_add_return(value, &mm->rss_stat.count[member]);
+	mm_trace_rss_stat(mm, member, count);
 }
 static inline void inc_mm_counter(struct mm_struct *mm, int member)
 {
-	atomic_long_inc(&mm->rss_stat.count[member]);
+	long count = atomic_long_inc_return(&mm->rss_stat.count[member]);
+	mm_trace_rss_stat(mm, member, count);
 }
 static inline void dec_mm_counter(struct mm_struct *mm, int member)
 {
-	atomic_long_dec(&mm->rss_stat.count[member]);
+	long count = atomic_long_dec_return(&mm->rss_stat.count[member]);
+	mm_trace_rss_stat(mm, member, count);
 }
 /* Optimized variant when page is already known not to be PageAnon */
 extern void setup_per_cpu_pageset(void);
-extern void zone_pcp_update(struct zone *zone);
-extern void zone_pcp_reset(struct zone *zone);
 /* page_alloc.c */
 extern int min_free_kbytes;
 extern int watermark_boost_factor;
 extern int sysctl_memory_failure_recovery;
 extern void shake_page(struct page *p, int access);
 extern atomic_long_t num_poisoned_pages __read_mostly;
-extern int soft_offline_page(struct page *page, int flags);
+extern int soft_offline_page(unsigned long pfn, int flags);
 /*

include/linux/mmzone.h+20−14

@@ -273,12 +273,12 @@ enum lru_list { · 272 unmodified lines
273		273
274	#define for_each_evictable_lru(lru) for (lru = 0; lru <= LRU_ACTIVE_FILE; lru++)	274	#define for_each_evictable_lru(lru) for (lru = 0; lru <= LRU_ACTIVE_FILE; lru++)
275		275
276	static inline int is_file_lru(enum lru_list lru)	276	static inline bool is_file_lru(enum lru_list lru)
277	{	277	{
278	return (lru == LRU_INACTIVE_FILE \|\| lru == LRU_ACTIVE_FILE);	278	return (lru == LRU_INACTIVE_FILE \|\| lru == LRU_ACTIVE_FILE);
279	}	279	}
280		280
281	static inline int is_active_lru(enum lru_list lru)	281	static inline bool is_active_lru(enum lru_list lru)
282	{	282	{
283	return (lru == LRU_ACTIVE_ANON \|\| lru == LRU_ACTIVE_FILE);	283	return (lru == LRU_ACTIVE_ANON \|\| lru == LRU_ACTIVE_FILE);
284	}	284	}
@@ -296,6 +296,12 @@ struct zone_reclaim_stat { · 11 unmodified lines
296	unsigned long recent_scanned[2];	296	unsigned long recent_scanned[2];
297	};	297	};
298		298
		299	enum lruvec_flags {
		300	LRUVEC_CONGESTED, /* lruvec has many dirty pages
		301	* backed by a congested BDI
		302	*/
		303	};
		304
299	struct lruvec {	305	struct lruvec {
300	struct list_head lists[NR_LRU_LISTS];	306	struct list_head lists[NR_LRU_LISTS];
301	struct zone_reclaim_stat reclaim_stat;	307	struct zone_reclaim_stat reclaim_stat;
@@ -303,12 +309,14 @@ struct lruvec { · 1 unmodified line
303	atomic_long_t inactive_age;	309	atomic_long_t inactive_age;
304	/* Refaults at the time of last reclaim cycle */	310	/* Refaults at the time of last reclaim cycle */
305	unsigned long refaults;	311	unsigned long refaults;
		312	/* Various lruvec state flags (enum lruvec_flags) */
		313	unsigned long flags;
306	#ifdef CONFIG_MEMCG	314	#ifdef CONFIG_MEMCG
307	struct pglist_data *pgdat;	315	struct pglist_data *pgdat;
308	#endif	316	#endif
309	};	317	};
310		318
311	/* Isolate unmapped file */	319	/* Isolate unmapped pages */
312	#define ISOLATE_UNMAPPED ((__force isolate_mode_t)0x2)	320	#define ISOLATE_UNMAPPED ((__force isolate_mode_t)0x2)
313	/* Isolate for asynchronous migration */	321	/* Isolate for asynchronous migration */
314	#define ISOLATE_ASYNC_MIGRATE ((__force isolate_mode_t)0x4)	322	#define ISOLATE_ASYNC_MIGRATE ((__force isolate_mode_t)0x4)
@@ -572,9 +580,6 @@ struct zone { · 257 unmodified lines
572	} ____cacheline_internodealigned_in_smp;	580	} ____cacheline_internodealigned_in_smp;
573		581
574	enum pgdat_flags {	582	enum pgdat_flags {
575	PGDAT_CONGESTED, /* pgdat has many dirty pages backed by
576	* a congested BDI
577	*/
578	PGDAT_DIRTY, /* reclaim scanning has recently found	583	PGDAT_DIRTY, /* reclaim scanning has recently found
579	* many dirty file pages at the tail	584	* many dirty file pages at the tail
580	* of the LRU.	585	* of the LRU.
@@ -777,7 +782,13 @@ typedef struct pglist_data { · 196 unmodified lines
777	#endif	782	#endif
778		783
779	/* Fields commonly accessed by the page reclaim scanner */	784	/* Fields commonly accessed by the page reclaim scanner */
780	struct lruvec lruvec;	785
		786	/*
		787	* NOTE: THIS IS UNUSED IF MEMCG IS ENABLED.
		788	*
		789	* Use mem_cgroup_lruvec() to look up lruvecs.
		790	*/
		791	struct lruvec __lruvec;
781		792
782	unsigned long flags;	793	unsigned long flags;
783		794
@@ -800,11 +811,6 @@ typedef struct pglist_data { · 16 unmodified lines
800	#define node_start_pfn(nid) (NODE_DATA(nid)->node_start_pfn)	811	#define node_start_pfn(nid) (NODE_DATA(nid)->node_start_pfn)
801	#define node_end_pfn(nid) pgdat_end_pfn(NODE_DATA(nid))	812	#define node_end_pfn(nid) pgdat_end_pfn(NODE_DATA(nid))
802		813
803	static inline struct lruvec node_lruvec(struct pglist_data pgdat)
804	{
805	return &pgdat->lruvec;
806	}
807
808	static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat)	814	static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat)
809	{	815	{
810	return pgdat->node_start_pfn + pgdat->node_spanned_pages;	816	return pgdat->node_start_pfn + pgdat->node_spanned_pages;
@@ -842,7 +848,7 @@ static inline struct pglist_data lruvec_pgdat(struct lruvec lruvec) · 31 unmodified lines
842	#ifdef CONFIG_MEMCG	848	#ifdef CONFIG_MEMCG
843	return lruvec->pgdat;	849	return lruvec->pgdat;
844	#else	850	#else
845	return container_of(lruvec, struct pglist_data, lruvec);	851	return container_of(lruvec, struct pglist_data, __lruvec);
846	#endif	852	#endif
847	}	853	}
848		854
@@ -1079,7 +1085,7 @@ static inline struct zoneref first_zones_zonelist(struct zonelist zonelist, · 230 unmodified lines
1079	/**	1085	/**
1080	* for_each_zone_zonelist_nodemask - helper macro to iterate over valid zones in a zonelist at or below a given zone index and within a nodemask	1086	* for_each_zone_zonelist_nodemask - helper macro to iterate over valid zones in a zonelist at or below a given zone index and within a nodemask
1081	* @zone - The current zone in the iterator	1087	* @zone - The current zone in the iterator
1082	* @z - The current pointer within zonelist->zones being iterated	1088	* @z - The current pointer within zonelist->_zonerefs being iterated
1083	* @zlist - The zonelist being iterated	1089	* @zlist - The zonelist being iterated
1084	* @highidx - The zone index of the highest zone to return	1090	* @highidx - The zone index of the highest zone to return
1085	* @nodemask - Nodemask allowed by the allocator	1091	* @nodemask - Nodemask allowed by the allocator

include/linux/moduleloader.h+1−1

@@ -91,7 +91,7 @@ void module_arch_cleanup(struct module *mod); · 90 unmodified lines
91	/* Any cleanup before freeing mod->module_init */	91	/* Any cleanup before freeing mod->module_init */
92	void module_arch_freeing_init(struct module *mod);	92	void module_arch_freeing_init(struct module *mod);
93		93
94	#ifdef CONFIG_KASAN	94	#if defined(CONFIG_KASAN) && !defined(CONFIG_KASAN_VMALLOC)
95	#include <linux/kasan.h>	95	#include <linux/kasan.h>
96	#define MODULE_ALIGN (PAGE_SIZE << KASAN_SHADOW_SCALE_SHIFT)	96	#define MODULE_ALIGN (PAGE_SIZE << KASAN_SHADOW_SCALE_SHIFT)
97	#else	97	#else

include/linux/page-isolation.h+2−2

@@ -30,7 +30,7 @@ static inline bool is_migrate_isolate(int migratetype) · 29 unmodified lines
30	}	30	}
31	#endif	31	#endif
32		32
33	#define SKIP_HWPOISON 0x1	33	#define MEMORY_OFFLINE 0x1
34	#define REPORT_FAILURE 0x2	34	#define REPORT_FAILURE 0x2
35		35
36	bool has_unmovable_pages(struct zone zone, struct page page, int count,	36	bool has_unmovable_pages(struct zone zone, struct page page, int count,
@@ -58,7 +58,7 @@ undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, · 21 unmodified lines
58	* Test all pages in [start_pfn, end_pfn) are isolated or not.	58	* Test all pages in [start_pfn, end_pfn) are isolated or not.
59	*/	59	*/
60	int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn,	60	int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn,
61	bool skip_hwpoisoned_pages);	61	int isol_flags);
62		62
63	struct page alloc_migrate_target(struct page page, unsigned long private);	63	struct page alloc_migrate_target(struct page page, unsigned long private);
64		64

include/linux/slab.h+0−20

@@ -561,26 +561,6 @@ static __always_inline void *kmalloc(size_t size, gfp_t flags) · 560 unmodified lines
561	return __kmalloc(size, flags);	561	return __kmalloc(size, flags);
562	}	562	}
563		563
564	/*
565	* Determine size used for the nth kmalloc cache.
566	* return size or 0 if a kmalloc cache for that
567	* size does not exist
568	*/
569	static __always_inline unsigned int kmalloc_size(unsigned int n)
570	{
571	#ifndef CONFIG_SLOB
572	if (n > 2)
573	return 1U << n;
574
575	if (n == 1 && KMALLOC_MIN_SIZE <= 32)
576	return 96;
577
578	if (n == 2 && KMALLOC_MIN_SIZE <= 64)
579	return 192;
580	#endif
581	return 0;
582	}
583
584	static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)	564	static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
585	{	565	{
586	#ifndef CONFIG_SLOB	566	#ifndef CONFIG_SLOB

include/linux/string.h+2−0

@@ -216,6 +216,8 @@ int bprintf(u32 bin_buf, size_t size, const char fmt, ...) __printf(3, 4); · 215 unmodified lines
216	extern ssize_t memory_read_from_buffer(void to, size_t count, loff_t ppos,	216	extern ssize_t memory_read_from_buffer(void to, size_t count, loff_t ppos,
217	const void *from, size_t available);	217	const void *from, size_t available);
218		218
		219	int ptr_to_hashval(const void ptr, unsigned long hashval_out);
		220
219	/**	221	/**
220	* strstarts - does @str start with @prefix?	222	* strstarts - does @str start with @prefix?
221	* @str: string to examine	223	* @str: string to examine

include/linux/swap.h+1−1

@@ -307,7 +307,7 @@ struct vma_swap_readahead { · 306 unmodified lines
307	};	307	};
308		308
309	/* linux/mm/workingset.c */	309	/* linux/mm/workingset.c */
310	void workingset_eviction(struct page page);	310	void workingset_eviction(struct page page, struct mem_cgroup *target_memcg);
311	void workingset_refault(struct page page, void shadow);	311	void workingset_refault(struct page page, void shadow);
312	void workingset_activation(struct page *page);	312	void workingset_activation(struct page *page);
313		313

include/linux/vmalloc.h+12−0

@@ -22,6 +22,18 @@ struct notifier_block; /* in notifier.h */ · 21 unmodified lines
22	#define VM_UNINITIALIZED 0x00000020 /* vm_struct is not fully initialized */	22	#define VM_UNINITIALIZED 0x00000020 /* vm_struct is not fully initialized */
23	#define VM_NO_GUARD 0x00000040 /* don't add guard page */	23	#define VM_NO_GUARD 0x00000040 /* don't add guard page */
24	#define VM_KASAN 0x00000080 /* has allocated kasan shadow memory */	24	#define VM_KASAN 0x00000080 /* has allocated kasan shadow memory */
		25
		26	/*
		27	* VM_KASAN is used slighly differently depending on CONFIG_KASAN_VMALLOC.
		28	*
		29	* If IS_ENABLED(CONFIG_KASAN_VMALLOC), VM_KASAN is set on a vm_struct after
		30	* shadow memory has been mapped. It's used to handle allocation errors so that
		31	* we don't try to poision shadow on free if it was never allocated.
		32	*
		33	* Otherwise, VM_KASAN is set for kasan_module_alloc() allocations and used to
		34	* determine which allocations need the module shadow freed.
		35	*/
		36
25	/*	37	/*
26	* Memory with VM_FLUSH_RESET_PERMS cannot be freed in an interrupt or with	38	* Memory with VM_FLUSH_RESET_PERMS cannot be freed in an interrupt or with
27	* vfree_atomic().	39	* vfree_atomic().

include/trace/events/kmem.h+47−0

@@ -316,6 +316,53 @@ TRACE_EVENT(mm_page_alloc_extfrag, · 315 unmodified lines
316	__entry->change_ownership)	316	__entry->change_ownership)
317	);	317	);
318		318
		319	/*
		320	* Required for uniquely and securely identifying mm in rss_stat tracepoint.
		321	*/
		322	#ifndef __PTR_TO_HASHVAL
		323	static unsigned int __maybe_unused mm_ptr_to_hash(const void *ptr)
		324	{
		325	int ret;
		326	unsigned long hashval;
		327
		328	ret = ptr_to_hashval(ptr, &hashval);
		329	if (ret)
		330	return 0;
		331
		332	/* The hashed value is only 32-bit */
		333	return (unsigned int)hashval;
		334	}
		335	#define __PTR_TO_HASHVAL
		336	#endif
		337
		338	TRACE_EVENT(rss_stat,
		339
		340	TP_PROTO(struct mm_struct *mm,
		341	int member,
		342	long count),
		343
		344	TP_ARGS(mm, member, count),
		345
		346	TP_STRUCT__entry(
		347	__field(unsigned int, mm_id)
		348	__field(unsigned int, curr)
		349	__field(int, member)
		350	__field(long, size)
		351	),
		352
		353	TP_fast_assign(
		354	__entry->mm_id = mm_ptr_to_hash(mm);
		355	__entry->curr = !!(current->mm == mm);
		356	__entry->member = member;
		357	__entry->size = (count << PAGE_SHIFT);
		358	),
		359
		360	TP_printk("mm_id=%u curr=%d member=%d size=%ldB",
		361	__entry->mm_id,
		362	__entry->curr,
		363	__entry->member,
		364	__entry->size)
		365	);
319	#endif /* _TRACE_KMEM_H */	366	#endif /* _TRACE_KMEM_H */
320		367
321	/* This part must be outside protection */	368	/* This part must be outside protection */

kernel/events/uprobes.c+1−1

@@ -1457,7 +1457,7 @@ static int xol_add_vma(struct mm_struct mm, struct xol_area area) · 1456 unmodified lines
1457	/* Try to map as high as possible, this is only a hint. */	1457	/* Try to map as high as possible, this is only a hint. */
1458	area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE,	1458	area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE,
1459	PAGE_SIZE, 0, 0);	1459	PAGE_SIZE, 0, 0);
1460	if (area->vaddr & ~PAGE_MASK) {	1460	if (IS_ERR_VALUE(area->vaddr)) {
1461	ret = area->vaddr;	1461	ret = area->vaddr;
1462	goto fail;	1462	goto fail;
1463	}	1463	}

kernel/fork.c+4−0

 #include <linux/livepatch.h>
 #include <linux/thread_info.h>
 #include <linux/stackleak.h>
+#include <linux/kasan.h>
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
 		if (!s)
 			continue;
+		/* Clear the KASAN shadow of the stack. */
+		kasan_unpoison_shadow(s->addr, THREAD_SIZE);
 		/* Clear stale pointers from reused stack. */
 		memset(s->addr, 0, THREAD_SIZE);

kernel/sysctl.c+1−1

@@ -1466,7 +1466,7 @@ static struct ctl_table vm_table[] = { · 1465 unmodified lines
1466	.procname = "drop_caches",	1466	.procname = "drop_caches",
1467	.data = &sysctl_drop_caches,	1467	.data = &sysctl_drop_caches,
1468	.maxlen = sizeof(int),	1468	.maxlen = sizeof(int),
1469	.mode = 0644,	1469	.mode = 0200,
1470	.proc_handler = drop_caches_sysctl_handler,	1470	.proc_handler = drop_caches_sysctl_handler,
1471	.extra1 = SYSCTL_ONE,	1471	.extra1 = SYSCTL_ONE,
1472	.extra2 = &four,	1472	.extra2 = &four,

lib/Kconfig.kasan+16−0

@@ -6,6 +6,9 @@ config HAVE_ARCH_KASAN · 5 unmodified lines
6	config HAVE_ARCH_KASAN_SW_TAGS	6	config HAVE_ARCH_KASAN_SW_TAGS
7	bool	7	bool
8		8
		9	config HAVE_ARCH_KASAN_VMALLOC
		10	bool
		11
9	config CC_HAS_KASAN_GENERIC	12	config CC_HAS_KASAN_GENERIC
10	def_bool $(cc-option, -fsanitize=kernel-address)	13	def_bool $(cc-option, -fsanitize=kernel-address)
11		14
@@ -142,6 +145,19 @@ config KASAN_SW_TAGS_IDENTIFY · 130 unmodified lines
142	(use-after-free or out-of-bounds) at the cost of increased	145	(use-after-free or out-of-bounds) at the cost of increased
143	memory consumption.	146	memory consumption.
144		147
		148	config KASAN_VMALLOC
		149	bool "Back mappings in vmalloc space with real shadow memory"
		150	depends on KASAN && HAVE_ARCH_KASAN_VMALLOC
		151	help
		152	By default, the shadow region for vmalloc space is the read-only
		153	zero page. This means that KASAN cannot detect errors involving
		154	vmalloc space.
		155
		156	Enabling this option will hook in to vmap/vmalloc and back those
		157	mappings with real shadow memory allocated on demand. This allows
		158	for KASAN to detect more sorts of errors (and to support vmapped
		159	stacks), but at the cost of higher memory usage.
		160
145	config TEST_KASAN	161	config TEST_KASAN
146	tristate "Module for testing KASAN for bug detection"	162	tristate "Module for testing KASAN for bug detection"
147	depends on m && KASAN	163	depends on m && KASAN

lib/test_kasan.c+26−0

@@ -19,6 +19,7 @@ · 18 unmodified lines
19	#include <linux/string.h>	19	#include <linux/string.h>
20	#include <linux/uaccess.h>	20	#include <linux/uaccess.h>
21	#include <linux/io.h>	21	#include <linux/io.h>
		22	#include <linux/vmalloc.h>
22		23
23	#include <asm/page.h>	24	#include <asm/page.h>
24		25
@@ -748,6 +749,30 @@ static noinline void __init kmalloc_double_kzfree(void) · 723 unmodified lines
748	kzfree(ptr);	749	kzfree(ptr);
749	}	750	}
750		751
		752	#ifdef CONFIG_KASAN_VMALLOC
		753	static noinline void __init vmalloc_oob(void)
		754	{
		755	void *area;
		756
		757	pr_info("vmalloc out-of-bounds\n");
		758
		759	/*
		760	* We have to be careful not to hit the guard page.
		761	* The MMU will catch that and crash us.
		762	*/
		763	area = vmalloc(3000);
		764	if (!area) {
		765	pr_err("Allocation failed\n");
		766	return;
		767	}
		768
		769	((volatile char *)area)[3100];
		770	vfree(area);
		771	}
		772	#else
		773	static void __init vmalloc_oob(void) {}
		774	#endif
		775
751	static int __init kmalloc_tests_init(void)	776	static int __init kmalloc_tests_init(void)
752	{	777	{
753	/*	778	/*
@@ -793,6 +818,7 @@ static int __init kmalloc_tests_init(void) · 39 unmodified lines
793	kasan_strings();	818	kasan_strings();
794	kasan_bitops();	819	kasan_bitops();
795	kmalloc_double_kzfree();	820	kmalloc_double_kzfree();
		821	vmalloc_oob();
796		822
797	kasan_restore_multi_shot(multishot);	823	kasan_restore_multi_shot(multishot);
798		824

lib/vsprintf.c+29−11

 early_initcall(initialize_ptr_random);
 /* Maps a pointer to a 32 bit unique identifier. */
+static inline int __ptr_to_hashval(const void *ptr, unsigned long *hashval_out)
+{
+	unsigned long hashval;
+	if (static_branch_unlikely(&not_filled_random_ptr_key))
+		return -EAGAIN;
+#ifdef CONFIG_64BIT
+	hashval = (unsigned long)siphash_1u64((u64)ptr, &ptr_key);
+	/*
+	 * Mask off the first 32 bits, this makes explicit that we have
+	 * modified the address (and 32 bits is plenty for a unique ID).
+	 */
+	hashval = hashval & 0xffffffff;
+#else
+	hashval = (unsigned long)siphash_1u32((u32)ptr, &ptr_key);
+#endif
+	*hashval_out = hashval;
+	return 0;
+}
+int ptr_to_hashval(const void *ptr, unsigned long *hashval_out)
+{
+	return __ptr_to_hashval(ptr, hashval_out);
+}
 static char *ptr_to_id(char *buf, char *end, const void *ptr,
 		       struct printf_spec spec)
 {
 	const char *str = sizeof(ptr) == 8 ? "(____ptrval____)" : "(ptrval)";
 	unsigned long hashval;
+	int ret;
 	/* When debugging early boot use non-cryptographically secure hash. */
 	if (unlikely(debug_boot_weak_hash)) {
 		return pointer_string(buf, end, (const void *)hashval, spec);
 	}
-	if (static_branch_unlikely(&not_filled_random_ptr_key)) {
+	ret = __ptr_to_hashval(ptr, &hashval);
+	if (ret) {
 		spec.field_width = 2 * sizeof(ptr);
 		/* string length must be less than default_width */
 		return error_string(buf, end, str, spec);
 	}
-#ifdef CONFIG_64BIT
-	hashval = (unsigned long)siphash_1u64((u64)ptr, &ptr_key);
-	/*
-	 * Mask off the first 32 bits, this makes explicit that we have
-	 * modified the address (and 32 bits is plenty for a unique ID).
-	 */
-	hashval = hashval & 0xffffffff;
-#else
-	hashval = (unsigned long)siphash_1u32((u32)ptr, &ptr_key);
-#endif
 	return pointer_string(buf, end, (const void *)hashval, spec);
 }

mm/Kconfig+20−20

@@ -29,7 +29,7 @@ config FLATMEM_MANUAL · 28 unmodified lines
29		29
30	For systems that have holes in their physical address	30	For systems that have holes in their physical address
31	spaces and for features like NUMA and memory hotplug,	31	spaces and for features like NUMA and memory hotplug,
32	choose "Sparse Memory"	32	choose "Sparse Memory".
33		33
34	If unsure, choose this option (Flat Memory) over any other.	34	If unsure, choose this option (Flat Memory) over any other.
35		35
@@ -122,9 +122,9 @@ config SPARSEMEM_VMEMMAP · 86 unmodified lines
122	depends on SPARSEMEM && SPARSEMEM_VMEMMAP_ENABLE	122	depends on SPARSEMEM && SPARSEMEM_VMEMMAP_ENABLE
123	default y	123	default y
124	help	124	help
125	SPARSEMEM_VMEMMAP uses a virtually mapped memmap to optimise	125	SPARSEMEM_VMEMMAP uses a virtually mapped memmap to optimise
126	pfn_to_page and page_to_pfn operations. This is the most	126	pfn_to_page and page_to_pfn operations. This is the most
127	efficient option when sufficient kernel resources are available.	127	efficient option when sufficient kernel resources are available.
128		128
129	config HAVE_MEMBLOCK_NODE_MAP	129	config HAVE_MEMBLOCK_NODE_MAP
130	bool	130	bool
@@ -160,9 +160,9 @@ config MEMORY_HOTPLUG_SPARSE · 29 unmodified lines
160	depends on SPARSEMEM && MEMORY_HOTPLUG	160	depends on SPARSEMEM && MEMORY_HOTPLUG
161		161
162	config MEMORY_HOTPLUG_DEFAULT_ONLINE	162	config MEMORY_HOTPLUG_DEFAULT_ONLINE
163	bool "Online the newly added memory blocks by default"	163	bool "Online the newly added memory blocks by default"
164	depends on MEMORY_HOTPLUG	164	depends on MEMORY_HOTPLUG
165	help	165	help
166	This option sets the default policy setting for memory hotplug	166	This option sets the default policy setting for memory hotplug
167	onlining policy (/sys/devices/system/memory/auto_online_blocks) which	167	onlining policy (/sys/devices/system/memory/auto_online_blocks) which
168	determines what happens to newly added memory regions. Policy setting	168	determines what happens to newly added memory regions. Policy setting
@@ -227,14 +227,14 @@ config COMPACTION · 58 unmodified lines
227	select MIGRATION	227	select MIGRATION
228	depends on MMU	228	depends on MMU
229	help	229	help
230	Compaction is the only memory management component to form	230	Compaction is the only memory management component to form
231	high order (larger physically contiguous) memory blocks	231	high order (larger physically contiguous) memory blocks
232	reliably. The page allocator relies on compaction heavily and	232	reliably. The page allocator relies on compaction heavily and
233	the lack of the feature can lead to unexpected OOM killer	233	the lack of the feature can lead to unexpected OOM killer
234	invocations for high order memory requests. You shouldn't	234	invocations for high order memory requests. You shouldn't
235	disable this option unless there really is a strong reason for	235	disable this option unless there really is a strong reason for
236	it and then we would be really interested to hear about that at	236	it and then we would be really interested to hear about that at
237	linux-mm@kvack.org.	237	linux-mm@kvack.org.
238		238
239	#	239	#
240	# support for page migration	240	# support for page migration
@@ -258,7 +258,7 @@ config ARCH_ENABLE_THP_MIGRATION · 17 unmodified lines
258	bool	258	bool
259		259
260	config CONTIG_ALLOC	260	config CONTIG_ALLOC
261	def_bool (MEMORY_ISOLATION && COMPACTION) \|\| CMA	261	def_bool (MEMORY_ISOLATION && COMPACTION) \|\| CMA
262		262
263	config PHYS_ADDR_T_64BIT	263	config PHYS_ADDR_T_64BIT
264	def_bool 64BIT	264	def_bool 64BIT
@@ -302,10 +302,10 @@ config KSM · 37 unmodified lines
302	root has set /sys/kernel/mm/ksm/run to 1 (if CONFIG_SYSFS is set).	302	root has set /sys/kernel/mm/ksm/run to 1 (if CONFIG_SYSFS is set).
303		303
304	config DEFAULT_MMAP_MIN_ADDR	304	config DEFAULT_MMAP_MIN_ADDR
305	int "Low address space to protect from user allocation"	305	int "Low address space to protect from user allocation"
306	depends on MMU	306	depends on MMU
307	default 4096	307	default 4096
308	help	308	help
309	This is the portion of low virtual memory which should be protected	309	This is the portion of low virtual memory which should be protected
310	from userspace allocation. Keeping a user from writing to low pages	310	from userspace allocation. Keeping a user from writing to low pages
311	can help reduce the impact of kernel NULL pointer bugs.	311	can help reduce the impact of kernel NULL pointer bugs.
@@ -408,7 +408,7 @@ choice · 96 unmodified lines
408	endchoice	408	endchoice
409		409
410	config ARCH_WANTS_THP_SWAP	410	config ARCH_WANTS_THP_SWAP
411	def_bool n	411	def_bool n
412		412
413	config THP_SWAP	413	config THP_SWAP
414	def_bool y	414	def_bool y

mm/cma.c+2−4

 static int __init cma_activate_area(struct cma *cma)
 {
-	int bitmap_size = BITS_TO_LONGS(cma_bitmap_maxno(cma)) * sizeof(long);
 	unsigned long base_pfn = cma->base_pfn, pfn = base_pfn;
 	unsigned i = cma->count >> pageblock_order;
 	struct zone *zone;
-	cma->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
+	cma->bitmap = bitmap_zalloc(cma_bitmap_maxno(cma), GFP_KERNEL);
 	if (!cma->bitmap) {
 		cma->count = 0;
 		return -ENOMEM;
 not_in_zone:
 	pr_err("CMA area %s could not be activated\n", cma->name);
-	kfree(cma->bitmap);
+	bitmap_free(cma->bitmap);
 	cma->count = 0;
 	return -EINVAL;
 }

mm/cma_debug.c+5−5

mm/filemap.c+29−25

 #ifdef CONFIG_MMU
 #define MMAP_LOTSAMISS  (100)
-static struct file *maybe_unlock_mmap_for_io(struct vm_fault *vmf,
-					     struct file *fpin)
-{
-	int flags = vmf->flags;
-	if (fpin)
-		return fpin;
-	/*
-	 * FAULT_FLAG_RETRY_NOWAIT means we don't want to wait on page locks or
-	 * anything, so we only pin the file and drop the mmap_sem if only
-	 * FAULT_FLAG_ALLOW_RETRY is set.
-	 */
-	if ((flags & (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT)) ==
-	    FAULT_FLAG_ALLOW_RETRY) {
-		fpin = get_file(vmf->vma->vm_file);
-		up_read(&vmf->vma->vm_mm->mmap_sem);
-	}
-	return fpin;
-}
 /*
  * lock_page_maybe_drop_mmap - lock the page, possibly dropping the mmap_sem
  * @vmf - the vm_fault for this fault.
 }
 EXPORT_SYMBOL(pagecache_write_end);
+/*
+ * Warn about a page cache invalidation failure during a direct I/O write.
+ */
+void dio_warn_stale_pagecache(struct file *filp)
+{
+	static DEFINE_RATELIMIT_STATE(_rs, 86400 * HZ, DEFAULT_RATELIMIT_BURST);
+	char pathname[128];
+	struct inode *inode = file_inode(filp);
+	char *path;
+	errseq_set(&inode->i_mapping->wb_err, -EIO);
+	if (__ratelimit(&_rs)) {
+		path = file_path(filp, pathname, sizeof(pathname));
+		if (IS_ERR(path))
+			path = "(unknown)";
+		pr_crit("Page cache invalidation failure on direct I/O.  Possible data corruption due to collision with buffered I/O!\n");
+		pr_crit("File: %s PID: %d Comm: %.20s\n", path, current->pid,
+			current->comm);
+	}
+}
 ssize_t
 generic_file_direct_write(struct kiocb *iocb, struct iov_iter *from)
 {
 	 * Most of the time we do not need this since dio_complete() will do
 	 * the invalidation for us. However there are some file systems that
 	 * do not end up with dio_complete() being called, so let's not break
-	 * them by removing it completely
+	 * them by removing it completely.
+	 *
+	 * Noticeable example is a blkdev_direct_IO().
+	 *
+	 * Skip invalidation for async writes or if mapping has no pages.
 	 */
-	if (mapping->nrpages)
+	if (written > 0 && mapping->nrpages &&
-		invalidate_inode_pages2_range(mapping,
+	    invalidate_inode_pages2_range(mapping, pos >> PAGE_SHIFT, end))
-					pos >> PAGE_SHIFT, end);
+		dio_warn_stale_pagecache(file);
 	if (written > 0) {
 		pos += written;

mm/gup.c+27−13

  *		Or NULL if the caller does not require them.
  * @nonblocking: whether waiting for disk IO or mmap_sem contention
  *
- * Returns number of pages pinned. This may be fewer than the number
+ * Returns either number of pages pinned (which may be less than the
- * requested. If nr_pages is 0 or negative, returns 0. If no pages
+ * number requested), or an error. Details about the return value:
- * were pinned, returns -errno. Each page returned must be released
+ *
- * with a put_page() call when it is finished with. vmas will only
+ * -- If nr_pages is 0, returns 0.
- * remain valid while mmap_sem is held.
+ * -- If nr_pages is >0, but no pages were pinned, returns -errno.
+ * -- If nr_pages is >0, and some pages were pinned, returns the number of
+ *    pages pinned. Again, this may be less than nr_pages.
+ *
+ * The caller is responsible for releasing returned @pages, via put_page().
+ *
+ * @vmas are valid only as long as mmap_sem is held.
  *
  * Must be called with mmap_sem held.  It may be released.  See below.
  *
  *		subsequently whether VM_FAULT_RETRY functionality can be
  *		utilised. Lock must initially be held.
  *
- * Returns number of pages pinned. This may be fewer than the number
+ * Returns either number of pages pinned (which may be less than the
- * requested. If nr_pages is 0 or negative, returns 0. If no pages
+ * number requested), or an error. Details about the return value:
- * were pinned, returns -errno. Each page returned must be released
+ *
- * with a put_page() call when it is finished with. vmas will only
+ * -- If nr_pages is 0, returns 0.
- * remain valid while mmap_sem is held.
+ * -- If nr_pages is >0, but no pages were pinned, returns -errno.
+ * -- If nr_pages is >0, and some pages were pinned, returns the number of
+ *    pages pinned. Again, this may be less than nr_pages.
+ *
+ * The caller is responsible for releasing returned @pages, via put_page().
+ *
+ * @vmas are valid only as long as mmap_sem is held.
  *
  * Must be called with mmap_sem held for read or write.
  *
 	bool drain_allow = true;
 	bool migrate_allow = true;
 	LIST_HEAD(cma_page_list);
+	long ret = nr_pages;
 check_again:
 	for (i = 0; i < nr_pages;) {
 		 * again migrating any new CMA pages which we failed to isolate
 		 * earlier.
 		 */
-		nr_pages = __get_user_pages_locked(tsk, mm, start, nr_pages,
+		ret = __get_user_pages_locked(tsk, mm, start, nr_pages,
 						   pages, vmas, NULL,
 						   gup_flags);
-		if ((nr_pages > 0) && migrate_allow) {
+		if ((ret > 0) && migrate_allow) {
+			nr_pages = ret;
 			drain_allow = true;
 			goto check_again;
 		}
 	}
-	return nr_pages;
+	return ret;
 }
 #else
 static long check_and_migrate_cma_pages(struct task_struct *tsk,

mm/huge_memory.c+1−1

@@ -3003,7 +3003,7 @@ next: · 3002 unmodified lines
3003		3003
3004	return 0;	3004	return 0;
3005	}	3005	}
3006	DEFINE_SIMPLE_ATTRIBUTE(split_huge_pages_fops, NULL, split_huge_pages_set,	3006	DEFINE_DEBUGFS_ATTRIBUTE(split_huge_pages_fops, NULL, split_huge_pages_set,
3007	"%llu\n");	3007	"%llu\n");
3008		3008
3009	static int __init split_huge_pages_debugfs(void)	3009	static int __init split_huge_pages_debugfs(void)

mm/hugetlb.c+91−197

 	long to;
 };
+/* Must be called with resv->lock held. Calling this with count_only == true
+ * will count the number of pages to be added but will not modify the linked
+ * list.
+ */
+static long add_reservation_in_range(struct resv_map *resv, long f, long t,
+				     bool count_only)
+{
+	long chg = 0;
+	struct list_head *head = &resv->regions;
+	struct file_region *rg = NULL, *trg = NULL, *nrg = NULL;
+	/* Locate the region we are before or in. */
+	list_for_each_entry(rg, head, link)
+		if (f <= rg->to)
+			break;
+	/* Round our left edge to the current segment if it encloses us. */
+	if (f > rg->from)
+		f = rg->from;
+	chg = t - f;
+	/* Check for and consume any regions we now overlap with. */
+	nrg = rg;
+	list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
+		if (&rg->link == head)
+			break;
+		if (rg->from > t)
+			break;
+		/* We overlap with this area, if it extends further than
+		 * us then we must extend ourselves.  Account for its
+		 * existing reservation.
+		 */
+		if (rg->to > t) {
+			chg += rg->to - t;
+			t = rg->to;
+		}
+		chg -= rg->to - rg->from;
+		if (!count_only && rg != nrg) {
+			list_del(&rg->link);
+			kfree(rg);
+		}
+	}
+	if (!count_only) {
+		nrg->from = f;
+		nrg->to = t;
+	}
+	return chg;
+}
 /*
  * Add the huge page range represented by [f, t) to the reserve
- * map.  In the normal case, existing regions will be expanded
+ * map.  Existing regions will be expanded to accommodate the specified
- * to accommodate the specified range.  Sufficient regions should
+ * range, or a region will be taken from the cache.  Sufficient regions
- * exist for expansion due to the previous call to region_chg
+ * must exist in the cache due to the previous call to region_chg with
- * with the same range.  However, it is possible that region_del
+ * the same range.
- * could have been called after region_chg and modifed the map
- * in such a way that no region exists to be expanded.  In this
- * case, pull a region descriptor from the cache associated with
- * the map and use that for the new range.
  *
  * Return the number of new huge pages added to the map.  This
  * number is greater than or equal to zero.
 static long region_add(struct resv_map *resv, long f, long t)
 {
 	struct list_head *head = &resv->regions;
-	struct file_region *rg, *nrg, *trg;
+	struct file_region *rg, *nrg;
 	long add = 0;
 	spin_lock(&resv->lock);
 	/*
 	 * If no region exists which can be expanded to include the
-	 * specified range, the list must have been modified by an
+	 * specified range, pull a region descriptor from the cache
-	 * interleving call to region_del().  Pull a region descriptor
+	 * and use it for this range.
-	 * from the cache and use it for this range.
 	 */
 	if (&rg->link == head || t < rg->from) {
 		VM_BUG_ON(resv->region_cache_count <= 0);
 		goto out_locked;
 	}
-	/* Round our left edge to the current segment if it encloses us. */
+	add = add_reservation_in_range(resv, f, t, false);
-	if (f > rg->from)
-		f = rg->from;
-	/* Check for and consume any regions we now overlap with. */
-	nrg = rg;
-	list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
-		if (&rg->link == head)
-			break;
-		if (rg->from > t)
-			break;
-		/* If this area reaches higher then extend our area to
-		 * include it completely.  If this is not the first area
-		 * which we intend to reuse, free it. */
-		if (rg->to > t)
-			t = rg->to;
-		if (rg != nrg) {
-			/* Decrement return value by the deleted range.
-			 * Another range will span this area so that by
-			 * end of routine add will be >= zero
-			 */
-			add -= (rg->to - rg->from);
-			list_del(&rg->link);
-			kfree(rg);
-		}
-	}
-	add += (nrg->from - f);		/* Added to beginning of region */
-	nrg->from = f;
-	add += t - nrg->to;		/* Added to end of region */
-	nrg->to = t;
 out_locked:
 	resv->adds_in_progress--;
  * call to region_add that will actually modify the reserve
  * map to add the specified range [f, t).  region_chg does
  * not change the number of huge pages represented by the
- * map.  However, if the existing regions in the map can not
+ * map.  A new file_region structure is added to the cache
- * be expanded to represent the new range, a new file_region
+ * as a placeholder, so that the subsequent region_add
- * structure is added to the map as a placeholder.  This is
+ * call will have all the regions it needs and will not fail.
- * so that the subsequent region_add call will have all the
- * regions it needs and will not fail.
- *
- * Upon entry, region_chg will also examine the cache of region descriptors
- * associated with the map.  If there are not enough descriptors cached, one
- * will be allocated for the in progress add operation.
  *
  * Returns the number of huge pages that need to be added to the existing
  * reservation map for the range [f, t).  This number is greater or equal to
  */
 static long region_chg(struct resv_map *resv, long f, long t)
 {
-	struct list_head *head = &resv->regions;
-	struct file_region *rg, *nrg = NULL;
 	long chg = 0;
-retry:
 	spin_lock(&resv->lock);
 retry_locked:
 	resv->adds_in_progress++;
 		spin_unlock(&resv->lock);
 		trg = kmalloc(sizeof(*trg), GFP_KERNEL);
-		if (!trg) {
+		if (!trg)
-			kfree(nrg);
 			return -ENOMEM;
-		}
 		spin_lock(&resv->lock);
 		list_add(&trg->link, &resv->region_cache);
 		goto retry_locked;
 	}
-	/* Locate the region we are before or in. */
+	chg = add_reservation_in_range(resv, f, t, true);
-	list_for_each_entry(rg, head, link)
-		if (f <= rg->to)
-			break;
-	/* If we are below the current region then a new region is required.
-	 * Subtle, allocate a new region at the position but make it zero
-	 * size such that we can guarantee to record the reservation. */
-	if (&rg->link == head || t < rg->from) {
-		if (!nrg) {
-			resv->adds_in_progress--;
-			spin_unlock(&resv->lock);
-			nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
-			if (!nrg)
-				return -ENOMEM;
-			nrg->from = f;
-			nrg->to   = f;
-			INIT_LIST_HEAD(&nrg->link);
-			goto retry;
-		}
-		list_add(&nrg->link, rg->link.prev);
-		chg = t - f;
-		goto out_nrg;
-	}
-	/* Round our left edge to the current segment if it encloses us. */
-	if (f > rg->from)
-		f = rg->from;
-	chg = t - f;
-	/* Check for and consume any regions we now overlap with. */
-	list_for_each_entry(rg, rg->link.prev, link) {
-		if (&rg->link == head)
-			break;
-		if (rg->from > t)
-			goto out;
-		/* We overlap with this area, if it extends further than
-		 * us then we must extend ourselves.  Account for its
-		 * existing reservation. */
-		if (rg->to > t) {
-			chg += rg->to - t;
-			t = rg->to;
-		}
-		chg -= rg->to - rg->from;
-	}
-out:
-	spin_unlock(&resv->lock);
-	/*  We already know we raced and no longer need the new region */
-	kfree(nrg);
-	return chg;
-out_nrg:
 	spin_unlock(&resv->lock);
 	return chg;
 }
 }
 #ifdef CONFIG_CONTIG_ALLOC
-static int __alloc_gigantic_page(unsigned long start_pfn,
-				unsigned long nr_pages, gfp_t gfp_mask)
-{
-	unsigned long end_pfn = start_pfn + nr_pages;
-	return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE,
-				  gfp_mask);
-}
-static bool pfn_range_valid_gigantic(struct zone *z,
-			unsigned long start_pfn, unsigned long nr_pages)
-{
-	unsigned long i, end_pfn = start_pfn + nr_pages;
-	struct page *page;
-	for (i = start_pfn; i < end_pfn; i++) {
-		page = pfn_to_online_page(i);
-		if (!page)
-			return false;
-		if (page_zone(page) != z)
-			return false;
-		if (PageReserved(page))
-			return false;
-		if (page_count(page) > 0)
-			return false;
-		if (PageHuge(page))
-			return false;
-	}
-	return true;
-}
-static bool zone_spans_last_pfn(const struct zone *zone,
-			unsigned long start_pfn, unsigned long nr_pages)
-{
-	unsigned long last_pfn = start_pfn + nr_pages - 1;
-	return zone_spans_pfn(zone, last_pfn);
-}
 static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
 		int nid, nodemask_t *nodemask)
 {
-	unsigned int order = huge_page_order(h);
+	unsigned long nr_pages = 1UL << huge_page_order(h);
-	unsigned long nr_pages = 1 << order;
-	unsigned long ret, pfn, flags;
-	struct zonelist *zonelist;
-	struct zone *zone;
-	struct zoneref *z;
-	zonelist = node_zonelist(nid, gfp_mask);
+	return alloc_contig_pages(nr_pages, gfp_mask, nid, nodemask);
-	for_each_zone_zonelist_nodemask(zone, z, zonelist, gfp_zone(gfp_mask), nodemask) {
-		spin_lock_irqsave(&zone->lock, flags);
-		pfn = ALIGN(zone->zone_start_pfn, nr_pages);
-		while (zone_spans_last_pfn(zone, pfn, nr_pages)) {
-			if (pfn_range_valid_gigantic(zone, pfn, nr_pages)) {
-				/*
-				 * We release the zone lock here because
-				 * alloc_contig_range() will also lock the zone
-				 * at some point. If there's an allocation
-				 * spinning on this lock, it may win the race
-				 * and cause alloc_contig_range() to fail...
-				 */
-				spin_unlock_irqrestore(&zone->lock, flags);
-				ret = __alloc_gigantic_page(pfn, nr_pages, gfp_mask);
-				if (!ret)
-					return pfn_to_page(pfn);
-				spin_lock_irqsave(&zone->lock, flags);
-			}
-			pfn += nr_pages;
-		}
-		spin_unlock_irqrestore(&zone->lock, flags);
-	}
-	return NULL;
 }
 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid);
 			 * handling userfault.  Reacquire after handling
 			 * fault to make calling code simpler.
 			 */
-			hash = hugetlb_fault_mutex_hash(h, mapping, idx, haddr);
+			hash = hugetlb_fault_mutex_hash(mapping, idx);
 			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
 			ret = handle_userfault(&vmf, VM_UFFD_MISSING);
 			mutex_lock(&hugetlb_fault_mutex_table[hash]);
 }
 #ifdef CONFIG_SMP
-u32 hugetlb_fault_mutex_hash(struct hstate *h, struct address_space *mapping,
+u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
-			    pgoff_t idx, unsigned long address)
 {
 	unsigned long key[2];
 	u32 hash;
 	key[0] = (unsigned long) mapping;
 	key[1] = idx;
-	hash = jhash2((u32 *)&key, sizeof(key)/sizeof(u32), 0);
+	hash = jhash2((u32 *)&key, sizeof(key)/(sizeof(u32)), 0);
 	return hash & (num_fault_mutexes - 1);
 }
  * For uniprocesor systems we always use a single mutex, so just
  * return 0 and avoid the hashing overhead.
  */
-u32 hugetlb_fault_mutex_hash(struct hstate *h, struct address_space *mapping,
+u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
-			    pgoff_t idx, unsigned long address)
 {
 	return 0;
 }
 	 * get spurious allocation failures if two CPUs race to instantiate
 	 * the same page in the page cache.
 	 */
-	hash = hugetlb_fault_mutex_hash(h, mapping, idx, haddr);
+	hash = hugetlb_fault_mutex_hash(mapping, idx);
 	mutex_lock(&hugetlb_fault_mutex_table[hash]);
 	entry = huge_ptep_get(ptep);
 				break;
 			}
 		}
+		/*
+		 * If subpage information not requested, update counters
+		 * and skip the same_page loop below.
+		 */
+		if (!pages && !vmas && !pfn_offset &&
+		    (vaddr + huge_page_size(h) < vma->vm_end) &&
+		    (remainder >= pages_per_huge_page(h))) {
+			vaddr += huge_page_size(h);
+			remainder -= pages_per_huge_page(h);
+			i += pages_per_huge_page(h);
+			spin_unlock(ptl);
+			continue;
+		}
 same_page:
 		if (pages) {
 			pages[i] = mem_map_offset(page, pfn_offset);
 	if (!vma_shareable(vma, addr))
 		return (pte_t *)pmd_alloc(mm, pud, addr);
-	i_mmap_lock_write(mapping);
+	i_mmap_lock_read(mapping);
 	vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
 		if (svma == vma)
 			continue;
 	spin_unlock(ptl);
 out:
 	pte = (pte_t *)pmd_alloc(mm, pud, addr);
-	i_mmap_unlock_write(mapping);
+	i_mmap_unlock_read(mapping);
 	return pte;
 }

mm/hwpoison-inject.c+2−2

@@ -67,8 +67,8 @@ static int hwpoison_unpoison(void *data, u64 val) · 66 unmodified lines
67	return unpoison_memory(val);	67	return unpoison_memory(val);
68	}	68	}
69		69
70	DEFINE_SIMPLE_ATTRIBUTE(hwpoison_fops, NULL, hwpoison_inject, "%lli\n");	70	DEFINE_DEBUGFS_ATTRIBUTE(hwpoison_fops, NULL, hwpoison_inject, "%lli\n");
71	DEFINE_SIMPLE_ATTRIBUTE(unpoison_fops, NULL, hwpoison_unpoison, "%lli\n");	71	DEFINE_DEBUGFS_ATTRIBUTE(unpoison_fops, NULL, hwpoison_unpoison, "%lli\n");
72		72
73	static void pfn_inject_exit(void)	73	static void pfn_inject_exit(void)
74	{	74	{

mm/internal.h+26−1

@@ -165,6 +165,9 @@ extern void post_alloc_hook(struct page *page, unsigned int order, · 164 unmodified lines
165	gfp_t gfp_flags);	165	gfp_t gfp_flags);
166	extern int user_min_free_kbytes;	166	extern int user_min_free_kbytes;
167		167
		168	extern void zone_pcp_update(struct zone *zone);
		169	extern void zone_pcp_reset(struct zone *zone);
		170
168	#if defined CONFIG_COMPACTION \|\| defined CONFIG_CMA	171	#if defined CONFIG_COMPACTION \|\| defined CONFIG_CMA
169		172
170	/*	173	/*
@@ -290,7 +293,8 @@ static inline bool is_data_mapping(vm_flags_t flags) · 119 unmodified lines
290		293
291	/* mm/util.c */	294	/* mm/util.c */
292	void __vma_link_list(struct mm_struct mm, struct vm_area_struct vma,	295	void __vma_link_list(struct mm_struct mm, struct vm_area_struct vma,
293	struct vm_area_struct prev, struct rb_node rb_parent);	296	struct vm_area_struct *prev);
		297	void __vma_unlink_list(struct mm_struct mm, struct vm_area_struct vma);
294		298
295	#ifdef CONFIG_MMU	299	#ifdef CONFIG_MMU
296	extern long populate_vma_page_range(struct vm_area_struct *vma,	300	extern long populate_vma_page_range(struct vm_area_struct *vma,
@@ -362,6 +366,27 @@ vma_address(struct page page, struct vm_area_struct vma) · 65 unmodified lines
362	return max(start, vma->vm_start);	366	return max(start, vma->vm_start);
363	}	367	}
364		368
		369	static inline struct file maybe_unlock_mmap_for_io(struct vm_fault vmf,
		370	struct file *fpin)
		371	{
		372	int flags = vmf->flags;
		373
		374	if (fpin)
		375	return fpin;
		376
		377	/*
		378	* FAULT_FLAG_RETRY_NOWAIT means we don't want to wait on page locks or
		379	* anything, so we only pin the file and drop the mmap_sem if only
		380	* FAULT_FLAG_ALLOW_RETRY is set.
		381	*/
		382	if ((flags & (FAULT_FLAG_ALLOW_RETRY \| FAULT_FLAG_RETRY_NOWAIT)) ==
		383	FAULT_FLAG_ALLOW_RETRY) {
		384	fpin = get_file(vmf->vma->vm_file);
		385	up_read(&vmf->vma->vm_mm->mmap_sem);
		386	}
		387	return fpin;
		388	}
		389
365	#else /* !CONFIG_MMU */	390	#else /* !CONFIG_MMU */
366	static inline void clear_page_mlock(struct page *page) { }	391	static inline void clear_page_mlock(struct page *page) { }
367	static inline void mlock_vma_page(struct page *page) { }	392	static inline void mlock_vma_page(struct page *page) { }

mm/kasan/common.c+233−0

@@ -36,6 +36,8 @@ · 35 unmodified lines
36	#include <linux/bug.h>	36	#include <linux/bug.h>
37	#include <linux/uaccess.h>	37	#include <linux/uaccess.h>
38		38
		39	#include <asm/tlbflush.h>
		40
39	#include "kasan.h"	41	#include "kasan.h"
40	#include "../slab.h"	42	#include "../slab.h"
41		43
@@ -590,6 +592,7 @@ void kasan_kfree_large(void *ptr, unsigned long ip) · 548 unmodified lines
590	/* The object will be poisoned by page_alloc. */	592	/* The object will be poisoned by page_alloc. */
591	}	593	}
592		594
		595	#ifndef CONFIG_KASAN_VMALLOC
593	int kasan_module_alloc(void *addr, size_t size)	596	int kasan_module_alloc(void *addr, size_t size)
594	{	597	{
595	void *ret;	598	void *ret;
@@ -625,6 +628,7 @@ void kasan_free_shadow(const struct vm_struct *vm) · 29 unmodified lines
625	if (vm->flags & VM_KASAN)	628	if (vm->flags & VM_KASAN)
626	vfree(kasan_mem_to_shadow(vm->addr));	629	vfree(kasan_mem_to_shadow(vm->addr));
627	}	630	}
		631	#endif
628		632
629	extern void __kasan_report(unsigned long addr, size_t size, bool is_write, unsigned long ip);	633	extern void __kasan_report(unsigned long addr, size_t size, bool is_write, unsigned long ip);
630		634
@@ -744,3 +748,232 @@ static int __init kasan_memhotplug_init(void) · 113 unmodified lines
744		748
745	core_initcall(kasan_memhotplug_init);	749	core_initcall(kasan_memhotplug_init);
746	#endif	750	#endif
		751
		752	#ifdef CONFIG_KASAN_VMALLOC
		753	static int kasan_populate_vmalloc_pte(pte_t *ptep, unsigned long addr,
		754	void *unused)
		755	{
		756	unsigned long page;
		757	pte_t pte;
		758
		759	if (likely(!pte_none(*ptep)))
		760	return 0;
		761
		762	page = __get_free_page(GFP_KERNEL);
		763	if (!page)
		764	return -ENOMEM;
		765
		766	memset((void *)page, KASAN_VMALLOC_INVALID, PAGE_SIZE);
		767	pte = pfn_pte(PFN_DOWN(__pa(page)), PAGE_KERNEL);
		768
		769	spin_lock(&init_mm.page_table_lock);
		770	if (likely(pte_none(*ptep))) {
		771	set_pte_at(&init_mm, addr, ptep, pte);
		772	page = 0;
		773	}
		774	spin_unlock(&init_mm.page_table_lock);
		775	if (page)
		776	free_page(page);
		777	return 0;
		778	}
		779
		780	int kasan_populate_vmalloc(unsigned long requested_size, struct vm_struct *area)
		781	{
		782	unsigned long shadow_start, shadow_end;
		783	int ret;
		784
		785	shadow_start = (unsigned long)kasan_mem_to_shadow(area->addr);
		786	shadow_start = ALIGN_DOWN(shadow_start, PAGE_SIZE);
		787	shadow_end = (unsigned long)kasan_mem_to_shadow(area->addr +
		788	area->size);
		789	shadow_end = ALIGN(shadow_end, PAGE_SIZE);
		790
		791	ret = apply_to_page_range(&init_mm, shadow_start,
		792	shadow_end - shadow_start,
		793	kasan_populate_vmalloc_pte, NULL);
		794	if (ret)
		795	return ret;
		796
		797	flush_cache_vmap(shadow_start, shadow_end);
		798
		799	kasan_unpoison_shadow(area->addr, requested_size);
		800
		801	area->flags \|= VM_KASAN;
		802
		803	/*
		804	* We need to be careful about inter-cpu effects here. Consider:
		805	*
		806	* CPU#0 CPU#1
		807	* WRITE_ONCE(p, vmalloc(100)); while (x = READ_ONCE(p)) ;
		808	* p[99] = 1;
		809	*
		810	* With compiler instrumentation, that ends up looking like this:
		811	*
		812	* CPU#0 CPU#1
		813	* // vmalloc() allocates memory
		814	* // let a = area->addr
		815	* // we reach kasan_populate_vmalloc
		816	* // and call kasan_unpoison_shadow:
		817	* STORE shadow(a), unpoison_val
		818	* ...
		819	* STORE shadow(a+99), unpoison_val x = LOAD p
		820	* // rest of vmalloc process <data dependency>
		821	* STORE p, a LOAD shadow(x+99)
		822	*
		823	* If there is no barrier between the end of unpoisioning the shadow
		824	* and the store of the result to p, the stores could be committed
		825	* in a different order by CPU#0, and CPU#1 could erroneously observe
		826	* poison in the shadow.
		827	*
		828	* We need some sort of barrier between the stores.
		829	*
		830	* In the vmalloc() case, this is provided by a smp_wmb() in
		831	* clear_vm_uninitialized_flag(). In the per-cpu allocator and in
		832	* get_vm_area() and friends, the caller gets shadow allocated but
		833	* doesn't have any pages mapped into the virtual address space that
		834	* has been reserved. Mapping those pages in will involve taking and
		835	* releasing a page-table lock, which will provide the barrier.
		836	*/
		837
		838	return 0;
		839	}
		840
		841	/*
		842	* Poison the shadow for a vmalloc region. Called as part of the
		843	* freeing process at the time the region is freed.
		844	*/
		845	void kasan_poison_vmalloc(void *start, unsigned long size)
		846	{
		847	size = round_up(size, KASAN_SHADOW_SCALE_SIZE);
		848	kasan_poison_shadow(start, size, KASAN_VMALLOC_INVALID);
		849	}
		850
		851	static int kasan_depopulate_vmalloc_pte(pte_t *ptep, unsigned long addr,
		852	void *unused)
		853	{
		854	unsigned long page;
		855
		856	page = (unsigned long)__va(pte_pfn(*ptep) << PAGE_SHIFT);
		857
		858	spin_lock(&init_mm.page_table_lock);
		859
		860	if (likely(!pte_none(*ptep))) {
		861	pte_clear(&init_mm, addr, ptep);
		862	free_page(page);
		863	}
		864	spin_unlock(&init_mm.page_table_lock);
		865
		866	return 0;
		867	}
		868
		869	/*
		870	* Release the backing for the vmalloc region [start, end), which
		871	* lies within the free region [free_region_start, free_region_end).
		872	*
		873	* This can be run lazily, long after the region was freed. It runs
		874	* under vmap_area_lock, so it's not safe to interact with the vmalloc/vmap
		875	* infrastructure.
		876	*
		877	* How does this work?
		878	* -------------------
		879	*
		880	* We have a region that is page aligned, labelled as A.
		881	* That might not map onto the shadow in a way that is page-aligned:
		882	*
		883	* start end
		884	* v v
		885	* \|????????\|????????\|AAAAAAAA\|AA....AA\|AAAAAAAA\|????????\| < vmalloc
		886	* -------- -------- -------- -------- --------
		887	* \| \| \| \| \|
		888	* \| \| \| /-------/ \|
		889	* \-------\\|/------/ \|/---------------/
		890	* \|\|\| \|\|
		891	* \|??AAAAAA\|AAAAAAAA\|AA??????\| < shadow
		892	* (1) (2) (3)
		893	*
		894	* First we align the start upwards and the end downwards, so that the
		895	* shadow of the region aligns with shadow page boundaries. In the
		896	* example, this gives us the shadow page (2). This is the shadow entirely
		897	* covered by this allocation.
		898	*
		899	* Then we have the tricky bits. We want to know if we can free the
		900	* partially covered shadow pages - (1) and (3) in the example. For this,
		901	* we are given the start and end of the free region that contains this
		902	* allocation. Extending our previous example, we could have:
		903	*
		904	* free_region_start free_region_end
		905	* \| start end \|
		906	* v v v v
		907	* \|FFFFFFFF\|FFFFFFFF\|AAAAAAAA\|AA....AA\|AAAAAAAA\|FFFFFFFF\| < vmalloc
		908	* -------- -------- -------- -------- --------
		909	* \| \| \| \| \|
		910	* \| \| \| /-------/ \|
		911	* \-------\\|/------/ \|/---------------/
		912	* \|\|\| \|\|
		913	* \|FFAAAAAA\|AAAAAAAA\|AAF?????\| < shadow
		914	* (1) (2) (3)
		915	*
		916	* Once again, we align the start of the free region up, and the end of
		917	* the free region down so that the shadow is page aligned. So we can free
		918	* page (1) - we know no allocation currently uses anything in that page,
		919	* because all of it is in the vmalloc free region. But we cannot free
		920	* page (3), because we can't be sure that the rest of it is unused.
		921	*
		922	* We only consider pages that contain part of the original region for
		923	* freeing: we don't try to free other pages from the free region or we'd
		924	* end up trying to free huge chunks of virtual address space.
		925	*
		926	* Concurrency
		927	* -----------
		928	*
		929	* How do we know that we're not freeing a page that is simultaneously
		930	* being used for a fresh allocation in kasan_populate_vmalloc(_pte)?
		931	*
		932	* We _can_ have kasan_release_vmalloc and kasan_populate_vmalloc running
		933	* at the same time. While we run under free_vmap_area_lock, the population
		934	* code does not.
		935	*
		936	* free_vmap_area_lock instead operates to ensure that the larger range
		937	* [free_region_start, free_region_end) is safe: because __alloc_vmap_area and
		938	* the per-cpu region-finding algorithm both run under free_vmap_area_lock,
		939	* no space identified as free will become used while we are running. This
		940	* means that so long as we are careful with alignment and only free shadow
		941	* pages entirely covered by the free region, we will not run in to any
		942	* trouble - any simultaneous allocations will be for disjoint regions.
		943	*/
		944	void kasan_release_vmalloc(unsigned long start, unsigned long end,
		945	unsigned long free_region_start,
		946	unsigned long free_region_end)
		947	{
		948	void shadow_start, shadow_end;
		949	unsigned long region_start, region_end;
		950
		951	region_start = ALIGN(start, PAGE_SIZE * KASAN_SHADOW_SCALE_SIZE);
		952	region_end = ALIGN_DOWN(end, PAGE_SIZE * KASAN_SHADOW_SCALE_SIZE);
		953
		954	free_region_start = ALIGN(free_region_start,
		955	PAGE_SIZE * KASAN_SHADOW_SCALE_SIZE);
		956
		957	if (start != region_start &&
		958	free_region_start < region_start)
		959	region_start -= PAGE_SIZE * KASAN_SHADOW_SCALE_SIZE;
		960
		961	free_region_end = ALIGN_DOWN(free_region_end,
		962	PAGE_SIZE * KASAN_SHADOW_SCALE_SIZE);
		963
		964	if (end != region_end &&
		965	free_region_end > region_end)
		966	region_end += PAGE_SIZE * KASAN_SHADOW_SCALE_SIZE;
		967
		968	shadow_start = kasan_mem_to_shadow((void *)region_start);
		969	shadow_end = kasan_mem_to_shadow((void *)region_end);
		970
		971	if (shadow_end > shadow_start) {
		972	apply_to_page_range(&init_mm, (unsigned long)shadow_start,
		973	(unsigned long)(shadow_end - shadow_start),
		974	kasan_depopulate_vmalloc_pte, NULL);
		975	flush_tlb_kernel_range((unsigned long)shadow_start,
		976	(unsigned long)shadow_end);
		977	}
		978	}
		979	#endif

mm/kasan/generic_report.c+3−0

 	case KASAN_ALLOCA_RIGHT:
 		bug_type = "alloca-out-of-bounds";
 		break;
+	case KASAN_VMALLOC_INVALID:
+		bug_type = "vmalloc-out-of-bounds";
+		break;
 	}
 	return bug_type;

mm/kasan/kasan.h+1−0

@@ -25,6 +25,7 @@ · 24 unmodified lines
25	#endif	25	#endif
26		26
27	#define KASAN_GLOBAL_REDZONE 0xFA /* redzone for global variable */	27	#define KASAN_GLOBAL_REDZONE 0xFA /* redzone for global variable */
		28	#define KASAN_VMALLOC_INVALID 0xF9 /* unallocated space in vmapped page */
28		29
29	/*	30	/*
30	* Stack redzone shadow values	31	* Stack redzone shadow values

mm/khugepaged.c+18−0

@@ -1602,6 +1602,24 @@ static void collapse_file(struct mm_struct *mm, · 1601 unmodified lines
1602	result = SCAN_FAIL;	1602	result = SCAN_FAIL;
1603	goto xa_unlocked;	1603	goto xa_unlocked;
1604	}	1604	}
		1605	} else if (PageDirty(page)) {
		1606	/*
		1607	* khugepaged only works on read-only fd,
		1608	* so this page is dirty because it hasn't
		1609	* been flushed since first write. There
		1610	* won't be new dirty pages.
		1611	*
		1612	* Trigger async flush here and hope the
		1613	* writeback is done when khugepaged
		1614	* revisits this page.
		1615	*
		1616	* This is a one-off situation. We are not
		1617	* forcing writeback in loop.
		1618	*/
		1619	xas_unlock_irq(&xas);
		1620	filemap_flush(mapping);
		1621	result = SCAN_FAIL;
		1622	goto xa_unlocked;
1605	} else if (trylock_page(page)) {	1623	} else if (trylock_page(page)) {
1606	get_page(page);	1624	get_page(page);
1607	xas_unlock_irq(&xas);	1625	xas_unlock_irq(&xas);

mm/madvise.c+7−7

@@ -864,13 +864,13 @@ static int madvise_inject_error(int behavior, · 863 unmodified lines
864	{	864	{
865	struct page *page;	865	struct page *page;
866	struct zone *zone;	866	struct zone *zone;
867	unsigned int order;	867	unsigned long size;
868		868
869	if (!capable(CAP_SYS_ADMIN))	869	if (!capable(CAP_SYS_ADMIN))
870	return -EPERM;	870	return -EPERM;
871		871
872		872
873	for (; start < end; start += PAGE_SIZE << order) {	873	for (; start < end; start += size) {
874	unsigned long pfn;	874	unsigned long pfn;
875	int ret;	875	int ret;
876		876
@@ -882,9 +882,9 @@ static int madvise_inject_error(int behavior, · 5 unmodified lines
882	/*	882	/*
883	* When soft offlining hugepages, after migrating the page	883	* When soft offlining hugepages, after migrating the page
884	* we dissolve it, therefore in the second loop "page" will	884	* we dissolve it, therefore in the second loop "page" will
885	* no longer be a compound page, and order will be 0.	885	* no longer be a compound page.
886	*/	886	*/
887	order = compound_order(compound_head(page));	887	size = page_size(compound_head(page));
888		888
889	if (PageHWPoison(page)) {	889	if (PageHWPoison(page)) {
890	put_page(page);	890	put_page(page);
@@ -895,7 +895,7 @@ static int madvise_inject_error(int behavior, · 4 unmodified lines
895	pr_info("Soft offlining pfn %#lx at process virtual address %#lx\n",	895	pr_info("Soft offlining pfn %#lx at process virtual address %#lx\n",
896	pfn, start);	896	pfn, start);
897		897
898	ret = soft_offline_page(page, MF_COUNT_INCREASED);	898	ret = soft_offline_page(pfn, MF_COUNT_INCREASED);
899	if (ret)	899	if (ret)
900	return ret;	900	return ret;
901	continue;	901	continue;
@@ -1059,9 +1059,9 @@ SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior) · 157 unmodified lines
1059	if (!madvise_behavior_valid(behavior))	1059	if (!madvise_behavior_valid(behavior))
1060	return error;	1060	return error;
1061		1061
1062	if (start & ~PAGE_MASK)	1062	if (!PAGE_ALIGNED(start))
1063	return error;	1063	return error;
1064	len = (len_in + ~PAGE_MASK) & PAGE_MASK;	1064	len = PAGE_ALIGN(len_in);
1065		1065
1066	/* Check to see whether len was rounded up from small -ve to zero */	1066	/* Check to see whether len was rounded up from small -ve to zero */
1067	if (len_in && !len)	1067	if (len_in && !len)

mm/memblock.c+75−36

  * at build time. The region arrays for the "memory" and "reserved"
  * types are initially sized to %INIT_MEMBLOCK_REGIONS and for the
  * "physmap" type to %INIT_PHYSMEM_REGIONS.
- * The :c:func:`memblock_allow_resize` enables automatic resizing of
+ * The memblock_allow_resize() enables automatic resizing of the region
- * the region arrays during addition of new regions. This feature
+ * arrays during addition of new regions. This feature should be used
- * should be used with care so that memory allocated for the region
+ * with care so that memory allocated for the region array will not
- * array will not overlap with areas that should be reserved, for
+ * overlap with areas that should be reserved, for example initrd.
- * example initrd.
  *
  * The early architecture setup should tell memblock what the physical
- * memory layout is by using :c:func:`memblock_add` or
+ * memory layout is by using memblock_add() or memblock_add_node()
- * :c:func:`memblock_add_node` functions. The first function does not
+ * functions. The first function does not assign the region to a NUMA
- * assign the region to a NUMA node and it is appropriate for UMA
+ * node and it is appropriate for UMA systems. Yet, it is possible to
- * systems. Yet, it is possible to use it on NUMA systems as well and
+ * use it on NUMA systems as well and assign the region to a NUMA node
- * assign the region to a NUMA node later in the setup process using
+ * later in the setup process using memblock_set_node(). The
- * :c:func:`memblock_set_node`. The :c:func:`memblock_add_node`
+ * memblock_add_node() performs such an assignment directly.
- * performs such an assignment directly.
  *
  * Once memblock is setup the memory can be allocated using one of the
  * API variants:
  *
- * * :c:func:`memblock_phys_alloc*` - these functions return the
+ * * memblock_phys_alloc*() - these functions return the **physical**
- *   **physical** address of the allocated memory
+ *   address of the allocated memory
- * * :c:func:`memblock_alloc*` - these functions return the **virtual**
+ * * memblock_alloc*() - these functions return the **virtual** address
- *   address of the allocated memory.
+ *   of the allocated memory.
  *
  * Note, that both API variants use implict assumptions about allowed
  * memory ranges and the fallback methods. Consult the documentation
- * of :c:func:`memblock_alloc_internal` and
+ * of memblock_alloc_internal() and memblock_alloc_range_nid()
- * :c:func:`memblock_alloc_range_nid` functions for more elaboarte
+ * functions for more elaborate description.
- * description.
  *
- * As the system boot progresses, the architecture specific
+ * As the system boot progresses, the architecture specific mem_init()
- * :c:func:`mem_init` function frees all the memory to the buddy page
+ * function frees all the memory to the buddy page allocator.
- * allocator.
  *
- * Unless an architecure enables %CONFIG_ARCH_KEEP_MEMBLOCK, the
+ * Unless an architecture enables %CONFIG_ARCH_KEEP_MEMBLOCK, the
  * memblock data structures will be discarded after the system
- * initialization compltes.
+ * initialization completes.
  */
 #ifndef CONFIG_NEED_MULTIPLE_NODES
  * @start: the lower bound of the memory region to allocate (phys address)
  * @end: the upper bound of the memory region to allocate (phys address)
  * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
+ * @exact_nid: control the allocation fall back to other nodes
  *
  * The allocation is performed from memory region limited by
- * memblock.current_limit if @max_addr == %MEMBLOCK_ALLOC_ACCESSIBLE.
+ * memblock.current_limit if @end == %MEMBLOCK_ALLOC_ACCESSIBLE.
  *
- * If the specified node can not hold the requested memory the
+ * If the specified node can not hold the requested memory and @exact_nid
- * allocation falls back to any node in the system
+ * is false, the allocation falls back to any node in the system.
  *
  * For systems with memory mirroring, the allocation is attempted first
  * from the regions with mirroring enabled and then retried from any
  */
 static phys_addr_t __init memblock_alloc_range_nid(phys_addr_t size,
 					phys_addr_t align, phys_addr_t start,
-					phys_addr_t end, int nid)
+					phys_addr_t end, int nid,
+					bool exact_nid)
 {
 	enum memblock_flags flags = choose_memblock_flags();
 	phys_addr_t found;
 	if (found && !memblock_reserve(found, size))
 		goto done;
-	if (nid != NUMA_NO_NODE) {
+	if (nid != NUMA_NO_NODE && !exact_nid) {
 		found = memblock_find_in_range_node(size, align, start,
 						    end, NUMA_NO_NODE,
 						    flags);
 					     phys_addr_t start,
 					     phys_addr_t end)
 {
-	return memblock_alloc_range_nid(size, align, start, end, NUMA_NO_NODE);
+	return memblock_alloc_range_nid(size, align, start, end, NUMA_NO_NODE,
+					false);
 }
 /**
 phys_addr_t __init memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid)
 {
 	return memblock_alloc_range_nid(size, align, 0,
-					MEMBLOCK_ALLOC_ACCESSIBLE, nid);
+					MEMBLOCK_ALLOC_ACCESSIBLE, nid, false);
 }
 /**
  * @min_addr: the lower bound of the memory region to allocate (phys address)
  * @max_addr: the upper bound of the memory region to allocate (phys address)
  * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
+ * @exact_nid: control the allocation fall back to other nodes
  *
  * Allocates memory block using memblock_alloc_range_nid() and
  * converts the returned physical address to virtual.
 static void * __init memblock_alloc_internal(
 				phys_addr_t size, phys_addr_t align,
 				phys_addr_t min_addr, phys_addr_t max_addr,
-				int nid)
+				int nid, bool exact_nid)
 {
 	phys_addr_t alloc;
 	if (max_addr > memblock.current_limit)
 		max_addr = memblock.current_limit;
-	alloc = memblock_alloc_range_nid(size, align, min_addr, max_addr, nid);
+	alloc = memblock_alloc_range_nid(size, align, min_addr, max_addr, nid,
+					exact_nid);
 	/* retry allocation without lower limit */
 	if (!alloc && min_addr)
-		alloc = memblock_alloc_range_nid(size, align, 0, max_addr, nid);
+		alloc = memblock_alloc_range_nid(size, align, 0, max_addr, nid,
+						exact_nid);
 	if (!alloc)
 		return NULL;
 	return phys_to_virt(alloc);
 }
+/**
+ * memblock_alloc_exact_nid_raw - allocate boot memory block on the exact node
+ * without zeroing memory
+ * @size: size of memory block to be allocated in bytes
+ * @align: alignment of the region and block's size
+ * @min_addr: the lower bound of the memory region from where the allocation
+ *	  is preferred (phys address)
+ * @max_addr: the upper bound of the memory region from where the allocation
+ *	      is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
+ *	      allocate only from memory limited by memblock.current_limit value
+ * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
+ *
+ * Public function, provides additional debug information (including caller
+ * info), if enabled. Does not zero allocated memory.
+ *
+ * Return:
+ * Virtual address of allocated memory block on success, NULL on failure.
+ */
+void * __init memblock_alloc_exact_nid_raw(
+			phys_addr_t size, phys_addr_t align,
+			phys_addr_t min_addr, phys_addr_t max_addr,
+			int nid)
+{
+	void *ptr;
+	memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
+		     __func__, (u64)size, (u64)align, nid, &min_addr,
+		     &max_addr, (void *)_RET_IP_);
+	ptr = memblock_alloc_internal(size, align,
+					   min_addr, max_addr, nid, true);
+	if (ptr && size > 0)
+		page_init_poison(ptr, size);
+	return ptr;
+}
 /**
  * memblock_alloc_try_nid_raw - allocate boot memory block without zeroing
  * memory and without panicking
 		     &max_addr, (void *)_RET_IP_);
 	ptr = memblock_alloc_internal(size, align,
-					   min_addr, max_addr, nid);
+					   min_addr, max_addr, nid, false);
 	if (ptr && size > 0)
 		page_init_poison(ptr, size);
 		     __func__, (u64)size, (u64)align, nid, &min_addr,
 		     &max_addr, (void *)_RET_IP_);
 	ptr = memblock_alloc_internal(size, align,
-					   min_addr, max_addr, nid);
+					   min_addr, max_addr, nid, false);
 	if (ptr)
 		memset(ptr, 0, size);

mm/memcontrol.c+33−134

@@ -108,7 +108,6 @@ static const char *const mem_cgroup_lru_names[] = { · 107 unmodified lines
108		108
109	#define THRESHOLDS_EVENTS_TARGET 128	109	#define THRESHOLDS_EVENTS_TARGET 128
110	#define SOFTLIMIT_EVENTS_TARGET 1024	110	#define SOFTLIMIT_EVENTS_TARGET 1024
111	#define NUMAINFO_EVENTS_TARGET 1024
112		111
113	/*	112	/*
114	* Cgroups above their limits are maintained in a RB-Tree, independent of	113	* Cgroups above their limits are maintained in a RB-Tree, independent of
@@ -778,7 +777,7 @@ void __mod_lruvec_slab_state(void *p, enum node_stat_item idx, int val) · 663 unmodified lines
778	if (!memcg \|\| memcg == root_mem_cgroup) {	777	if (!memcg \|\| memcg == root_mem_cgroup) {
779	__mod_node_page_state(pgdat, idx, val);	778	__mod_node_page_state(pgdat, idx, val);
780	} else {	779	} else {
781	lruvec = mem_cgroup_lruvec(pgdat, memcg);	780	lruvec = mem_cgroup_lruvec(memcg, pgdat);
782	__mod_lruvec_state(lruvec, idx, val);	781	__mod_lruvec_state(lruvec, idx, val);
783	}	782	}
784	rcu_read_unlock();	783	rcu_read_unlock();
@@ -877,9 +876,6 @@ static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg, · 92 unmodified lines
877	case MEM_CGROUP_TARGET_SOFTLIMIT:	876	case MEM_CGROUP_TARGET_SOFTLIMIT:
878	next = val + SOFTLIMIT_EVENTS_TARGET;	877	next = val + SOFTLIMIT_EVENTS_TARGET;
879	break;	878	break;
880	case MEM_CGROUP_TARGET_NUMAINFO:
881	next = val + NUMAINFO_EVENTS_TARGET;
882	break;
883	default:	879	default:
884	break;	880	break;
885	}	881	}
@@ -899,21 +895,12 @@ static void memcg_check_events(struct mem_cgroup memcg, struct page page) · 13 unmodified lines
899	if (unlikely(mem_cgroup_event_ratelimit(memcg,	895	if (unlikely(mem_cgroup_event_ratelimit(memcg,
900	MEM_CGROUP_TARGET_THRESH))) {	896	MEM_CGROUP_TARGET_THRESH))) {
901	bool do_softlimit;	897	bool do_softlimit;
902	bool do_numainfo __maybe_unused;
903		898
904	do_softlimit = mem_cgroup_event_ratelimit(memcg,	899	do_softlimit = mem_cgroup_event_ratelimit(memcg,
905	MEM_CGROUP_TARGET_SOFTLIMIT);	900	MEM_CGROUP_TARGET_SOFTLIMIT);
906	#if MAX_NUMNODES > 1
907	do_numainfo = mem_cgroup_event_ratelimit(memcg,
908	MEM_CGROUP_TARGET_NUMAINFO);
909	#endif
910	mem_cgroup_threshold(memcg);	901	mem_cgroup_threshold(memcg);
911	if (unlikely(do_softlimit))	902	if (unlikely(do_softlimit))
912	mem_cgroup_update_tree(memcg, page);	903	mem_cgroup_update_tree(memcg, page);
913	#if MAX_NUMNODES > 1
914	if (unlikely(do_numainfo))
915	atomic_inc(&memcg->numainfo_events);
916	#endif
917	}	904	}
918	}	905	}
919		906
@@ -1052,7 +1039,7 @@ struct mem_cgroup mem_cgroup_iter(struct mem_cgroup root, · 132 unmodified lines
1052	struct mem_cgroup_per_node *mz;	1039	struct mem_cgroup_per_node *mz;
1053		1040
1054	mz = mem_cgroup_nodeinfo(root, reclaim->pgdat->node_id);	1041	mz = mem_cgroup_nodeinfo(root, reclaim->pgdat->node_id);
1055	iter = &mz->iter[reclaim->priority];	1042	iter = &mz->iter;
1056		1043
1057	if (prev && reclaim->generation != iter->generation)	1044	if (prev && reclaim->generation != iter->generation)
1058	goto out_unlock;	1045	goto out_unlock;
@@ -1152,15 +1139,11 @@ static void __invalidate_reclaim_iterators(struct mem_cgroup *from, · 93 unmodified lines
1152	struct mem_cgroup_reclaim_iter *iter;	1139	struct mem_cgroup_reclaim_iter *iter;
1153	struct mem_cgroup_per_node *mz;	1140	struct mem_cgroup_per_node *mz;
1154	int nid;	1141	int nid;
1155	int i;
1156		1142
1157	for_each_node(nid) {	1143	for_each_node(nid) {
1158	mz = mem_cgroup_nodeinfo(from, nid);	1144	mz = mem_cgroup_nodeinfo(from, nid);
1159	for (i = 0; i <= DEF_PRIORITY; i++) {	1145	iter = &mz->iter;
1160	iter = &mz->iter[i];	1146	cmpxchg(&iter->position, dead_memcg, NULL);
1161	cmpxchg(&iter->position,
1162	dead_memcg, NULL);
1163	}
1164	}	1147	}
1165	}	1148	}
1166		1149
@@ -1238,7 +1221,7 @@ struct lruvec mem_cgroup_page_lruvec(struct page page, struct pglist_data *pgd · 71 unmodified lines
1238	struct lruvec *lruvec;	1221	struct lruvec *lruvec;
1239		1222
1240	if (mem_cgroup_disabled()) {	1223	if (mem_cgroup_disabled()) {
1241	lruvec = &pgdat->lruvec;	1224	lruvec = &pgdat->__lruvec;
1242	goto out;	1225	goto out;
1243	}	1226	}
1244		1227
@@ -1595,104 +1578,6 @@ static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask, · 350 unmodified lines
1595	return ret;	1578	return ret;
1596	}	1579	}
1597		1580
1598	#if MAX_NUMNODES > 1
1599
1600	/**
1601	* test_mem_cgroup_node_reclaimable
1602	* @memcg: the target memcg
1603	* @nid: the node ID to be checked.
1604	* @noswap : specify true here if the user wants flle only information.
1605	*
1606	* This function returns whether the specified memcg contains any
1607	* reclaimable pages on a node. Returns true if there are any reclaimable
1608	* pages in the node.
1609	*/
1610	static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
1611	int nid, bool noswap)
1612	{
1613	struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg);
1614
1615	if (lruvec_page_state(lruvec, NR_INACTIVE_FILE) \|\|
1616	lruvec_page_state(lruvec, NR_ACTIVE_FILE))
1617	return true;
1618	if (noswap \|\| !total_swap_pages)
1619	return false;
1620	if (lruvec_page_state(lruvec, NR_INACTIVE_ANON) \|\|
1621	lruvec_page_state(lruvec, NR_ACTIVE_ANON))
1622	return true;
1623	return false;
1624
1625	}
1626
1627	/*
1628	* Always updating the nodemask is not very good - even if we have an empty
1629	* list or the wrong list here, we can start from some node and traverse all
1630	* nodes based on the zonelist. So update the list loosely once per 10 secs.
1631	*
1632	*/
1633	static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
1634	{
1635	int nid;
1636	/*
1637	* numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
1638	* pagein/pageout changes since the last update.
1639	*/
1640	if (!atomic_read(&memcg->numainfo_events))
1641	return;
1642	if (atomic_inc_return(&memcg->numainfo_updating) > 1)
1643	return;
1644
1645	/* make a nodemask where this memcg uses memory from */
1646	memcg->scan_nodes = node_states[N_MEMORY];
1647
1648	for_each_node_mask(nid, node_states[N_MEMORY]) {
1649
1650	if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
1651	node_clear(nid, memcg->scan_nodes);
1652	}
1653
1654	atomic_set(&memcg->numainfo_events, 0);
1655	atomic_set(&memcg->numainfo_updating, 0);
1656	}
1657
1658	/*
1659	* Selecting a node where we start reclaim from. Because what we need is just
1660	* reducing usage counter, start from anywhere is O,K. Considering
1661	* memory reclaim from current node, there are pros. and cons.
1662	*
1663	* Freeing memory from current node means freeing memory from a node which
1664	* we'll use or we've used. So, it may make LRU bad. And if several threads
1665	* hit limits, it will see a contention on a node. But freeing from remote
1666	* node means more costs for memory reclaim because of memory latency.
1667	*
1668	* Now, we use round-robin. Better algorithm is welcomed.
1669	*/
1670	int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1671	{
1672	int node;
1673
1674	mem_cgroup_may_update_nodemask(memcg);
1675	node = memcg->last_scanned_node;
1676
1677	node = next_node_in(node, memcg->scan_nodes);
1678	/*
1679	* mem_cgroup_may_update_nodemask might have seen no reclaimmable pages
1680	* last time it really checked all the LRUs due to rate limiting.
1681	* Fallback to the current node in that case for simplicity.
1682	*/
1683	if (unlikely(node == MAX_NUMNODES))
1684	node = numa_node_id();
1685
1686	memcg->last_scanned_node = node;
1687	return node;
1688	}
1689	#else
1690	int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1691	{
1692	return 0;
1693	}
1694	#endif
1695
1696	static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,	1581	static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
1697	pg_data_t *pgdat,	1582	pg_data_t *pgdat,
1698	gfp_t gfp_mask,	1583	gfp_t gfp_mask,
@@ -1705,7 +1590,6 @@ static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg, · 6 unmodified lines
1705	unsigned long nr_scanned;	1590	unsigned long nr_scanned;
1706	struct mem_cgroup_reclaim_cookie reclaim = {	1591	struct mem_cgroup_reclaim_cookie reclaim = {
1707	.pgdat = pgdat,	1592	.pgdat = pgdat,
1708	.priority = 0,
1709	};	1593	};
1710		1594
1711	excess = soft_limit_excess(root_memcg);	1595	excess = soft_limit_excess(root_memcg);
@@ -3750,7 +3634,7 @@ static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css, · 2038 unmodified lines
3750	static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,	3634	static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
3751	int nid, unsigned int lru_mask)	3635	int nid, unsigned int lru_mask)
3752	{	3636	{
3753	struct lruvec *lruvec = mem_cgroup_lruvec(NODE_DATA(nid), memcg);	3637	struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
3754	unsigned long nr = 0;	3638	unsigned long nr = 0;
3755	enum lru_list lru;	3639	enum lru_list lru;
3756		3640
@@ -5078,7 +4962,6 @@ static struct mem_cgroup *mem_cgroup_alloc(void) · 1321 unmodified lines
5078	goto fail;	4962	goto fail;
5079		4963
5080	INIT_WORK(&memcg->high_work, high_work_func);	4964	INIT_WORK(&memcg->high_work, high_work_func);
5081	memcg->last_scanned_node = MAX_NUMNODES;
5082	INIT_LIST_HEAD(&memcg->oom_notify);	4965	INIT_LIST_HEAD(&memcg->oom_notify);
5083	mutex_init(&memcg->thresholds_lock);	4966	mutex_init(&memcg->thresholds_lock);
5084	spin_lock_init(&memcg->move_lock);	4967	spin_lock_init(&memcg->move_lock);
@@ -5455,8 +5338,8 @@ static int mem_cgroup_move_account(struct page *page, · 370 unmodified lines
5455	anon = PageAnon(page);	5338	anon = PageAnon(page);
5456		5339
5457	pgdat = page_pgdat(page);	5340	pgdat = page_pgdat(page);
5458	from_vec = mem_cgroup_lruvec(pgdat, from);	5341	from_vec = mem_cgroup_lruvec(from, pgdat);
5459	to_vec = mem_cgroup_lruvec(pgdat, to);	5342	to_vec = mem_cgroup_lruvec(to, pgdat);
5460		5343
5461	spin_lock_irqsave(&from->move_lock, flags);	5344	spin_lock_irqsave(&from->move_lock, flags);
5462		5345
@@ -6096,7 +5979,8 @@ static ssize_t memory_high_write(struct kernfs_open_file *of, · 633 unmodified lines
6096	char *buf, size_t nbytes, loff_t off)	5979	char *buf, size_t nbytes, loff_t off)
6097	{	5980	{
6098	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));	5981	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6099	unsigned long nr_pages;	5982	unsigned int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
		5983	bool drained = false;
6100	unsigned long high;	5984	unsigned long high;
6101	int err;	5985	int err;
6102		5986
@@ -6107,12 +5991,29 @@ static ssize_t memory_high_write(struct kernfs_open_file *of, · 4 unmodified lines
6107		5991
6108	memcg->high = high;	5992	memcg->high = high;
6109		5993
6110	nr_pages = page_counter_read(&memcg->memory);	5994	for (;;) {
6111	if (nr_pages > high)	5995	unsigned long nr_pages = page_counter_read(&memcg->memory);
6112	try_to_free_mem_cgroup_pages(memcg, nr_pages - high,	5996	unsigned long reclaimed;
6113	GFP_KERNEL, true);	5997
		5998	if (nr_pages <= high)
		5999	break;
		6000
		6001	if (signal_pending(current))
		6002	break;
		6003
		6004	if (!drained) {
		6005	drain_all_stock(memcg);
		6006	drained = true;
		6007	continue;
		6008	}
		6009
		6010	reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
		6011	GFP_KERNEL, true);
		6012
		6013	if (!reclaimed && !nr_retries--)
		6014	break;
		6015	}
6114		6016
6115	memcg_wb_domain_size_changed(memcg);
6116	return nbytes;	6017	return nbytes;
6117	}	6018	}
6118		6019
@@ -6144,10 +6045,8 @@ static ssize_t memory_max_write(struct kernfs_open_file *of, · 25 unmodified lines
6144	if (nr_pages <= max)	6045	if (nr_pages <= max)
6145	break;	6046	break;
6146		6047
6147	if (signal_pending(current)) {	6048	if (signal_pending(current))
6148	err = -EINTR;
6149	break;	6049	break;
6150	}
6151		6050
6152	if (!drained) {	6051	if (!drained) {
6153	drain_all_stock(memcg);	6052	drain_all_stock(memcg);

mm/memory-failure.c+23−38

 /*
  * Schedule a process for later kill.
  * Uses GFP_ATOMIC allocations to avoid potential recursions in the VM.
- * TBD would GFP_NOIO be enough?
  */
 static void add_to_kill(struct task_struct *tsk, struct page *p,
 		       struct vm_area_struct *vma,
-		       struct list_head *to_kill,
+		       struct list_head *to_kill)
-		       struct to_kill **tkc)
 {
 	struct to_kill *tk;
-	if (*tkc) {
+	tk = kmalloc(sizeof(struct to_kill), GFP_ATOMIC);
-		tk = *tkc;
+	if (!tk) {
-		*tkc = NULL;
+		pr_err("Memory failure: Out of memory while machine check handling\n");
-	} else {
+		return;
-		tk = kmalloc(sizeof(struct to_kill), GFP_ATOMIC);
-		if (!tk) {
-			pr_err("Memory failure: Out of memory while machine check handling\n");
-			return;
-		}
 	}
 	tk->addr = page_address_in_vma(p, vma);
 	if (is_zone_device_page(p))
 		tk->size_shift = dev_pagemap_mapping_shift(p, vma);
 	else
-		tk->size_shift = compound_order(compound_head(p)) + PAGE_SHIFT;
+		tk->size_shift = page_shift(compound_head(p));
 	/*
 	 * Send SIGKILL if "tk->addr == -EFAULT". Also, as
 		kfree(tk);
 		return;
 	}
 	get_task_struct(tsk);
 	tk->tsk = tsk;
 	list_add_tail(&tk->nd, to_kill);
  * Collect processes when the error hit an anonymous page.
  */
 static void collect_procs_anon(struct page *page, struct list_head *to_kill,
-			      struct to_kill **tkc, int force_early)
+				int force_early)
 {
 	struct vm_area_struct *vma;
 	struct task_struct *tsk;
 			if (!page_mapped_in_vma(page, vma))
 				continue;
 			if (vma->vm_mm == t->mm)
-				add_to_kill(t, page, vma, to_kill, tkc);
+				add_to_kill(t, page, vma, to_kill);
 		}
 	}
 	read_unlock(&tasklist_lock);
  * Collect processes when the error hit a file mapped page.
  */
 static void collect_procs_file(struct page *page, struct list_head *to_kill,
-			      struct to_kill **tkc, int force_early)
+				int force_early)
 {
 	struct vm_area_struct *vma;
 	struct task_struct *tsk;
 			 * to be informed of all such data corruptions.
 			 */
 			if (vma->vm_mm == t->mm)
-				add_to_kill(t, page, vma, to_kill, tkc);
+				add_to_kill(t, page, vma, to_kill);
 		}
 	}
 	read_unlock(&tasklist_lock);
 /*
  * Collect the processes who have the corrupted page mapped to kill.
- * This is done in two steps for locking reasons.
- * First preallocate one tokill structure outside the spin locks,
- * so that we can kill at least one process reasonably reliable.
  */
 static void collect_procs(struct page *page, struct list_head *tokill,
 				int force_early)
 {
-	struct to_kill *tk;
 	if (!page->mapping)
 		return;
-	tk = kmalloc(sizeof(struct to_kill), GFP_NOIO);
-	if (!tk)
-		return;
 	if (PageAnon(page))
-		collect_procs_anon(page, tokill, &tk, force_early);
+		collect_procs_anon(page, tokill, force_early);
 	else
-		collect_procs_file(page, tokill, &tk, force_early);
+		collect_procs_file(page, tokill, force_early);
-	kfree(tk);
 }
 static const char *action_name[] = {
 		if (!gotten)
 			break;
 		if (entry.flags & MF_SOFT_OFFLINE)
-			soft_offline_page(pfn_to_page(entry.pfn), entry.flags);
+			soft_offline_page(entry.pfn, entry.flags);
 		else
 			memory_failure(entry.pfn, entry.flags);
 	}
 /**
  * soft_offline_page - Soft offline a page.
- * @page: page to offline
+ * @pfn: pfn to soft-offline
  * @flags: flags. Same as memory_failure().
  *
  * Returns 0 on success, otherwise negated errno.
  * This is not a 100% solution for all memory, but tries to be
  * ``good enough'' for the majority of memory.
  */
-int soft_offline_page(struct page *page, int flags)
+int soft_offline_page(unsigned long pfn, int flags)
 {
 	int ret;
-	unsigned long pfn = page_to_pfn(page);
+	struct page *page;
-	if (is_zone_device_page(page)) {
+	if (!pfn_valid(pfn))
-		pr_debug_ratelimited("soft_offline: %#lx page is device page\n",
+		return -ENXIO;
-				pfn);
+	/* Only online pages can be soft-offlined (esp., not ZONE_DEVICE). */
-		if (flags & MF_COUNT_INCREASED)
+	page = pfn_to_online_page(pfn);
-			put_page(page);
+	if (!page)
 		return -EIO;
-	}
 	if (PageHWPoison(page)) {
 		pr_info("soft offline: %#lx page already poisoned\n", pfn);

mm/memory.c+40−12

@@ -72,6 +72,8 @@ · 71 unmodified lines
72	#include <linux/oom.h>	72	#include <linux/oom.h>
73	#include <linux/numa.h>	73	#include <linux/numa.h>
74		74
		75	#include <trace/events/kmem.h>
		76
75	#include <asm/io.h>	77	#include <asm/io.h>
76	#include <asm/mmu_context.h>	78	#include <asm/mmu_context.h>
77	#include <asm/pgalloc.h>	79	#include <asm/pgalloc.h>
@@ -152,6 +154,10 @@ static int __init init_zero_pfn(void) · 74 unmodified lines
152	}	154	}
153	core_initcall(init_zero_pfn);	155	core_initcall(init_zero_pfn);
154		156
		157	void mm_trace_rss_stat(struct mm_struct *mm, int member, long count)
		158	{
		159	trace_rss_stat(mm, member, count);
		160	}
155		161
156	#if defined(SPLIT_RSS_COUNTING)	162	#if defined(SPLIT_RSS_COUNTING)
157		163
@@ -2289,10 +2295,11 @@ static vm_fault_t do_page_mkwrite(struct vm_fault *vmf) · 2131 unmodified lines
2289	*	2295	*
2290	* The function expects the page to be locked and unlocks it.	2296	* The function expects the page to be locked and unlocks it.
2291	*/	2297	*/
2292	static void fault_dirty_shared_page(struct vm_area_struct *vma,	2298	static vm_fault_t fault_dirty_shared_page(struct vm_fault *vmf)
2293	struct page *page)
2294	{	2299	{
		2300	struct vm_area_struct *vma = vmf->vma;
2295	struct address_space *mapping;	2301	struct address_space *mapping;
		2302	struct page *page = vmf->page;
2296	bool dirtied;	2303	bool dirtied;
2297	bool page_mkwrite = vma->vm_ops && vma->vm_ops->page_mkwrite;	2304	bool page_mkwrite = vma->vm_ops && vma->vm_ops->page_mkwrite;
2298		2305
@@ -2307,16 +2314,30 @@ static void fault_dirty_shared_page(struct vm_area_struct *vma, · 8 unmodified lines
2307	mapping = page_rmapping(page);	2314	mapping = page_rmapping(page);
2308	unlock_page(page);	2315	unlock_page(page);
2309		2316
		2317	if (!page_mkwrite)
		2318	file_update_time(vma->vm_file);
		2319
		2320	/*
		2321	* Throttle page dirtying rate down to writeback speed.
		2322	*
		2323	* mapping may be NULL here because some device drivers do not
		2324	* set page.mapping but still dirty their pages
		2325	*
		2326	* Drop the mmap_sem before waiting on IO, if we can. The file
		2327	* is pinning the mapping, as per above.
		2328	*/
2310	if ((dirtied \|\| page_mkwrite) && mapping) {	2329	if ((dirtied \|\| page_mkwrite) && mapping) {
2311	/*	2330	struct file *fpin;
2312	* Some device drivers do not set page.mapping	2331
2313	* but still dirty their pages	2332	fpin = maybe_unlock_mmap_for_io(vmf, NULL);
2314	*/
2315	balance_dirty_pages_ratelimited(mapping);	2333	balance_dirty_pages_ratelimited(mapping);
		2334	if (fpin) {
		2335	fput(fpin);
		2336	return VM_FAULT_RETRY;
		2337	}
2316	}	2338	}
2317		2339
2318	if (!page_mkwrite)	2340	return 0;
2319	file_update_time(vma->vm_file);
2320	}	2341	}
2321		2342
2322	/*	2343	/*
@@ -2571,6 +2592,7 @@ static vm_fault_t wp_page_shared(struct vm_fault *vmf) · 248 unmodified lines
2571	__releases(vmf->ptl)	2592	__releases(vmf->ptl)
2572	{	2593	{
2573	struct vm_area_struct *vma = vmf->vma;	2594	struct vm_area_struct *vma = vmf->vma;
		2595	vm_fault_t ret = VM_FAULT_WRITE;
2574		2596
2575	get_page(vmf->page);	2597	get_page(vmf->page);
2576		2598
@@ -2594,10 +2616,10 @@ static vm_fault_t wp_page_shared(struct vm_fault *vmf) · 17 unmodified lines
2594	wp_page_reuse(vmf);	2616	wp_page_reuse(vmf);
2595	lock_page(vmf->page);	2617	lock_page(vmf->page);
2596	}	2618	}
2597	fault_dirty_shared_page(vma, vmf->page);	2619	ret \|= fault_dirty_shared_page(vmf);
2598	put_page(vmf->page);	2620	put_page(vmf->page);
2599		2621
2600	return VM_FAULT_WRITE;	2622	return ret;
2601	}	2623	}
2602		2624
2603	/*	2625	/*
@@ -3083,7 +3105,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) · 479 unmodified lines
3083		3105
3084	/*	3106	/*
3085	* The memory barrier inside __SetPageUptodate makes sure that	3107	* The memory barrier inside __SetPageUptodate makes sure that
3086	* preceeding stores to the page contents become visible before	3108	* preceding stores to the page contents become visible before
3087	* the set_pte_at() write.	3109	* the set_pte_at() write.
3088	*/	3110	*/
3089	__SetPageUptodate(page);	3111	__SetPageUptodate(page);
@@ -3641,7 +3663,7 @@ static vm_fault_t do_shared_fault(struct vm_fault *vmf) · 551 unmodified lines
3641	return ret;	3663	return ret;
3642	}	3664	}
3643		3665
3644	fault_dirty_shared_page(vma, vmf->page);	3666	ret \|= fault_dirty_shared_page(vmf);
3645	return ret;	3667	return ret;
3646	}	3668	}
3647		3669
@@ -3988,6 +4010,7 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, · 340 unmodified lines
3988	vmf.pud = pud_alloc(mm, p4d, address);	4010	vmf.pud = pud_alloc(mm, p4d, address);
3989	if (!vmf.pud)	4011	if (!vmf.pud)
3990	return VM_FAULT_OOM;	4012	return VM_FAULT_OOM;
		4013	retry_pud:
3991	if (pud_none(*vmf.pud) && __transparent_hugepage_enabled(vma)) {	4014	if (pud_none(*vmf.pud) && __transparent_hugepage_enabled(vma)) {
3992	ret = create_huge_pud(&vmf);	4015	ret = create_huge_pud(&vmf);
3993	if (!(ret & VM_FAULT_FALLBACK))	4016	if (!(ret & VM_FAULT_FALLBACK))
@@ -4014,6 +4037,11 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, · 20 unmodified lines
4014	vmf.pmd = pmd_alloc(mm, vmf.pud, address);	4037	vmf.pmd = pmd_alloc(mm, vmf.pud, address);
4015	if (!vmf.pmd)	4038	if (!vmf.pmd)
4016	return VM_FAULT_OOM;	4039	return VM_FAULT_OOM;
		4040
		4041	/* Huge pud page fault raced with pmd_alloc? */
		4042	if (pud_trans_unstable(vmf.pud))
		4043	goto retry_pud;
		4044
4017	if (pmd_none(*vmf.pmd) && __transparent_hugepage_enabled(vma)) {	4045	if (pmd_none(*vmf.pmd) && __transparent_hugepage_enabled(vma)) {
4018	ret = create_huge_pmd(&vmf);	4046	ret = create_huge_pmd(&vmf);
4019	if (!(ret & VM_FAULT_FALLBACK))	4047	if (!(ret & VM_FAULT_FALLBACK))

mm/memory_hotplug.c+56−30

  * and restore_online_page_callback() for generic callback restore.
  */
-static void generic_online_page(struct page *page, unsigned int order);
 static online_page_callback_t online_page_callback = generic_online_page;
 static DEFINE_MUTEX(online_page_callback_lock);
 	return 0;
 }
+static int check_hotplug_memory_addressable(unsigned long pfn,
+					    unsigned long nr_pages)
+{
+	const u64 max_addr = PFN_PHYS(pfn + nr_pages) - 1;
+	if (max_addr >> MAX_PHYSMEM_BITS) {
+		const u64 max_allowed = (1ull << (MAX_PHYSMEM_BITS + 1)) - 1;
+		WARN(1,
+		     "Hotplugged memory exceeds maximum addressable address, range=%#llx-%#llx, maximum=%#llx\n",
+		     (u64)PFN_PHYS(pfn), max_addr, max_allowed);
+		return -E2BIG;
+	}
+	return 0;
+}
 /*
  * Reasonably generic function for adding memory.  It is
  * expected that archs that support memory hotplug will
 	unsigned long nr, start_sec, end_sec;
 	struct vmem_altmap *altmap = restrictions->altmap;
+	err = check_hotplug_memory_addressable(pfn, nr_pages);
+	if (err)
+		return err;
 	if (altmap) {
 		/*
 		 * Validate altmap is within bounds of the total request
 }
 EXPORT_SYMBOL_GPL(restore_online_page_callback);
-void __online_page_set_limits(struct page *page)
+void generic_online_page(struct page *page, unsigned int order)
-{
-}
-EXPORT_SYMBOL_GPL(__online_page_set_limits);
-void __online_page_increment_counters(struct page *page)
-{
-	adjust_managed_page_count(page, 1);
-}
-EXPORT_SYMBOL_GPL(__online_page_increment_counters);
-void __online_page_free(struct page *page)
-{
-	__free_reserved_page(page);
-}
-EXPORT_SYMBOL_GPL(__online_page_free);
-static void generic_online_page(struct page *page, unsigned int order)
 {
 	kernel_map_pages(page, 1 << order, 1);
 	__free_pages_core(page, order);
 		totalhigh_pages_add(1UL << order);
 #endif
 }
+EXPORT_SYMBOL_GPL(generic_online_page);
 static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages,
 			void *arg)
 	if (!zone_spans_pfn(zone, pfn))
 		return false;
-	return !has_unmovable_pages(zone, page, 0, MIGRATE_MOVABLE, SKIP_HWPOISON);
+	return !has_unmovable_pages(zone, page, 0, MIGRATE_MOVABLE,
+				    MEMORY_OFFLINE);
 }
 /* Checks if this range of memory is likely to be hot-removable. */
 	return ret;
 }
-/*
+/* Mark all sections offline and remove all free pages from the buddy. */
- * remove from free_area[] and mark all as Reserved.
- */
 static int
 offline_isolated_pages_cb(unsigned long start, unsigned long nr_pages,
 			void *data)
 check_pages_isolated_cb(unsigned long start_pfn, unsigned long nr_pages,
 			void *data)
 {
-	return test_pages_isolated(start_pfn, start_pfn + nr_pages, true);
+	return test_pages_isolated(start_pfn, start_pfn + nr_pages,
+				   MEMORY_OFFLINE);
 }
 static int __init cmdline_parse_movable_node(char *p)
 		node_clear_state(node, N_MEMORY);
 }
+static int count_system_ram_pages_cb(unsigned long start_pfn,
+				     unsigned long nr_pages, void *data)
+{
+	unsigned long *nr_system_ram_pages = data;
+	*nr_system_ram_pages += nr_pages;
+	return 0;
+}
 static int __ref __offline_pages(unsigned long start_pfn,
 		  unsigned long end_pfn)
 {
-	unsigned long pfn, nr_pages;
+	unsigned long pfn, nr_pages = 0;
 	unsigned long offlined_pages = 0;
 	int ret, node, nr_isolate_pageblock;
 	unsigned long flags;
 	mem_hotplug_begin();
+	/*
+	 * Don't allow to offline memory blocks that contain holes.
+	 * Consequently, memory blocks with holes can never get onlined
+	 * via the hotplug path - online_pages() - as hotplugged memory has
+	 * no holes. This way, we e.g., don't have to worry about marking
+	 * memory holes PG_reserved, don't need pfn_valid() checks, and can
+	 * avoid using walk_system_ram_range() later.
+	 */
+	walk_system_ram_range(start_pfn, end_pfn - start_pfn, &nr_pages,
+			      count_system_ram_pages_cb);
+	if (nr_pages != end_pfn - start_pfn) {
+		ret = -EINVAL;
+		reason = "memory holes";
+		goto failed_removal;
+	}
 	/* This makes hotplug much easier...and readable.
 	   we assume this for now. .*/
 	if (!test_pages_in_a_zone(start_pfn, end_pfn, &valid_start,
 	zone = page_zone(pfn_to_page(valid_start));
 	node = zone_to_nid(zone);
-	nr_pages = end_pfn - start_pfn;
 	/* set above range as isolated */
 	ret = start_isolate_page_range(start_pfn, end_pfn,
 				       MIGRATE_MOVABLE,
-				       SKIP_HWPOISON | REPORT_FAILURE);
+				       MEMORY_OFFLINE | REPORT_FAILURE);
 	if (ret < 0) {
 		reason = "failure to isolate range";
 		goto failed_removal;
 	/* remove memmap entry */
 	firmware_map_remove(start, start + size, "System RAM");
-	memblock_free(start, size);
-	memblock_remove(start, size);
 	/* remove memory block devices before removing memory */
 	remove_memory_block_devices(start, size);
 	arch_remove_memory(nid, start, size, NULL);
+	memblock_free(start, size);
+	memblock_remove(start, size);
 	__release_memory_resource(start, size);
 	try_offline_node(nid);

mm/mempolicy.c+31−16

 	struct list_head *pagelist;
 	unsigned long flags;
 	nodemask_t *nmask;
-	struct vm_area_struct *prev;
+	unsigned long start;
+	unsigned long end;
+	struct vm_area_struct *first;
 };
 /*
 	unsigned long endvma = vma->vm_end;
 	unsigned long flags = qp->flags;
+	/* range check first */
+	VM_BUG_ON((vma->vm_start > start) || (vma->vm_end < end));
+	if (!qp->first) {
+		qp->first = vma;
+		if (!(flags & MPOL_MF_DISCONTIG_OK) &&
+			(qp->start < vma->vm_start))
+			/* hole at head side of range */
+			return -EFAULT;
+	}
+	if (!(flags & MPOL_MF_DISCONTIG_OK) &&
+		((vma->vm_end < qp->end) &&
+		(!vma->vm_next || vma->vm_end < vma->vm_next->vm_start)))
+		/* hole at middle or tail of range */
+		return -EFAULT;
 	/*
 	 * Need check MPOL_MF_STRICT to return -EIO if possible
 	 * regardless of vma_migratable
 	if (endvma > end)
 		endvma = end;
-	if (vma->vm_start > start)
-		start = vma->vm_start;
-	if (!(flags & MPOL_MF_DISCONTIG_OK)) {
-		if (!vma->vm_next && vma->vm_end < end)
-			return -EFAULT;
-		if (qp->prev && qp->prev->vm_end < vma->vm_start)
-			return -EFAULT;
-	}
-	qp->prev = vma;
 	if (flags & MPOL_MF_LAZY) {
 		/* Similar to task_numa_work, skip inaccessible VMAs */
 		nodemask_t *nodes, unsigned long flags,
 		struct list_head *pagelist)
 {
+	int err;
 	struct queue_pages qp = {
 		.pagelist = pagelist,
 		.flags = flags,
 		.nmask = nodes,
-		.prev = NULL,
+		.start = start,
+		.end = end,
+		.first = NULL,
 	};
-	return walk_page_range(mm, start, end, &queue_pages_walk_ops, &qp);
+	err = walk_page_range(mm, start, end, &queue_pages_walk_ops, &qp);
+	if (!qp.first)
+		/* whole range in hole */
+		err = -EFAULT;
+	return err;
 }
 /*
 	unsigned long vmend;
 	vma = find_vma(mm, start);
-	if (!vma || vma->vm_start > start)
+	VM_BUG_ON(!vma);
-		return -EFAULT;
 	prev = vma->vm_prev;
 	if (start > vma->vm_start)

mm/migrate.c+6−10

 				   enum migrate_reason reason)
 {
 	int rc = MIGRATEPAGE_SUCCESS;
-	struct page *newpage;
+	struct page *newpage = NULL;
 	if (!thp_migration_supported() && PageTransHuge(page))
 		return -ENOMEM;
-	newpage = get_new_page(page, private);
-	if (!newpage)
-		return -ENOMEM;
 	if (page_count(page) == 1) {
 		/* page was freed from under us. So we are done. */
 		ClearPageActive(page);
 				__ClearPageIsolated(page);
 			unlock_page(page);
 		}
-		if (put_new_page)
-			put_new_page(newpage, private);
-		else
-			put_page(newpage);
 		goto out;
 	}
+	newpage = get_new_page(page, private);
+	if (!newpage)
+		return -ENOMEM;
 	rc = __unmap_and_move(page, newpage, force, mode);
 	if (rc == MIGRATEPAGE_SUCCESS)
 		set_page_owner_migrate_reason(newpage, reason);
 		if (!zone_watermark_ok(zone, 0,
 				       high_wmark_pages(zone) +
 				       nr_migrate_pages,
-				       0, 0))
+				       ZONE_MOVABLE, 0))
 			continue;
 		return true;
 	}

mm/mmap.c+21−42

 	struct vm_area_struct *prev, struct rb_node **rb_link,
 	struct rb_node *rb_parent)
 {
-	__vma_link_list(mm, vma, prev, rb_parent);
+	__vma_link_list(mm, vma, prev);
 	__vma_link_rb(mm, vma, rb_link, rb_parent);
 }
 static __always_inline void __vma_unlink_common(struct mm_struct *mm,
 						struct vm_area_struct *vma,
-						struct vm_area_struct *prev,
-						bool has_prev,
 						struct vm_area_struct *ignore)
 {
-	struct vm_area_struct *next;
 	vma_rb_erase_ignore(vma, &mm->mm_rb, ignore);
-	next = vma->vm_next;
+	__vma_unlink_list(mm, vma);
-	if (has_prev)
-		prev->vm_next = next;
-	else {
-		prev = vma->vm_prev;
-		if (prev)
-			prev->vm_next = next;
-		else
-			mm->mmap = next;
-	}
-	if (next)
-		next->vm_prev = prev;
 	/* Kill the cache */
 	vmacache_invalidate(mm);
 }
-static inline void __vma_unlink_prev(struct mm_struct *mm,
-				     struct vm_area_struct *vma,
-				     struct vm_area_struct *prev)
-{
-	__vma_unlink_common(mm, vma, prev, true, vma);
-}
 /*
  * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that
  * is already present in an i_mmap tree without adjusting the tree.
 				remove_next = 1 + (end > next->vm_end);
 				VM_WARN_ON(remove_next == 2 &&
 					   end != next->vm_next->vm_end);
-				VM_WARN_ON(remove_next == 1 &&
-					   end != next->vm_end);
 				/* trim end to next, for case 6 first pass */
 				end = next->vm_end;
 			}
 		 * us to remove next before dropping the locks.
 		 */
 		if (remove_next != 3)
-			__vma_unlink_prev(mm, next, vma);
+			__vma_unlink_common(mm, next, next);
 		else
 			/*
 			 * vma is not before next if they've been
 			 * "next" (which is stored in post-swap()
 			 * "vma").
 			 */
-			__vma_unlink_common(mm, next, NULL, false, vma);
+			__vma_unlink_common(mm, next, vma);
 		if (file)
 			__remove_shared_vm_struct(next, file, mapping);
 	} else if (insert) {
  * the area passed down from mprotect_fixup, never extending beyond one
  * vma, PPPPPP is the prev vma specified, and NNNNNN the next vma after:
  *
- *     AAAA             AAAA                AAAA          AAAA
+ *     AAAA             AAAA                   AAAA
- *    PPPPPPNNNNNN    PPPPPPNNNNNN    PPPPPPNNNNNN    PPPPNNNNXXXX
+ *    PPPPPPNNNNNN    PPPPPPNNNNNN       PPPPPPNNNNNN
- *    cannot merge    might become    might become    might become
+ *    cannot merge    might become       might become
- *                    PPNNNNNNNNNN    PPPPPPPPPPNN    PPPPPPPPPPPP 6 or
+ *                    PPNNNNNNNNNN       PPPPPPPPPPNN
- *    mmap, brk or    case 4 below    case 5 below    PPPPPPPPXXXX 7 or
+ *    mmap, brk or    case 4 below       case 5 below
- *    mremap move:                                    PPPPXXXXXXXX 8
+ *    mremap move:
- *        AAAA
+ *                        AAAA               AAAA
- *    PPPP    NNNN    PPPPPPPPPPPP    PPPPPPPPNNNN    PPPPNNNNNNNN
+ *                    PPPP    NNNN       PPPPNNNNXXXX
- *    might become    case 1 below    case 2 below    case 3 below
+ *                    might become       might become
+ *                    PPPPPPPPPPPP 1 or  PPPPPPPPPPPP 6 or
+ *                    PPPPPPPPNNNN 2 or  PPPPPPPPXXXX 7 or
+ *                    PPPPNNNNNNNN 3     PPPPXXXXXXXX 8
  *
  * It is important for case 8 that the vma NNNN overlapping the
  * region AAAA is never going to extended over XXXX. Instead XXXX must
 	 * that it represents a valid section of the address space.
 	 */
 	addr = get_unmapped_area(file, addr, len, pgoff, flags);
-	if (offset_in_page(addr))
+	if (IS_ERR_VALUE(addr))
 		return addr;
 	if (flags & MAP_FIXED_NOREPLACE) {
 	struct rb_node **rb_link, *rb_parent;
 	pgoff_t pgoff = addr >> PAGE_SHIFT;
 	int error;
+	unsigned long mapped_addr;
 	/* Until we need other flags, refuse anything except VM_EXEC. */
 	if ((flags & (~VM_EXEC)) != 0)
 		return -EINVAL;
 	flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
-	error = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
+	mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
-	if (offset_in_page(error))
+	if (IS_ERR_VALUE(mapped_addr))
-		return error;
+		return mapped_addr;
 	error = mlock_future_check(mm, mm->def_flags, len);
 	if (error)

mm/mprotect.c+4−4

 			if (prot_numa) {
 				struct page *page;
+				/* Avoid TLB flush if possible */
+				if (pte_protnone(oldpte))
+					continue;
 				page = vm_normal_page(vma, addr, oldpte);
 				if (!page || PageKsm(page))
 					continue;
 				if (page_is_file_cache(page) && PageDirty(page))
 					continue;
-				/* Avoid TLB flush if possible */
-				if (pte_protnone(oldpte))
-					continue;
 				/*
 				 * Don't mess with PTEs if page is already on the node
 				 * a single-threaded process is running on.

mm/mremap.c+2−2

@@ -558,7 +558,7 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len, · 557 unmodified lines
558	ret = get_unmapped_area(vma->vm_file, new_addr, new_len, vma->vm_pgoff +	558	ret = get_unmapped_area(vma->vm_file, new_addr, new_len, vma->vm_pgoff +
559	((addr - vma->vm_start) >> PAGE_SHIFT),	559	((addr - vma->vm_start) >> PAGE_SHIFT),
560	map_flags);	560	map_flags);
561	if (offset_in_page(ret))	561	if (IS_ERR_VALUE(ret))
562	goto out1;	562	goto out1;
563		563
564	ret = move_vma(vma, addr, old_len, new_len, new_addr, locked, uf,	564	ret = move_vma(vma, addr, old_len, new_len, new_addr, locked, uf,
@@ -706,7 +706,7 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, · 141 unmodified lines
706	vma->vm_pgoff +	706	vma->vm_pgoff +
707	((addr - vma->vm_start) >> PAGE_SHIFT),	707	((addr - vma->vm_start) >> PAGE_SHIFT),
708	map_flags);	708	map_flags);
709	if (offset_in_page(new_addr)) {	709	if (IS_ERR_VALUE(new_addr)) {
710	ret = new_addr;	710	ret = new_addr;
711	goto out;	711	goto out;
712	}	712	}

mm/nommu.c+2−8

 	if (rb_prev)
 		prev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
-	__vma_link_list(mm, vma, prev, parent);
+	__vma_link_list(mm, vma, prev);
 }
 /*
 	/* remove from the MM's tree and list */
 	rb_erase(&vma->vm_rb, &mm->mm_rb);
-	if (vma->vm_prev)
+	__vma_unlink_list(mm, vma);
-		vma->vm_prev->vm_next = vma->vm_next;
-	else
-		mm->mmap = vma->vm_next;
-	if (vma->vm_next)
-		vma->vm_next->vm_prev = vma->vm_prev;
 }
 /*

mm/page_alloc.c+119−18

@@ -5354,6 +5354,7 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) · 5353 unmodified lines
5354	" min:%lukB"	5354	" min:%lukB"
5355	" low:%lukB"	5355	" low:%lukB"
5356	" high:%lukB"	5356	" high:%lukB"
		5357	" reserved_highatomic:%luKB"
5357	" active_anon:%lukB"	5358	" active_anon:%lukB"
5358	" inactive_anon:%lukB"	5359	" inactive_anon:%lukB"
5359	" active_file:%lukB"	5360	" active_file:%lukB"
@@ -5375,6 +5376,7 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) · 15 unmodified lines
5375	K(min_wmark_pages(zone)),	5376	K(min_wmark_pages(zone)),
5376	K(low_wmark_pages(zone)),	5377	K(low_wmark_pages(zone)),
5377	K(high_wmark_pages(zone)),	5378	K(high_wmark_pages(zone)),
		5379	K(zone->nr_reserved_highatomic),
5378	K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)),	5380	K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)),
5379	K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)),	5381	K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)),
5380	K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)),	5382	K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)),
@@ -6711,7 +6713,7 @@ static void __meminit pgdat_init_internals(struct pglist_data *pgdat) · 1330 unmodified lines
6711		6713
6712	pgdat_page_ext_init(pgdat);	6714	pgdat_page_ext_init(pgdat);
6713	spin_lock_init(&pgdat->lru_lock);	6715	spin_lock_init(&pgdat->lru_lock);
6714	lruvec_init(node_lruvec(pgdat));	6716	lruvec_init(&pgdat->__lruvec);
6715	}	6717	}
6716		6718
6717	static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid,	6719	static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid,
@@ -7988,6 +7990,15 @@ int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write, · 1270 unmodified lines
7988	return 0;	7990	return 0;
7989	}	7991	}
7990		7992
		7993	static void __zone_pcp_update(struct zone *zone)
		7994	{
		7995	unsigned int cpu;
		7996
		7997	for_each_possible_cpu(cpu)
		7998	pageset_set_high_and_batch(zone,
		7999	per_cpu_ptr(zone->pageset, cpu));
		8000	}
		8001
7991	/*	8002	/*
7992	* percpu_pagelist_fraction - changes the pcp->high for each zone on each	8003	* percpu_pagelist_fraction - changes the pcp->high for each zone on each
7993	* cpu. It is the fraction of total pages in each zone that a hot per cpu	8004	* cpu. It is the fraction of total pages in each zone that a hot per cpu
@@ -8019,13 +8030,8 @@ int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write, · 25 unmodified lines
8019	if (percpu_pagelist_fraction == old_percpu_pagelist_fraction)	8030	if (percpu_pagelist_fraction == old_percpu_pagelist_fraction)
8020	goto out;	8031	goto out;
8021		8032
8022	for_each_populated_zone(zone) {	8033	for_each_populated_zone(zone)
8023	unsigned int cpu;	8034	__zone_pcp_update(zone);
8024
8025	for_each_possible_cpu(cpu)
8026	pageset_set_high_and_batch(zone,
8027	per_cpu_ptr(zone->pageset, cpu));
8028	}
8029	out:	8035	out:
8030	mutex_unlock(&pcp_batch_high_lock);	8036	mutex_unlock(&pcp_batch_high_lock);
8031	return ret;	8037	return ret;
@@ -8261,7 +8267,7 @@ bool has_unmovable_pages(struct zone zone, struct page page, int count, · 229 unmodified lines
8261	* The HWPoisoned page may be not in buddy system, and	8267	* The HWPoisoned page may be not in buddy system, and
8262	* page_count() is not 0.	8268	* page_count() is not 0.
8263	*/	8269	*/
8264	if ((flags & SKIP_HWPOISON) && PageHWPoison(page))	8270	if ((flags & MEMORY_OFFLINE) && PageHWPoison(page))
8265	continue;	8271	continue;
8266		8272
8267	if (__PageMovable(page))	8273	if (__PageMovable(page))
@@ -8477,7 +8483,7 @@ int alloc_contig_range(unsigned long start, unsigned long end, · 209 unmodified lines
8477	}	8483	}
8478		8484
8479	/* Make sure the range is really isolated. */	8485	/* Make sure the range is really isolated. */
8480	if (test_pages_isolated(outer_start, end, false)) {	8486	if (test_pages_isolated(outer_start, end, 0)) {
8481	pr_info_ratelimited("%s: [%lx, %lx) PFNs busy\n",	8487	pr_info_ratelimited("%s: [%lx, %lx) PFNs busy\n",
8482	__func__, outer_start, end);	8488	__func__, outer_start, end);
8483	ret = -EBUSY;	8489	ret = -EBUSY;
@@ -8502,6 +8508,107 @@ done: · 18 unmodified lines
8502	pfn_max_align_up(end), migratetype);	8508	pfn_max_align_up(end), migratetype);
8503	return ret;	8509	return ret;
8504	}	8510	}
		8511
		8512	static int __alloc_contig_pages(unsigned long start_pfn,
		8513	unsigned long nr_pages, gfp_t gfp_mask)
		8514	{
		8515	unsigned long end_pfn = start_pfn + nr_pages;
		8516
		8517	return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE,
		8518	gfp_mask);
		8519	}
		8520
		8521	static bool pfn_range_valid_contig(struct zone *z, unsigned long start_pfn,
		8522	unsigned long nr_pages)
		8523	{
		8524	unsigned long i, end_pfn = start_pfn + nr_pages;
		8525	struct page *page;
		8526
		8527	for (i = start_pfn; i < end_pfn; i++) {
		8528	page = pfn_to_online_page(i);
		8529	if (!page)
		8530	return false;
		8531
		8532	if (page_zone(page) != z)
		8533	return false;
		8534
		8535	if (PageReserved(page))
		8536	return false;
		8537
		8538	if (page_count(page) > 0)
		8539	return false;
		8540
		8541	if (PageHuge(page))
		8542	return false;
		8543	}
		8544	return true;
		8545	}
		8546
		8547	static bool zone_spans_last_pfn(const struct zone *zone,
		8548	unsigned long start_pfn, unsigned long nr_pages)
		8549	{
		8550	unsigned long last_pfn = start_pfn + nr_pages - 1;
		8551
		8552	return zone_spans_pfn(zone, last_pfn);
		8553	}
		8554
		8555	/**
		8556	* alloc_contig_pages() -- tries to find and allocate contiguous range of pages
		8557	* @nr_pages: Number of contiguous pages to allocate
		8558	* @gfp_mask: GFP mask to limit search and used during compaction
		8559	* @nid: Target node
		8560	* @nodemask: Mask for other possible nodes
		8561	*
		8562	* This routine is a wrapper around alloc_contig_range(). It scans over zones
		8563	* on an applicable zonelist to find a contiguous pfn range which can then be
		8564	* tried for allocation with alloc_contig_range(). This routine is intended
		8565	* for allocation requests which can not be fulfilled with the buddy allocator.
		8566	*
		8567	* The allocated memory is always aligned to a page boundary. If nr_pages is a
		8568	* power of two then the alignment is guaranteed to be to the given nr_pages
		8569	* (e.g. 1GB request would be aligned to 1GB).
		8570	*
		8571	* Allocated pages can be freed with free_contig_range() or by manually calling
		8572	* __free_page() on each allocated page.
		8573	*
		8574	* Return: pointer to contiguous pages on success, or NULL if not successful.
		8575	*/
		8576	struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask,
		8577	int nid, nodemask_t *nodemask)
		8578	{
		8579	unsigned long ret, pfn, flags;
		8580	struct zonelist *zonelist;
		8581	struct zone *zone;
		8582	struct zoneref *z;
		8583
		8584	zonelist = node_zonelist(nid, gfp_mask);
		8585	for_each_zone_zonelist_nodemask(zone, z, zonelist,
		8586	gfp_zone(gfp_mask), nodemask) {
		8587	spin_lock_irqsave(&zone->lock, flags);
		8588
		8589	pfn = ALIGN(zone->zone_start_pfn, nr_pages);
		8590	while (zone_spans_last_pfn(zone, pfn, nr_pages)) {
		8591	if (pfn_range_valid_contig(zone, pfn, nr_pages)) {
		8592	/*
		8593	* We release the zone lock here because
		8594	* alloc_contig_range() will also lock the zone
		8595	* at some point. If there's an allocation
		8596	* spinning on this lock, it may win the race
		8597	* and cause alloc_contig_range() to fail...
		8598	*/
		8599	spin_unlock_irqrestore(&zone->lock, flags);
		8600	ret = __alloc_contig_pages(pfn, nr_pages,
		8601	gfp_mask);
		8602	if (!ret)
		8603	return pfn_to_page(pfn);
		8604	spin_lock_irqsave(&zone->lock, flags);
		8605	}
		8606	pfn += nr_pages;
		8607	}
		8608	spin_unlock_irqrestore(&zone->lock, flags);
		8609	}
		8610	return NULL;
		8611	}
8505	#endif /* CONFIG_CONTIG_ALLOC */	8612	#endif /* CONFIG_CONTIG_ALLOC */
8506		8613
8507	void free_contig_range(unsigned long pfn, unsigned int nr_pages)	8614	void free_contig_range(unsigned long pfn, unsigned int nr_pages)
@@ -8523,11 +8630,8 @@ void free_contig_range(unsigned long pfn, unsigned int nr_pages) · 15 unmodified lines
8523	*/	8630	*/
8524	void __meminit zone_pcp_update(struct zone *zone)	8631	void __meminit zone_pcp_update(struct zone *zone)
8525	{	8632	{
8526	unsigned cpu;
8527	mutex_lock(&pcp_batch_high_lock);	8633	mutex_lock(&pcp_batch_high_lock);
8528	for_each_possible_cpu(cpu)	8634	__zone_pcp_update(zone);
8529	pageset_set_high_and_batch(zone,
8530	per_cpu_ptr(zone->pageset, cpu));
8531	mutex_unlock(&pcp_batch_high_lock);	8635	mutex_unlock(&pcp_batch_high_lock);
8532	}	8636	}
8533		8637
@@ -8560,7 +8664,7 @@ __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) · 26 unmodified lines
8560	{	8664	{
8561	struct page *page;	8665	struct page *page;
8562	struct zone *zone;	8666	struct zone *zone;
8563	unsigned int order, i;	8667	unsigned int order;
8564	unsigned long pfn;	8668	unsigned long pfn;
8565	unsigned long flags;	8669	unsigned long flags;
8566	unsigned long offlined_pages = 0;	8670	unsigned long offlined_pages = 0;
@@ -8588,7 +8692,6 @@ __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) · 21 unmodified lines
8588	*/	8692	*/
8589	if (unlikely(!PageBuddy(page) && PageHWPoison(page))) {	8693	if (unlikely(!PageBuddy(page) && PageHWPoison(page))) {
8590	pfn++;	8694	pfn++;
8591	SetPageReserved(page);
8592	offlined_pages++;	8695	offlined_pages++;
8593	continue;	8696	continue;
8594	}	8697	}
@@ -8602,8 +8705,6 @@ __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) · 7 unmodified lines
8602	pfn, 1 << order, end_pfn);	8705	pfn, 1 << order, end_pfn);
8603	#endif	8706	#endif
8604	del_page_from_free_area(page, &zone->free_area[order]);	8707	del_page_from_free_area(page, &zone->free_area[order]);
8605	for (i = 0; i < (1 << order); i++)
8606	SetPageReserved((page+i));
8607	pfn += (1 << order);	8708	pfn += (1 << order);
8608	}	8709	}
8609	spin_unlock_irqrestore(&zone->lock, flags);	8710	spin_unlock_irqrestore(&zone->lock, flags);

mm/page_io.c+13−2

 #include <linux/writeback.h>
 #include <linux/frontswap.h>
 #include <linux/blkdev.h>
+#include <linux/psi.h>
 #include <linux/uio.h>
 #include <linux/sched/task.h>
 #include <asm/pgtable.h>
 	struct swap_info_struct *sis = page_swap_info(page);
 	blk_qc_t qc;
 	struct gendisk *disk;
+	unsigned long pflags;
 	VM_BUG_ON_PAGE(!PageSwapCache(page) && !synchronous, page);
 	VM_BUG_ON_PAGE(!PageLocked(page), page);
 	VM_BUG_ON_PAGE(PageUptodate(page), page);
+	/*
+	 * Count submission time as memory stall. When the device is congested,
+	 * or the submitting cgroup IO-throttled, submission can be a
+	 * significant part of overall IO time.
+	 */
+	psi_memstall_enter(&pflags);
 	if (frontswap_load(page) == 0) {
 		SetPageUptodate(page);
 		unlock_page(page);
 		ret = mapping->a_ops->readpage(swap_file, page);
 		if (!ret)
 			count_vm_event(PSWPIN);
-		return ret;
+		goto out;
 	}
 	ret = bdev_read_page(sis->bdev, swap_page_sector(page), page);
 		}
 		count_vm_event(PSWPIN);
-		return 0;
+		goto out;
 	}
 	ret = 0;
 	bio_put(bio);
 out:
+	psi_memstall_leave(&pflags);
 	return ret;
 }

mm/page_isolation.c+6−6

  * @migratetype:	Migrate type to set in error recovery.
  * @flags:		The following flags are allowed (they can be combined in
  *			a bit mask)
- *			SKIP_HWPOISON - ignore hwpoison pages
+ *			MEMORY_OFFLINE - isolate to offline (!allocate) memory
+ *					 e.g., skip over PageHWPoison() pages
  *			REPORT_FAILURE - report details about the failure to
  *			isolate the range
  *
  */
 static unsigned long
 __test_page_isolated_in_pageblock(unsigned long pfn, unsigned long end_pfn,
-				  bool skip_hwpoisoned_pages)
+				  int flags)
 {
 	struct page *page;
 			 * simple way to verify that as VM_BUG_ON(), though.
 			 */
 			pfn += 1 << page_order(page);
-		else if (skip_hwpoisoned_pages && PageHWPoison(page))
+		else if ((flags & MEMORY_OFFLINE) && PageHWPoison(page))
 			/* A HWPoisoned page cannot be also PageBuddy */
 			pfn++;
 		else
 /* Caller should ensure that requested range is in a single zone */
 int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn,
-			bool skip_hwpoisoned_pages)
+			int isol_flags)
 {
 	unsigned long pfn, flags;
 	struct page *page;
 	/* Check all pages are free or marked as ISOLATED */
 	zone = page_zone(page);
 	spin_lock_irqsave(&zone->lock, flags);
-	pfn = __test_page_isolated_in_pageblock(start_pfn, end_pfn,
+	pfn = __test_page_isolated_in_pageblock(start_pfn, end_pfn, isol_flags);
-						skip_hwpoisoned_pages);
 	spin_unlock_irqrestore(&zone->lock, flags);
 	trace_test_pages_isolated(start_pfn, end_pfn, pfn);

mm/pgtable-generic.c+9−0

 	pgd_clear(pgd);
 }
+#ifndef __PAGETABLE_P4D_FOLDED
 void p4d_clear_bad(p4d_t *p4d)
 {
 	p4d_ERROR(*p4d);
 	p4d_clear(p4d);
 }
+#endif
+#ifndef __PAGETABLE_PUD_FOLDED
 void pud_clear_bad(pud_t *pud)
 {
 	pud_ERROR(*pud);
 	pud_clear(pud);
 }
+#endif
+/*
+ * Note that the pmd variant below can't be stub'ed out just as for p4d/pud
+ * above. pmd folding is special and typically pmd_* macros refer to upper
+ * level even when folded
+ */
 void pmd_clear_bad(pmd_t *pmd)
 {
 	pmd_ERROR(*pmd);

mm/rmap.c+45−20

  * Attach the anon_vmas from src to dst.
  * Returns 0 on success, -ENOMEM on failure.
  *
- * If dst->anon_vma is NULL this function tries to find and reuse existing
+ * anon_vma_clone() is called by __vma_split(), __split_vma(), copy_vma() and
- * anon_vma which has no vmas and only one child anon_vma. This prevents
+ * anon_vma_fork(). The first three want an exact copy of src, while the last
- * degradation of anon_vma hierarchy to endless linear chain in case of
+ * one, anon_vma_fork(), may try to reuse an existing anon_vma to prevent
- * constantly forking task. On the other hand, an anon_vma with more than one
+ * endless growth of anon_vma. Since dst->anon_vma is set to NULL before call,
- * child isn't reused even if there was no alive vma, thus rmap walker has a
+ * we can identify this case by checking (!dst->anon_vma && src->anon_vma).
- * good chance of avoiding scanning the whole hierarchy when it searches where
+ *
- * page is mapped.
+ * If (!dst->anon_vma && src->anon_vma) is true, this function tries to find
+ * and reuse existing anon_vma which has no vmas and only one child anon_vma.
+ * This prevents degradation of anon_vma hierarchy to endless linear chain in
+ * case of constantly forking task. On the other hand, an anon_vma with more
+ * than one child isn't reused even if there was no alive vma, thus rmap
+ * walker has a good chance of avoiding scanning the whole hierarchy when it
+ * searches where page is mapped.
  */
 int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src)
 {
 	struct anon_vma_chain *avc, *pavc;
 	struct anon_vma *root = NULL;
+	struct vm_area_struct *prev = dst->vm_prev, *pprev = src->vm_prev;
+	/*
+	 * If parent share anon_vma with its vm_prev, keep this sharing in in
+	 * child.
+	 *
+	 * 1. Parent has vm_prev, which implies we have vm_prev.
+	 * 2. Parent and its vm_prev have the same anon_vma.
+	 */
+	if (!dst->anon_vma && src->anon_vma &&
+	    pprev && pprev->anon_vma == src->anon_vma)
+		dst->anon_vma = prev->anon_vma;
 	list_for_each_entry_reverse(pavc, &src->anon_vma_chain, same_vma) {
 		struct anon_vma *anon_vma;
 		 * will always reuse it. Root anon_vma is never reused:
 		 * it has self-parent reference and at least one child.
 		 */
-		if (!dst->anon_vma && anon_vma != src->anon_vma &&
+		if (!dst->anon_vma && src->anon_vma &&
-				anon_vma->degree < 2)
+		    anon_vma != src->anon_vma && anon_vma->degree < 2)
 			dst->anon_vma = anon_vma;
 	}
 	if (dst->anon_vma)
  * chain and verify that the page in question is indeed mapped in it
  * [ something equivalent to page_mapped_in_vma() ].
  *
- * Since anon_vma's slab is DESTROY_BY_RCU and we know from page_remove_rmap()
+ * Since anon_vma's slab is SLAB_TYPESAFE_BY_RCU and we know from
- * that the anon_vma pointer from page->mapping is valid if there is a
+ * page_remove_rmap() that the anon_vma pointer from page->mapping is valid
- * mapcount, we can dereference the anon_vma after observing those.
+ * if there is a mapcount, we can dereference the anon_vma after observing
+ * those.
  */
 struct anon_vma *page_get_anon_vma(struct page *page)
 {
 static void __page_check_anon_rmap(struct page *page,
 	struct vm_area_struct *vma, unsigned long address)
 {
-#ifdef CONFIG_DEBUG_VM
 	/*
 	 * The page's anon-rmap details (mapping and index) are guaranteed to
 	 * be set up correctly at this point.
 	 * are initially only visible via the pagetables, and the pte is locked
 	 * over the call to page_add_new_anon_rmap.
 	 */
-	BUG_ON(page_anon_vma(page)->root != vma->anon_vma->root);
+	VM_BUG_ON_PAGE(page_anon_vma(page)->root != vma->anon_vma->root, page);
-	BUG_ON(page_to_pgoff(page) != linear_page_index(vma, address));
+	VM_BUG_ON_PAGE(page_to_pgoff(page) != linear_page_index(vma, address),
-#endif
+		       page);
 }
 /**
 	if (TestClearPageDoubleMap(page)) {
 		/*
 		 * Subpages can be mapped with PTEs too. Check how many of
-		 * themi are still mapped.
+		 * them are still mapped.
 		 */
 		for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) {
 			if (atomic_add_negative(-1, &page[i]._mapcount))
 				nr++;
 		}
+		/*
+		 * Queue the page for deferred split if at least one small
+		 * page of the compound page is unmapped, but at least one
+		 * small page is still mapped.
+		 */
+		if (nr && nr < HPAGE_PMD_NR)
+			deferred_split_huge_page(page);
 	} else {
 		nr = HPAGE_PMD_NR;
 	}
 	if (unlikely(PageMlocked(page)))
 		clear_page_mlock(page);
-	if (nr) {
+	if (nr)
 		__mod_node_page_state(page_pgdat(page), NR_ANON_MAPPED, -nr);
-		deferred_split_huge_page(page);
-	}
 }
 /**

mm/shmem.c+17−12

 	if (list_empty(&info->swaplist))
 		list_add(&info->swaplist, &shmem_swaplist);
-	if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
+	if (add_to_swap_cache(page, swap,
+			__GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN) == 0) {
 		spin_lock_irq(&info->lock);
 		shmem_recalc_inode(inode);
 		info->swapped++;
 		    shmem_falloc->waitq &&
 		    vmf->pgoff >= shmem_falloc->start &&
 		    vmf->pgoff < shmem_falloc->next) {
+			struct file *fpin;
 			wait_queue_head_t *shmem_falloc_waitq;
 			DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function);
 			ret = VM_FAULT_NOPAGE;
-			if ((vmf->flags & FAULT_FLAG_ALLOW_RETRY) &&
+			fpin = maybe_unlock_mmap_for_io(vmf, NULL);
-			   !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) {
+			if (fpin)
-				/* It's polite to up mmap_sem if we can */
-				up_read(&vma->vm_mm->mmap_sem);
 				ret = VM_FAULT_RETRY;
-			}
 			shmem_falloc_waitq = shmem_falloc->waitq;
 			prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait,
 			spin_lock(&inode->i_lock);
 			finish_wait(shmem_falloc_waitq, &shmem_fault_wait);
 			spin_unlock(&inode->i_lock);
+			if (fpin)
+				fput(fpin);
 			return ret;
 		}
 		spin_unlock(&inode->i_lock);
 			return -EPERM;
 		/*
-		 * Since the F_SEAL_FUTURE_WRITE seals allow for a MAP_SHARED
+		 * Since an F_SEAL_FUTURE_WRITE sealed memfd can be mapped as
-		 * read-only mapping, take care to not allow mprotect to revert
+		 * MAP_SHARED and read-only, take care to not allow mprotect to
-		 * protections.
+		 * revert protections on such mappings. Do this only for shared
+		 * mappings. For private mappings, don't need to mask
+		 * VM_MAYWRITE as we still want them to be COW-writable.
 		 */
-		vma->vm_flags &= ~(VM_MAYWRITE);
+		if (vma->vm_flags & VM_SHARED)
+			vma->vm_flags &= ~(VM_MAYWRITE);
 	}
 	file_accessed(file);
 		}
 		shmem_falloc.waitq = &shmem_falloc_waitq;
-		shmem_falloc.start = unmap_start >> PAGE_SHIFT;
+		shmem_falloc.start = (u64)unmap_start >> PAGE_SHIFT;
 		shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT;
 		spin_lock(&inode->i_lock);
 		inode->i_private = &shmem_falloc;
 static ssize_t shmem_enabled_show(struct kobject *kobj,
 		struct kobj_attribute *attr, char *buf)
 {
-	int values[] = {
+	static const int values[] = {
 		SHMEM_HUGE_ALWAYS,
 		SHMEM_HUGE_WITHIN_SIZE,
 		SHMEM_HUGE_ADVISE,

mm/slab.c+4−3

 	 * structures first.  Without this, further allocations will bug.
 	 */
 	kmalloc_caches[KMALLOC_NORMAL][INDEX_NODE] = create_kmalloc_cache(
-				kmalloc_info[INDEX_NODE].name,
+				kmalloc_info[INDEX_NODE].name[KMALLOC_NORMAL],
-				kmalloc_size(INDEX_NODE), ARCH_KMALLOC_FLAGS,
+				kmalloc_info[INDEX_NODE].size,
-				0, kmalloc_size(INDEX_NODE));
+				ARCH_KMALLOC_FLAGS, 0,
+				kmalloc_info[INDEX_NODE].size);
 	slab_state = PARTIAL_NODE;
 	setup_kmalloc_cache_index_table();

mm/slab.h+3−3

@@ -139,7 +139,7 @@ extern struct kmem_cache *kmem_cache; · 138 unmodified lines
139		139
140	/* A table of kmalloc cache names and sizes */	140	/* A table of kmalloc cache names and sizes */
141	extern const struct kmalloc_info_struct {	141	extern const struct kmalloc_info_struct {
142	const char *name;	142	const char *name[NR_KMALLOC_TYPES];
143	unsigned int size;	143	unsigned int size;
144	} kmalloc_info[];	144	} kmalloc_info[];
145		145
@@ -369,7 +369,7 @@ static __always_inline int memcg_charge_slab(struct page *page, · 223 unmodified lines
369	if (ret)	369	if (ret)
370	goto out;	370	goto out;
371		371
372	lruvec = mem_cgroup_lruvec(page_pgdat(page), memcg);	372	lruvec = mem_cgroup_lruvec(memcg, page_pgdat(page));
373	mod_lruvec_state(lruvec, cache_vmstat_idx(s), 1 << order);	373	mod_lruvec_state(lruvec, cache_vmstat_idx(s), 1 << order);
374		374
375	/* transer try_charge() page references to kmem_cache */	375	/* transer try_charge() page references to kmem_cache */
@@ -393,7 +393,7 @@ static __always_inline void memcg_uncharge_slab(struct page *page, int order, · 17 unmodified lines
393	rcu_read_lock();	393	rcu_read_lock();
394	memcg = READ_ONCE(s->memcg_params.memcg);	394	memcg = READ_ONCE(s->memcg_params.memcg);
395	if (likely(!mem_cgroup_is_root(memcg))) {	395	if (likely(!mem_cgroup_is_root(memcg))) {
396	lruvec = mem_cgroup_lruvec(page_pgdat(page), memcg);	396	lruvec = mem_cgroup_lruvec(memcg, page_pgdat(page));
397	mod_lruvec_state(lruvec, cache_vmstat_idx(s), -(1 << order));	397	mod_lruvec_state(lruvec, cache_vmstat_idx(s), -(1 << order));
398	memcg_kmem_uncharge_memcg(page, order, memcg);	398	memcg_kmem_uncharge_memcg(page, order, memcg);
399	} else {	399	} else {

mm/slab_common.c+53−46

 	return kmalloc_caches[kmalloc_type(flags)][index];
 }
+#ifdef CONFIG_ZONE_DMA
+#define INIT_KMALLOC_INFO(__size, __short_size)			\
+{								\
+	.name[KMALLOC_NORMAL]  = "kmalloc-" #__short_size,	\
+	.name[KMALLOC_RECLAIM] = "kmalloc-rcl-" #__short_size,	\
+	.name[KMALLOC_DMA]     = "dma-kmalloc-" #__short_size,	\
+	.size = __size,						\
+}
+#else
+#define INIT_KMALLOC_INFO(__size, __short_size)			\
+{								\
+	.name[KMALLOC_NORMAL]  = "kmalloc-" #__short_size,	\
+	.name[KMALLOC_RECLAIM] = "kmalloc-rcl-" #__short_size,	\
+	.size = __size,						\
+}
+#endif
 /*
  * kmalloc_info[] is to make slub_debug=,kmalloc-xx option work at boot time.
  * kmalloc_index() supports up to 2^26=64MB, so the final entry of the table is
  * kmalloc-67108864.
  */
 const struct kmalloc_info_struct kmalloc_info[] __initconst = {
-	{NULL,                      0},		{"kmalloc-96",             96},
+	INIT_KMALLOC_INFO(0, 0),
-	{"kmalloc-192",           192},		{"kmalloc-8",               8},
+	INIT_KMALLOC_INFO(96, 96),
-	{"kmalloc-16",             16},		{"kmalloc-32",             32},
+	INIT_KMALLOC_INFO(192, 192),
-	{"kmalloc-64",             64},		{"kmalloc-128",           128},
+	INIT_KMALLOC_INFO(8, 8),
-	{"kmalloc-256",           256},		{"kmalloc-512",           512},
+	INIT_KMALLOC_INFO(16, 16),
-	{"kmalloc-1k",           1024},		{"kmalloc-2k",           2048},
+	INIT_KMALLOC_INFO(32, 32),
-	{"kmalloc-4k",           4096},		{"kmalloc-8k",           8192},
+	INIT_KMALLOC_INFO(64, 64),
-	{"kmalloc-16k",         16384},		{"kmalloc-32k",         32768},
+	INIT_KMALLOC_INFO(128, 128),
-	{"kmalloc-64k",         65536},		{"kmalloc-128k",       131072},
+	INIT_KMALLOC_INFO(256, 256),
-	{"kmalloc-256k",       262144},		{"kmalloc-512k",       524288},
+	INIT_KMALLOC_INFO(512, 512),
-	{"kmalloc-1M",        1048576},		{"kmalloc-2M",        2097152},
+	INIT_KMALLOC_INFO(1024, 1k),
-	{"kmalloc-4M",        4194304},		{"kmalloc-8M",        8388608},
+	INIT_KMALLOC_INFO(2048, 2k),
-	{"kmalloc-16M",      16777216},		{"kmalloc-32M",      33554432},
+	INIT_KMALLOC_INFO(4096, 4k),
-	{"kmalloc-64M",      67108864}
+	INIT_KMALLOC_INFO(8192, 8k),
+	INIT_KMALLOC_INFO(16384, 16k),
+	INIT_KMALLOC_INFO(32768, 32k),
+	INIT_KMALLOC_INFO(65536, 64k),
+	INIT_KMALLOC_INFO(131072, 128k),
+	INIT_KMALLOC_INFO(262144, 256k),
+	INIT_KMALLOC_INFO(524288, 512k),
+	INIT_KMALLOC_INFO(1048576, 1M),
+	INIT_KMALLOC_INFO(2097152, 2M),
+	INIT_KMALLOC_INFO(4194304, 4M),
+	INIT_KMALLOC_INFO(8388608, 8M),
+	INIT_KMALLOC_INFO(16777216, 16M),
+	INIT_KMALLOC_INFO(33554432, 32M),
+	INIT_KMALLOC_INFO(67108864, 64M)
 };
 /*
 	}
 }
-static const char *
-kmalloc_cache_name(const char *prefix, unsigned int size)
-{
-	static const char units[3] = "\0kM";
-	int idx = 0;
-	while (size >= 1024 && (size % 1024 == 0)) {
-		size /= 1024;
-		idx++;
-	}
-	return kasprintf(GFP_NOWAIT, "%s-%u%c", prefix, size, units[idx]);
-}
 static void __init
-new_kmalloc_cache(int idx, int type, slab_flags_t flags)
+new_kmalloc_cache(int idx, enum kmalloc_cache_type type, slab_flags_t flags)
 {
-	const char *name;
+	if (type == KMALLOC_RECLAIM)
-	if (type == KMALLOC_RECLAIM) {
 		flags |= SLAB_RECLAIM_ACCOUNT;
-		name = kmalloc_cache_name("kmalloc-rcl",
-						kmalloc_info[idx].size);
-		BUG_ON(!name);
-	} else {
-		name = kmalloc_info[idx].name;
-	}
-	kmalloc_caches[type][idx] = create_kmalloc_cache(name,
+	kmalloc_caches[type][idx] = create_kmalloc_cache(
+					kmalloc_info[idx].name[type],
 					kmalloc_info[idx].size, flags, 0,
 					kmalloc_info[idx].size);
 }
  */
 void __init create_kmalloc_caches(slab_flags_t flags)
 {
-	int i, type;
+	int i;
+	enum kmalloc_cache_type type;
 	for (type = KMALLOC_NORMAL; type <= KMALLOC_RECLAIM; type++) {
 		for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
 		struct kmem_cache *s = kmalloc_caches[KMALLOC_NORMAL][i];
 		if (s) {
-			unsigned int size = kmalloc_size(i);
-			const char *n = kmalloc_cache_name("dma-kmalloc", size);
-			BUG_ON(!n);
 			kmalloc_caches[KMALLOC_DMA][i] = create_kmalloc_cache(
-				n, size, SLAB_CACHE_DMA | flags, 0, 0);
+				kmalloc_info[i].name[KMALLOC_DMA],
+				kmalloc_info[i].size,
+				SLAB_CACHE_DMA | flags, 0, 0);
 		}
 	}
 #endif

mm/slub.c+16−20

  * minimal so we rely on the page allocators per cpu caches for
  * fast frees and allocs.
  *
- * Overloading of page flags that are otherwise used for LRU management.
+ * page->frozen		The slab is frozen and exempt from list processing.
- *
- * PageActive 		The slab is frozen and exempt from list processing.
  * 			This means that the slab is dedicated to a purpose
  * 			such as satisfying allocations for a specific
  * 			processor. Objects may be freed in the slab while
  * 			free objects in addition to the regular freelist
  * 			that requires the slab lock.
  *
- * PageError		Slab requires special handling due to debug
+ * SLAB_DEBUG_FLAGS	Slab requires special handling due to debug
  * 			options set. This moves	slab handling out of
  * 			the fast path and disables lockless freelists.
  */
 {
 	u8 *fault;
 	u8 *end;
+	u8 *addr = page_address(page);
 	metadata_access_enable();
 	fault = memchr_inv(start, value, bytes);
 		end--;
 	slab_bug(s, "%s overwritten", what);
-	pr_err("INFO: 0x%p-0x%p. First byte 0x%x instead of 0x%x\n",
+	pr_err("INFO: 0x%p-0x%p @offset=%tu. First byte 0x%x instead of 0x%x\n",
-					fault, end - 1, fault[0], value);
+					fault, end - 1, fault - addr,
+					fault[0], value);
 	print_trailer(s, page, object);
 	restore_bytes(s, what, value, fault, end);
 	while (end > fault && end[-1] == POISON_INUSE)
 		end--;
-	slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1);
+	slab_err(s, page, "Padding overwritten. 0x%p-0x%p @offset=%tu",
+			fault, end - 1, fault - start);
 	print_section(KERN_ERR, "Padding ", pad, remainder);
 	restore_bytes(s, "slab padding", POISON_INUSE, fault, end);
 #endif
 #ifdef CONFIG_SLUB_DEBUG
-static int validate_slab(struct kmem_cache *s, struct page *page,
+static void validate_slab(struct kmem_cache *s, struct page *page,
 						unsigned long *map)
 {
 	void *p;
 	void *addr = page_address(page);
-	if (!check_slab(s, page) ||
+	if (!check_slab(s, page) || !on_freelist(s, page, NULL))
-			!on_freelist(s, page, NULL))
+		return;
-		return 0;
 	/* Now we know that a valid freelist exists */
 	bitmap_zero(map, page->objects);
 	get_map(s, page, map);
 	for_each_object(p, s, addr, page->objects) {
-		if (test_bit(slab_index(p, s, addr), map))
+		u8 val = test_bit(slab_index(p, s, addr), map) ?
-			if (!check_object(s, page, p, SLUB_RED_INACTIVE))
+			 SLUB_RED_INACTIVE : SLUB_RED_ACTIVE;
-				return 0;
-	}
-	for_each_object(p, s, addr, page->objects)
+		if (!check_object(s, page, p, val))
-		if (!test_bit(slab_index(p, s, addr), map))
+			break;
-			if (!check_object(s, page, p, SLUB_RED_ACTIVE))
+	}
-				return 0;
-	return 1;
 }
 static void validate_slab_slab(struct kmem_cache *s, struct page *page,

mm/sparse.c+10−8

 	if (map)
 		return map;
-	map = memblock_alloc_try_nid(size,
+	map = memblock_alloc_try_nid_raw(size, size, addr,
-					  PAGE_SIZE, addr,
 					  MEMBLOCK_ALLOC_ACCESSIBLE, nid);
 	if (!map)
 		panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa\n",
 {
 	phys_addr_t addr = __pa(MAX_DMA_ADDRESS);
 	WARN_ON(sparsemap_buf);	/* forgot to call sparse_buffer_fini()? */
-	sparsemap_buf =
+	/*
-		memblock_alloc_try_nid_raw(size, PAGE_SIZE,
+	 * Pre-allocated buffer is mainly used by __populate_section_memmap
-						addr,
+	 * and we want it to be properly aligned to the section size - this is
-						MEMBLOCK_ALLOC_ACCESSIBLE, nid);
+	 * especially the case for VMEMMAP which maps memmap to PMDs
+	 */
+	sparsemap_buf = memblock_alloc_exact_nid_raw(size, section_map_size(),
+					addr, MEMBLOCK_ALLOC_ACCESSIBLE, nid);
 	sparsemap_buf_end = sparsemap_buf + size;
 }
 #endif
 #ifdef CONFIG_SPARSEMEM_VMEMMAP
-static struct page *populate_section_memmap(unsigned long pfn,
+static struct page * __meminit populate_section_memmap(unsigned long pfn,
 		unsigned long nr_pages, int nid, struct vmem_altmap *altmap)
 {
 	return __populate_section_memmap(pfn, nr_pages, nid, altmap);
 	vmemmap_free(start, end, NULL);
 }
 #else
-struct page *populate_section_memmap(unsigned long pfn,
+struct page * __meminit populate_section_memmap(unsigned long pfn,
 		unsigned long nr_pages, int nid, struct vmem_altmap *altmap)
 {
 	struct page *page, *ret;

mm/swap.c+24−5

 void mark_page_accessed(struct page *page)
 {
 	page = compound_head(page);
-	if (!PageActive(page) && !PageUnevictable(page) &&
-			PageReferenced(page)) {
+	if (!PageReferenced(page)) {
+		SetPageReferenced(page);
+	} else if (PageUnevictable(page)) {
+		/*
+		 * Unevictable pages are on the "LRU_UNEVICTABLE" list. But,
+		 * this list is never rotated or maintained, so marking an
+		 * evictable page accessed has no effect.
+		 */
+	} else if (!PageActive(page)) {
 		/*
 		 * If the page is on the LRU, queue it for activation via
 		 * activate_page_pvecs. Otherwise, assume the page is on a
 		ClearPageReferenced(page);
 		if (page_is_file_cache(page))
 			workingset_activation(page);
-	} else if (!PageReferenced(page)) {
-		SetPageReferenced(page);
 	}
 	if (page_is_idle(page))
 		clear_page_idle(page);
  */
 void lru_add_drain_all(void)
 {
+	static seqcount_t seqcount = SEQCNT_ZERO(seqcount);
 	static DEFINE_MUTEX(lock);
 	static struct cpumask has_work;
-	int cpu;
+	int cpu, seq;
 	/*
 	 * Make sure nobody triggers this path before mm_percpu_wq is fully
 	if (WARN_ON(!mm_percpu_wq))
 		return;
+	seq = raw_read_seqcount_latch(&seqcount);
 	mutex_lock(&lock);
+	/*
+	 * Piggyback on drain started and finished while we waited for lock:
+	 * all pages pended at the time of our enter were drained from vectors.
+	 */
+	if (__read_seqcount_retry(&seqcount, seq))
+		goto done;
+	raw_write_seqcount_latch(&seqcount);
 	cpumask_clear(&has_work);
 	for_each_online_cpu(cpu) {
 	for_each_cpu(cpu, &has_work)
 		flush_work(&per_cpu(lru_add_drain_work, cpu));
+done:
 	mutex_unlock(&lock);
 }
 #else

mm/swapfile.c+7−0

@@ -2887,6 +2887,13 @@ static int claim_swapfile(struct swap_info_struct p, struct inode inode) · 2886 unmodified lines
2887	error = set_blocksize(p->bdev, PAGE_SIZE);	2887	error = set_blocksize(p->bdev, PAGE_SIZE);
2888	if (error < 0)	2888	if (error < 0)
2889	return error;	2889	return error;
		2890	/*
		2891	* Zoned block devices contain zones that have a sequential
		2892	* write only restriction. Hence zoned block devices are not
		2893	* suitable for swapping. Disallow them here.
		2894	*/
		2895	if (blk_queue_is_zoned(p->bdev->bd_queue))
		2896	return -EINVAL;
2890	p->flags \|= SWP_BLKDEV;	2897	p->flags \|= SWP_BLKDEV;
2891	} else if (S_ISREG(inode->i_mode)) {	2898	} else if (S_ISREG(inode->i_mode)) {
2892	p->bdev = inode->i_sb->s_bdev;	2899	p->bdev = inode->i_sb->s_bdev;

mm/userfaultfd.c+37−36

 #include <asm/tlbflush.h>
 #include "internal.h"
+static __always_inline
+struct vm_area_struct *find_dst_vma(struct mm_struct *dst_mm,
+				    unsigned long dst_start,
+				    unsigned long len)
+{
+	/*
+	 * Make sure that the dst range is both valid and fully within a
+	 * single existing vma.
+	 */
+	struct vm_area_struct *dst_vma;
+	dst_vma = find_vma(dst_mm, dst_start);
+	if (!dst_vma)
+		return NULL;
+	if (dst_start < dst_vma->vm_start ||
+	    dst_start + len > dst_vma->vm_end)
+		return NULL;
+	/*
+	 * Check the vma is registered in uffd, this is required to
+	 * enforce the VM_MAYWRITE check done at uffd registration
+	 * time.
+	 */
+	if (!dst_vma->vm_userfaultfd_ctx.ctx)
+		return NULL;
+	return dst_vma;
+}
 static int mcopy_atomic_pte(struct mm_struct *dst_mm,
 			    pmd_t *dst_pmd,
 			    struct vm_area_struct *dst_vma,
 	/*
 	 * The memory barrier inside __SetPageUptodate makes sure that
-	 * preceeding stores to the page contents become visible before
+	 * preceding stores to the page contents become visible before
 	 * the set_pte_at() write.
 	 */
 	__SetPageUptodate(page);
 	unsigned long src_addr, dst_addr;
 	long copied;
 	struct page *page;
-	struct hstate *h;
 	unsigned long vma_hpagesize;
 	pgoff_t idx;
 	u32 hash;
 	 */
 	if (!dst_vma) {
 		err = -ENOENT;
-		dst_vma = find_vma(dst_mm, dst_start);
+		dst_vma = find_dst_vma(dst_mm, dst_start, len);
 		if (!dst_vma || !is_vm_hugetlb_page(dst_vma))
 			goto out_unlock;
-		/*
-		 * Check the vma is registered in uffd, this is
-		 * required to enforce the VM_MAYWRITE check done at
-		 * uffd registration time.
-		 */
-		if (!dst_vma->vm_userfaultfd_ctx.ctx)
-			goto out_unlock;
-		if (dst_start < dst_vma->vm_start ||
-		    dst_start + len > dst_vma->vm_end)
-			goto out_unlock;
 		err = -EINVAL;
 		if (vma_hpagesize != vma_kernel_pagesize(dst_vma))
 		vm_shared = dst_vma->vm_flags & VM_SHARED;
 	}
-	if (WARN_ON(dst_addr & (vma_hpagesize - 1) ||
-		    (len - copied) & (vma_hpagesize - 1)))
-		goto out_unlock;
 	/*
 	 * If not shared, ensure the dst_vma has a anon_vma.
 	 */
 			goto out_unlock;
 	}
-	h = hstate_vma(dst_vma);
 	while (src_addr < src_start + len) {
 		pte_t dst_pteval;
 		BUG_ON(dst_addr >= dst_start + len);
-		VM_BUG_ON(dst_addr & ~huge_page_mask(h));
 		/*
 		 * Serialize via hugetlb_fault_mutex
 		 */
 		idx = linear_page_index(dst_vma, dst_addr);
 		mapping = dst_vma->vm_file->f_mapping;
-		hash = hugetlb_fault_mutex_hash(h, mapping, idx, dst_addr);
+		hash = hugetlb_fault_mutex_hash(mapping, idx);
 		mutex_lock(&hugetlb_fault_mutex_table[hash]);
 		err = -ENOMEM;
-		dst_pte = huge_pte_alloc(dst_mm, dst_addr, huge_page_size(h));
+		dst_pte = huge_pte_alloc(dst_mm, dst_addr, vma_hpagesize);
 		if (!dst_pte) {
 			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
 			goto out_unlock;
 			err = copy_huge_page_from_user(page,
 						(const void __user *)src_addr,
-						pages_per_huge_page(h), true);
+						vma_hpagesize / PAGE_SIZE,
+						true);
 			if (unlikely(err)) {
 				err = -EFAULT;
 				goto out;
 	 * both valid and fully within a single existing vma.
 	 */
 	err = -ENOENT;
-	dst_vma = find_vma(dst_mm, dst_start);
+	dst_vma = find_dst_vma(dst_mm, dst_start, len);
 	if (!dst_vma)
 		goto out_unlock;
-	/*
-	 * Check the vma is registered in uffd, this is required to
-	 * enforce the VM_MAYWRITE check done at uffd registration
-	 * time.
-	 */
-	if (!dst_vma->vm_userfaultfd_ctx.ctx)
-		goto out_unlock;
-	if (dst_start < dst_vma->vm_start ||
-	    dst_start + len > dst_vma->vm_end)
-		goto out_unlock;
 	err = -EINVAL;
 	/*

mm/util.c+16−6

 EXPORT_SYMBOL(memdup_user_nul);
 void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
-		struct vm_area_struct *prev, struct rb_node *rb_parent)
+		struct vm_area_struct *prev)
 {
 	struct vm_area_struct *next;
 		next = prev->vm_next;
 		prev->vm_next = vma;
 	} else {
+		next = mm->mmap;
 		mm->mmap = vma;
-		if (rb_parent)
-			next = rb_entry(rb_parent,
-					struct vm_area_struct, vm_rb);
-		else
-			next = NULL;
 	}
 	vma->vm_next = next;
 	if (next)
 		next->vm_prev = vma;
 }
+void __vma_unlink_list(struct mm_struct *mm, struct vm_area_struct *vma)
+{
+	struct vm_area_struct *prev, *next;
+	next = vma->vm_next;
+	prev = vma->vm_prev;
+	if (prev)
+		prev->vm_next = next;
+	else
+		mm->mmap = next;
+	if (next)
+		next->vm_prev = prev;
+}
 /* Check if the vma is being used as a stack by this task */
 int vma_is_stack_for_current(struct vm_area_struct *vma)
 {

mm/vmalloc.c+134−58

@@ -331,6 +331,7 @@ EXPORT_SYMBOL(vmalloc_to_pfn); · 330 unmodified lines
331		331
332		332
333	static DEFINE_SPINLOCK(vmap_area_lock);	333	static DEFINE_SPINLOCK(vmap_area_lock);
		334	static DEFINE_SPINLOCK(free_vmap_area_lock);
334	/* Export for kexec only */	335	/* Export for kexec only */
335	LIST_HEAD(vmap_area_list);	336	LIST_HEAD(vmap_area_list);
336	static LLIST_HEAD(vmap_purge_list);	337	static LLIST_HEAD(vmap_purge_list);
@@ -682,7 +683,7 @@ insert_vmap_area_augment(struct vmap_area *va, · 345 unmodified lines
682	* free area is inserted. If VA has been merged, it is	683	* free area is inserted. If VA has been merged, it is
683	* freed.	684	* freed.
684	*/	685	*/
685	static __always_inline void	686	static __always_inline struct vmap_area *
686	merge_or_add_vmap_area(struct vmap_area *va,	687	merge_or_add_vmap_area(struct vmap_area *va,
687	struct rb_root root, struct list_head head)	688	struct rb_root root, struct list_head head)
688	{	689	{
@@ -749,7 +750,10 @@ merge_or_add_vmap_area(struct vmap_area *va, · 60 unmodified lines
749		750
750	/* Free vmap_area object. */	751	/* Free vmap_area object. */
751	kmem_cache_free(vmap_area_cachep, va);	752	kmem_cache_free(vmap_area_cachep, va);
752	return;	753
		754	/* Point to the new merged area. */
		755	va = sibling;
		756	merged = true;
753	}	757	}
754	}	758	}
755		759
@@ -758,6 +762,8 @@ insert: · 2 unmodified lines
758	link_va(va, root, parent, link, head);	762	link_va(va, root, parent, link, head);
759	augment_tree_propagate_from(va);	763	augment_tree_propagate_from(va);
760	}	764	}
		765
		766	return va;
761	}	767	}
762		768
763	static __always_inline bool	769	static __always_inline bool
@@ -968,6 +974,19 @@ adjust_va_to_fit_type(struct vmap_area *va, · 204 unmodified lines
968	* There are a few exceptions though, as an example it is	974	* There are a few exceptions though, as an example it is
969	* a first allocation (early boot up) when we have "one"	975	* a first allocation (early boot up) when we have "one"
970	* big free space that has to be split.	976	* big free space that has to be split.
		977	*
		978	* Also we can hit this path in case of regular "vmap"
		979	* allocations, if "this" current CPU was not preloaded.
		980	* See the comment in alloc_vmap_area() why. If so, then
		981	* GFP_NOWAIT is used instead to get an extra object for
		982	* split purpose. That is rare and most time does not
		983	* occur.
		984	*
		985	* What happens if an allocation gets failed. Basically,
		986	* an "overflow" path is triggered to purge lazily freed
		987	* areas to free some memory, then, the "retry" path is
		988	* triggered to repeat one more time. See more details
		989	* in alloc_vmap_area() function.
971	*/	990	*/
972	lva = kmem_cache_alloc(vmap_area_cachep, GFP_NOWAIT);	991	lva = kmem_cache_alloc(vmap_area_cachep, GFP_NOWAIT);
973	if (!lva)	992	if (!lva)
@@ -1063,9 +1082,9 @@ static struct vmap_area *alloc_vmap_area(unsigned long size, · 89 unmodified lines
1063	return ERR_PTR(-EBUSY);	1082	return ERR_PTR(-EBUSY);
1064		1083
1065	might_sleep();	1084	might_sleep();
		1085	gfp_mask = gfp_mask & GFP_RECLAIM_MASK;
1066		1086
1067	va = kmem_cache_alloc_node(vmap_area_cachep,	1087	va = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node);
1068	gfp_mask & GFP_RECLAIM_MASK, node);
1069	if (unlikely(!va))	1088	if (unlikely(!va))
1070	return ERR_PTR(-ENOMEM);	1089	return ERR_PTR(-ENOMEM);
1071		1090
@@ -1073,49 +1092,55 @@ static struct vmap_area *alloc_vmap_area(unsigned long size, · 1 unmodified line
1073	* Only scan the relevant parts containing pointers to other objects	1092	* Only scan the relevant parts containing pointers to other objects
1074	* to avoid false negatives.	1093	* to avoid false negatives.
1075	*/	1094	*/
1076	kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask & GFP_RECLAIM_MASK);	1095	kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask);
1077		1096
1078	retry:	1097	retry:
1079	/*	1098	/*
1080	* Preload this CPU with one extra vmap_area object to ensure	1099	* Preload this CPU with one extra vmap_area object. It is used
1081	* that we have it available when fit type of free area is	1100	* when fit type of free area is NE_FIT_TYPE. Please note, it
1082	* NE_FIT_TYPE.	1101	* does not guarantee that an allocation occurs on a CPU that
		1102	* is preloaded, instead we minimize the case when it is not.
		1103	* It can happen because of cpu migration, because there is a
		1104	* race until the below spinlock is taken.
1083	*	1105	*
1084	* The preload is done in non-atomic context, thus it allows us	1106	* The preload is done in non-atomic context, thus it allows us
1085	* to use more permissive allocation masks to be more stable under	1107	* to use more permissive allocation masks to be more stable under
1086	* low memory condition and high memory pressure.	1108	* low memory condition and high memory pressure. In rare case,
		1109	* if not preloaded, GFP_NOWAIT is used.
1087	*	1110	*
1088	* Even if it fails we do not really care about that. Just proceed	1111	* Set "pva" to NULL here, because of "retry" path.
1089	* as it is. "overflow" path will refill the cache we allocate from.
1090	*/	1112	*/
1091	preempt_disable();	1113	pva = NULL;
1092	if (!__this_cpu_read(ne_fit_preload_node)) {
1093	preempt_enable();
1094	pva = kmem_cache_alloc_node(vmap_area_cachep, GFP_KERNEL, node);
1095	preempt_disable();
1096
1097	if (__this_cpu_cmpxchg(ne_fit_preload_node, NULL, pva)) {
1098	if (pva)
1099	kmem_cache_free(vmap_area_cachep, pva);
1100	}
1101	}
1102		1114
1103	spin_lock(&vmap_area_lock);	1115	if (!this_cpu_read(ne_fit_preload_node))
1104	preempt_enable();	1116	/*
		1117	* Even if it fails we do not really care about that.
		1118	* Just proceed as it is. If needed "overflow" path
		1119	* will refill the cache we allocate from.
		1120	*/
		1121	pva = kmem_cache_alloc_node(vmap_area_cachep, gfp_mask, node);
		1122
		1123	spin_lock(&free_vmap_area_lock);
		1124
		1125	if (pva && __this_cpu_cmpxchg(ne_fit_preload_node, NULL, pva))
		1126	kmem_cache_free(vmap_area_cachep, pva);
1105		1127
1106	/*	1128	/*
1107	* If an allocation fails, the "vend" address is	1129	* If an allocation fails, the "vend" address is
1108	* returned. Therefore trigger the overflow path.	1130	* returned. Therefore trigger the overflow path.
1109	*/	1131	*/
1110	addr = __alloc_vmap_area(size, align, vstart, vend);	1132	addr = __alloc_vmap_area(size, align, vstart, vend);
		1133	spin_unlock(&free_vmap_area_lock);
		1134
1111	if (unlikely(addr == vend))	1135	if (unlikely(addr == vend))
1112	goto overflow;	1136	goto overflow;
1113		1137
1114	va->va_start = addr;	1138	va->va_start = addr;
1115	va->va_end = addr + size;	1139	va->va_end = addr + size;
1116	va->vm = NULL;	1140	va->vm = NULL;
1117	insert_vmap_area(va, &vmap_area_root, &vmap_area_list);
1118		1141
		1142	spin_lock(&vmap_area_lock);
		1143	insert_vmap_area(va, &vmap_area_root, &vmap_area_list);
1119	spin_unlock(&vmap_area_lock);	1144	spin_unlock(&vmap_area_lock);
1120		1145
1121	BUG_ON(!IS_ALIGNED(va->va_start, align));	1146	BUG_ON(!IS_ALIGNED(va->va_start, align));
@@ -1125,7 +1150,6 @@ retry: · 3 unmodified lines
1125	return va;	1150	return va;
1126		1151
1127	overflow:	1152	overflow:
1128	spin_unlock(&vmap_area_lock);
1129	if (!purged) {	1153	if (!purged) {
1130	purge_vmap_area_lazy();	1154	purge_vmap_area_lazy();
1131	purged = 1;	1155	purged = 1;
@@ -1161,28 +1185,24 @@ int unregister_vmap_purge_notifier(struct notifier_block *nb) · 29 unmodified lines
1161	}	1185	}
1162	EXPORT_SYMBOL_GPL(unregister_vmap_purge_notifier);	1186	EXPORT_SYMBOL_GPL(unregister_vmap_purge_notifier);
1163		1187
1164	static void __free_vmap_area(struct vmap_area *va)	1188	/*
		1189	* Free a region of KVA allocated by alloc_vmap_area
		1190	*/
		1191	static void free_vmap_area(struct vmap_area *va)
1165	{	1192	{
1166	/*	1193	/*
1167	* Remove from the busy tree/list.	1194	* Remove from the busy tree/list.
1168	*/	1195	*/
		1196	spin_lock(&vmap_area_lock);
1169	unlink_va(va, &vmap_area_root);	1197	unlink_va(va, &vmap_area_root);
		1198	spin_unlock(&vmap_area_lock);
1170		1199
1171	/*	1200	/*
1172	* Merge VA with its neighbors, otherwise just add it.	1201	* Insert/Merge it back to the free tree/list.
1173	*/	1202	*/
1174	merge_or_add_vmap_area(va,	1203	spin_lock(&free_vmap_area_lock);
1175	&free_vmap_area_root, &free_vmap_area_list);	1204	merge_or_add_vmap_area(va, &free_vmap_area_root, &free_vmap_area_list);
1176	}	1205	spin_unlock(&free_vmap_area_lock);
1177
1178	/*
1179	* Free a region of KVA allocated by alloc_vmap_area
1180	*/
1181	static void free_vmap_area(struct vmap_area *va)
1182	{
1183	spin_lock(&vmap_area_lock);
1184	__free_vmap_area(va);
1185	spin_unlock(&vmap_area_lock);
1186	}	1206	}
1187		1207
1188	/*	1208	/*
@@ -1275,24 +1295,30 @@ static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end) · 86 unmodified lines
1275	flush_tlb_kernel_range(start, end);	1295	flush_tlb_kernel_range(start, end);
1276	resched_threshold = lazy_max_pages() << 1;	1296	resched_threshold = lazy_max_pages() << 1;
1277		1297
1278	spin_lock(&vmap_area_lock);	1298	spin_lock(&free_vmap_area_lock);
1279	llist_for_each_entry_safe(va, n_va, valist, purge_list) {	1299	llist_for_each_entry_safe(va, n_va, valist, purge_list) {
1280	unsigned long nr = (va->va_end - va->va_start) >> PAGE_SHIFT;	1300	unsigned long nr = (va->va_end - va->va_start) >> PAGE_SHIFT;
		1301	unsigned long orig_start = va->va_start;
		1302	unsigned long orig_end = va->va_end;
1281		1303
1282	/*	1304	/*
1283	* Finally insert or merge lazily-freed area. It is	1305	* Finally insert or merge lazily-freed area. It is
1284	* detached and there is no need to "unlink" it from	1306	* detached and there is no need to "unlink" it from
1285	* anything.	1307	* anything.
1286	*/	1308	*/
1287	merge_or_add_vmap_area(va,	1309	va = merge_or_add_vmap_area(va, &free_vmap_area_root,
1288	&free_vmap_area_root, &free_vmap_area_list);	1310	&free_vmap_area_list);
		1311
		1312	if (is_vmalloc_or_module_addr((void *)orig_start))
		1313	kasan_release_vmalloc(orig_start, orig_end,
		1314	va->va_start, va->va_end);
1289		1315
1290	atomic_long_sub(nr, &vmap_lazy_nr);	1316	atomic_long_sub(nr, &vmap_lazy_nr);
1291		1317
1292	if (atomic_long_read(&vmap_lazy_nr) < resched_threshold)	1318	if (atomic_long_read(&vmap_lazy_nr) < resched_threshold)
1293	cond_resched_lock(&vmap_area_lock);	1319	cond_resched_lock(&free_vmap_area_lock);
1294	}	1320	}
1295	spin_unlock(&vmap_area_lock);	1321	spin_unlock(&free_vmap_area_lock);
1296	return true;	1322	return true;
1297	}	1323	}
1298		1324
@@ -2014,15 +2040,21 @@ int map_vm_area(struct vm_struct area, pgprot_t prot, struct page *pages) · 715 unmodified lines
2014	}	2040	}
2015	EXPORT_SYMBOL_GPL(map_vm_area);	2041	EXPORT_SYMBOL_GPL(map_vm_area);
2016		2042
2017	static void setup_vmalloc_vm(struct vm_struct vm, struct vmap_area va,	2043	static inline void setup_vmalloc_vm_locked(struct vm_struct *vm,
2018	unsigned long flags, const void *caller)	2044	struct vmap_area va, unsigned long flags, const void caller)
2019	{	2045	{
2020	spin_lock(&vmap_area_lock);
2021	vm->flags = flags;	2046	vm->flags = flags;
2022	vm->addr = (void *)va->va_start;	2047	vm->addr = (void *)va->va_start;
2023	vm->size = va->va_end - va->va_start;	2048	vm->size = va->va_end - va->va_start;
2024	vm->caller = caller;	2049	vm->caller = caller;
2025	va->vm = vm;	2050	va->vm = vm;
		2051	}
		2052
		2053	static void setup_vmalloc_vm(struct vm_struct vm, struct vmap_area va,
		2054	unsigned long flags, const void *caller)
		2055	{
		2056	spin_lock(&vmap_area_lock);
		2057	setup_vmalloc_vm_locked(vm, va, flags, caller);
2026	spin_unlock(&vmap_area_lock);	2058	spin_unlock(&vmap_area_lock);
2027	}	2059	}
2028		2060
@@ -2068,6 +2100,22 @@ static struct vm_struct *__get_vm_area_node(unsigned long size, · 39 unmodified lines
2068		2100
2069	setup_vmalloc_vm(area, va, flags, caller);	2101	setup_vmalloc_vm(area, va, flags, caller);
2070		2102
		2103	/*
		2104	* For KASAN, if we are in vmalloc space, we need to cover the shadow
		2105	* area with real memory. If we come here through VM_ALLOC, this is
		2106	* done by a higher level function that has access to the true size,
		2107	* which might not be a full page.
		2108	*
		2109	* We assume module space comes via VM_ALLOC path.
		2110	*/
		2111	if (is_vmalloc_addr(area->addr) && !(area->flags & VM_ALLOC)) {
		2112	if (kasan_populate_vmalloc(area->size, area)) {
		2113	unmap_vmap_area(va);
		2114	kfree(area);
		2115	return NULL;
		2116	}
		2117	}
		2118
2071	return area;	2119	return area;
2072	}	2120	}
2073		2121
@@ -2245,6 +2293,9 @@ static void __vunmap(const void *addr, int deallocate_pages) · 171 unmodified lines
2245	debug_check_no_locks_freed(area->addr, get_vm_area_size(area));	2293	debug_check_no_locks_freed(area->addr, get_vm_area_size(area));
2246	debug_check_no_obj_freed(area->addr, get_vm_area_size(area));	2294	debug_check_no_obj_freed(area->addr, get_vm_area_size(area));
2247		2295
		2296	if (area->flags & VM_KASAN)
		2297	kasan_poison_vmalloc(area->addr, area->size);
		2298
2248	vm_remove_mappings(area, deallocate_pages);	2299	vm_remove_mappings(area, deallocate_pages);
2249		2300
2250	if (deallocate_pages) {	2301	if (deallocate_pages) {
@@ -2440,7 +2491,7 @@ static void __vmalloc_area_node(struct vm_struct area, gfp_t gfp_mask, · 189 unmodified lines
2440	goto fail;	2491	goto fail;
2441	}	2492	}
2442	area->pages[i] = page;	2493	area->pages[i] = page;
2443	if (gfpflags_allow_blocking(gfp_mask\|highmem_mask))	2494	if (gfpflags_allow_blocking(gfp_mask))
2444	cond_resched();	2495	cond_resched();
2445	}	2496	}
2446	atomic_long_add(area->nr_pages, &nr_vmalloc_pages);	2497	atomic_long_add(area->nr_pages, &nr_vmalloc_pages);
@@ -2497,6 +2548,11 @@ void *__vmalloc_node_range(unsigned long size, unsigned long align, · 50 unmodified lines
2497	if (!addr)	2548	if (!addr)
2498	return NULL;	2549	return NULL;
2499		2550
		2551	if (is_vmalloc_or_module_addr(area->addr)) {
		2552	if (kasan_populate_vmalloc(real_size, area))
		2553	return NULL;
		2554	}
		2555
2500	/*	2556	/*
2501	* In this function, newly allocated vm_struct has VM_UNINITIALIZED	2557	* In this function, newly allocated vm_struct has VM_UNINITIALIZED
2502	* flag. It means that vm_struct is not fully initialized.	2558	* flag. It means that vm_struct is not fully initialized.
@@ -3282,7 +3338,7 @@ struct vm_struct *pcpu_get_vm_areas(const unsigned long offsets, · 779 unmodified lines
3282	goto err_free;	3338	goto err_free;
3283	}	3339	}
3284	retry:	3340	retry:
3285	spin_lock(&vmap_area_lock);	3341	spin_lock(&free_vmap_area_lock);
3286		3342
3287	/* start scanning - we scan from the top, begin with the last area */	3343	/* start scanning - we scan from the top, begin with the last area */
3288	area = term_area = last_area;	3344	area = term_area = last_area;
@@ -3364,29 +3420,44 @@ retry: · 75 unmodified lines
3364	va = vas[area];	3420	va = vas[area];
3365	va->va_start = start;	3421	va->va_start = start;
3366	va->va_end = start + size;	3422	va->va_end = start + size;
3367
3368	insert_vmap_area(va, &vmap_area_root, &vmap_area_list);
3369	}	3423	}
3370		3424
3371	spin_unlock(&vmap_area_lock);	3425	spin_unlock(&free_vmap_area_lock);
3372		3426
3373	/* insert all vm's */	3427	/* insert all vm's */
3374	for (area = 0; area < nr_vms; area++)	3428	spin_lock(&vmap_area_lock);
3375	setup_vmalloc_vm(vms[area], vas[area], VM_ALLOC,	3429	for (area = 0; area < nr_vms; area++) {
		3430	insert_vmap_area(vas[area], &vmap_area_root, &vmap_area_list);
		3431
		3432	setup_vmalloc_vm_locked(vms[area], vas[area], VM_ALLOC,
3376	pcpu_get_vm_areas);	3433	pcpu_get_vm_areas);
		3434	}
		3435	spin_unlock(&vmap_area_lock);
		3436
		3437	/* populate the shadow space outside of the lock */
		3438	for (area = 0; area < nr_vms; area++) {
		3439	/* assume success here */
		3440	kasan_populate_vmalloc(sizes[area], vms[area]);
		3441	}
3377		3442
3378	kfree(vas);	3443	kfree(vas);
3379	return vms;	3444	return vms;
3380		3445
3381	recovery:	3446	recovery:
3382	/* Remove previously inserted areas. */	3447	/*
		3448	* Remove previously allocated areas. There is no
		3449	* need in removing these areas from the busy tree,
		3450	* because they are inserted only on the final step
		3451	* and when pcpu_get_vm_areas() is success.
		3452	*/
3383	while (area--) {	3453	while (area--) {
3384	__free_vmap_area(vas[area]);	3454	merge_or_add_vmap_area(vas[area], &free_vmap_area_root,
		3455	&free_vmap_area_list);
3385	vas[area] = NULL;	3456	vas[area] = NULL;
3386	}	3457	}
3387		3458
3388	overflow:	3459	overflow:
3389	spin_unlock(&vmap_area_lock);	3460	spin_unlock(&free_vmap_area_lock);
3390	if (!purged) {	3461	if (!purged) {
3391	purge_vmap_area_lazy();	3462	purge_vmap_area_lazy();
3392	purged = true;	3463	purged = true;
@@ -3437,9 +3508,12 @@ void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms) · 44 unmodified lines
3437		3508
3438	#ifdef CONFIG_PROC_FS	3509	#ifdef CONFIG_PROC_FS
3439	static void s_start(struct seq_file m, loff_t *pos)	3510	static void s_start(struct seq_file m, loff_t *pos)
		3511	__acquires(&vmap_purge_lock)
3440	__acquires(&vmap_area_lock)	3512	__acquires(&vmap_area_lock)
3441	{	3513	{
		3514	mutex_lock(&vmap_purge_lock);
3442	spin_lock(&vmap_area_lock);	3515	spin_lock(&vmap_area_lock);
		3516
3443	return seq_list_start(&vmap_area_list, *pos);	3517	return seq_list_start(&vmap_area_list, *pos);
3444	}	3518	}
3445		3519
@@ -3449,8 +3523,10 @@ static void s_next(struct seq_file m, void p, loff_t pos) · 3 unmodified lines
3449	}	3523	}
3450		3524
3451	static void s_stop(struct seq_file m, void p)	3525	static void s_stop(struct seq_file m, void p)
		3526	__releases(&vmap_purge_lock)
3452	__releases(&vmap_area_lock)	3527	__releases(&vmap_area_lock)
3453	{	3528	{
		3529	mutex_unlock(&vmap_purge_lock);
3454	spin_unlock(&vmap_area_lock);	3530	spin_unlock(&vmap_area_lock);
3455	}	3531	}
3456		3532

mm/vmscan.c+327−335

@@ -79,6 +79,13 @@ struct scan_control { · 78 unmodified lines
79	*/	79	*/
80	struct mem_cgroup *target_mem_cgroup;	80	struct mem_cgroup *target_mem_cgroup;
81		81
		82	/* Can active pages be deactivated as part of reclaim? */
		83	#define DEACTIVATE_ANON 1
		84	#define DEACTIVATE_FILE 2
		85	unsigned int may_deactivate:2;
		86	unsigned int force_deactivate:1;
		87	unsigned int skipped_deactivate:1;
		88
82	/* Writepage batching in laptop mode; RECLAIM_WRITE */	89	/* Writepage batching in laptop mode; RECLAIM_WRITE */
83	unsigned int may_writepage:1;	90	unsigned int may_writepage:1;
84		91
@@ -101,6 +108,12 @@ struct scan_control { · 16 unmodified lines
101	/* One of the zones is ready for compaction */	108	/* One of the zones is ready for compaction */
102	unsigned int compaction_ready:1;	109	unsigned int compaction_ready:1;
103		110
		111	/* There is easily reclaimable cold cache in the current node */
		112	unsigned int cache_trim_mode:1;
		113
		114	/* The file pages on the current node are dangerously low */
		115	unsigned int file_is_tiny:1;
		116
104	/* Allocation order */	117	/* Allocation order */
105	s8 order;	118	s8 order;
106		119
@@ -239,13 +252,13 @@ static void unregister_memcg_shrinker(struct shrinker *shrinker) · 132 unmodified lines
239	up_write(&shrinker_rwsem);	252	up_write(&shrinker_rwsem);
240	}	253	}
241		254
242	static bool global_reclaim(struct scan_control *sc)	255	static bool cgroup_reclaim(struct scan_control *sc)
243	{	256	{
244	return !sc->target_mem_cgroup;	257	return sc->target_mem_cgroup;
245	}	258	}
246		259
247	/**	260	/**
248	* sane_reclaim - is the usual dirty throttling mechanism operational?	261	* writeback_throttling_sane - is the usual dirty throttling mechanism available?
249	* @sc: scan_control in question	262	* @sc: scan_control in question
250	*	263	*
251	* The normal page dirty throttling mechanism in balance_dirty_pages() is	264	* The normal page dirty throttling mechanism in balance_dirty_pages() is
@@ -257,11 +270,9 @@ static bool global_reclaim(struct scan_control *sc) · 5 unmodified lines
257	* This function tests whether the vmscan currently in progress can assume	270	* This function tests whether the vmscan currently in progress can assume
258	* that the normal dirty throttling mechanism is operational.	271	* that the normal dirty throttling mechanism is operational.
259	*/	272	*/
260	static bool sane_reclaim(struct scan_control *sc)	273	static bool writeback_throttling_sane(struct scan_control *sc)
261	{	274	{
262	struct mem_cgroup *memcg = sc->target_mem_cgroup;	275	if (!cgroup_reclaim(sc))
263
264	if (!memcg)
265	return true;	276	return true;
266	#ifdef CONFIG_CGROUP_WRITEBACK	277	#ifdef CONFIG_CGROUP_WRITEBACK
267	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))	278	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
@@ -269,29 +280,6 @@ static bool sane_reclaim(struct scan_control *sc) · 1 unmodified line
269	#endif	280	#endif
270	return false;	281	return false;
271	}	282	}
272
273	static void set_memcg_congestion(pg_data_t *pgdat,
274	struct mem_cgroup *memcg,
275	bool congested)
276	{
277	struct mem_cgroup_per_node *mn;
278
279	if (!memcg)
280	return;
281
282	mn = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
283	WRITE_ONCE(mn->congested, congested);
284	}
285
286	static bool memcg_congested(pg_data_t *pgdat,
287	struct mem_cgroup *memcg)
288	{
289	struct mem_cgroup_per_node *mn;
290
291	mn = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
292	return READ_ONCE(mn->congested);
293
294	}
295	#else	283	#else
296	static int prealloc_memcg_shrinker(struct shrinker *shrinker)	284	static int prealloc_memcg_shrinker(struct shrinker *shrinker)
297	{	285	{
@@ -302,27 +290,15 @@ static void unregister_memcg_shrinker(struct shrinker *shrinker) · 4 unmodified lines
302	{	290	{
303	}	291	}
304		292
305	static bool global_reclaim(struct scan_control *sc)	293	static bool cgroup_reclaim(struct scan_control *sc)
306	{	294	{
307	return true;	295	return false;
308	}	296	}
309		297
310	static bool sane_reclaim(struct scan_control *sc)	298	static bool writeback_throttling_sane(struct scan_control *sc)
311	{	299	{
312	return true;	300	return true;
313	}	301	}
314
315	static inline void set_memcg_congestion(struct pglist_data *pgdat,
316	struct mem_cgroup *memcg, bool congested)
317	{
318	}
319
320	static inline bool memcg_congested(struct pglist_data *pgdat,
321	struct mem_cgroup *memcg)
322	{
323	return false;
324
325	}
326	#endif	302	#endif
327		303
328	/*	304	/*
@@ -351,32 +327,21 @@ unsigned long zone_reclaimable_pages(struct zone *zone) · 22 unmodified lines
351	*/	327	*/
352	unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx)	328	unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx)
353	{	329	{
354	unsigned long lru_size = 0;	330	unsigned long size = 0;
355	int zid;	331	int zid;
356		332
357	if (!mem_cgroup_disabled()) {	333	for (zid = 0; zid <= zone_idx && zid < MAX_NR_ZONES; zid++) {
358	for (zid = 0; zid < MAX_NR_ZONES; zid++)
359	lru_size += mem_cgroup_get_zone_lru_size(lruvec, lru, zid);
360	} else
361	lru_size = node_page_state(lruvec_pgdat(lruvec), NR_LRU_BASE + lru);
362
363	for (zid = zone_idx + 1; zid < MAX_NR_ZONES; zid++) {
364	struct zone *zone = &lruvec_pgdat(lruvec)->node_zones[zid];	334	struct zone *zone = &lruvec_pgdat(lruvec)->node_zones[zid];
365	unsigned long size;
366		335
367	if (!managed_zone(zone))	336	if (!managed_zone(zone))
368	continue;	337	continue;
369		338
370	if (!mem_cgroup_disabled())	339	if (!mem_cgroup_disabled())
371	size = mem_cgroup_get_zone_lru_size(lruvec, lru, zid);	340	size += mem_cgroup_get_zone_lru_size(lruvec, lru, zid);
372	else	341	else
373	size = zone_page_state(&lruvec_pgdat(lruvec)->node_zones[zid],	342	size += zone_page_state(zone, NR_ZONE_LRU_BASE + lru);
374	NR_ZONE_LRU_BASE + lru);
375	lru_size -= min(size, lru_size);
376	}	343	}
377		344	return size;
378	return lru_size;
379
380	}	345	}
381		346
382	/*	347	/*
@@ -775,7 +740,7 @@ static inline int is_page_cache_freeable(struct page *page) · 392 unmodified lines
775	return page_count(page) - page_has_private(page) == 1 + page_cache_pins;	740	return page_count(page) - page_has_private(page) == 1 + page_cache_pins;
776	}	741	}
777		742
778	static int may_write_to_inode(struct inode inode, struct scan_control sc)	743	static int may_write_to_inode(struct inode *inode)
779	{	744	{
780	if (current->flags & PF_SWAPWRITE)	745	if (current->flags & PF_SWAPWRITE)
781	return 1;	746	return 1;
@@ -823,8 +788,7 @@ typedef enum { · 41 unmodified lines
823	* pageout is called by shrink_page_list() for each dirty page.	788	* pageout is called by shrink_page_list() for each dirty page.
824	* Calls ->writepage().	789	* Calls ->writepage().
825	*/	790	*/
826	static pageout_t pageout(struct page page, struct address_space mapping,	791	static pageout_t pageout(struct page page, struct address_space mapping)
827	struct scan_control *sc)
828	{	792	{
829	/*	793	/*
830	* If the page is dirty, only perform writeback if that write	794	* If the page is dirty, only perform writeback if that write
@@ -860,7 +824,7 @@ static pageout_t pageout(struct page page, struct address_space mapping, · 29 unmodified lines
860	}	824	}
861	if (mapping->a_ops->writepage == NULL)	825	if (mapping->a_ops->writepage == NULL)
862	return PAGE_ACTIVATE;	826	return PAGE_ACTIVATE;
863	if (!may_write_to_inode(mapping->host, sc))	827	if (!may_write_to_inode(mapping->host))
864	return PAGE_KEEP;	828	return PAGE_KEEP;
865		829
866	if (clear_page_dirty_for_io(page)) {	830	if (clear_page_dirty_for_io(page)) {
@@ -899,7 +863,7 @@ static pageout_t pageout(struct page page, struct address_space mapping, · 32 unmodified lines
899	* gets returned with a refcount of 0.	863	* gets returned with a refcount of 0.
900	*/	864	*/
901	static int __remove_mapping(struct address_space mapping, struct page page,	865	static int __remove_mapping(struct address_space mapping, struct page page,
902	bool reclaimed)	866	bool reclaimed, struct mem_cgroup *target_memcg)
903	{	867	{
904	unsigned long flags;	868	unsigned long flags;
905	int refcount;	869	int refcount;
@@ -971,7 +935,7 @@ static int __remove_mapping(struct address_space mapping, struct page page, · 65 unmodified lines
971	*/	935	*/
972	if (reclaimed && page_is_file_cache(page) &&	936	if (reclaimed && page_is_file_cache(page) &&
973	!mapping_exiting(mapping) && !dax_mapping(mapping))	937	!mapping_exiting(mapping) && !dax_mapping(mapping))
974	shadow = workingset_eviction(page);	938	shadow = workingset_eviction(page, target_memcg);
975	__delete_from_page_cache(page, shadow);	939	__delete_from_page_cache(page, shadow);
976	xa_unlock_irqrestore(&mapping->i_pages, flags);	940	xa_unlock_irqrestore(&mapping->i_pages, flags);
977		941
@@ -994,7 +958,7 @@ cannot_free: · 16 unmodified lines
994	*/	958	*/
995	int remove_mapping(struct address_space mapping, struct page page)	959	int remove_mapping(struct address_space mapping, struct page page)
996	{	960	{
997	if (__remove_mapping(mapping, page, false)) {	961	if (__remove_mapping(mapping, page, false, NULL)) {
998	/*	962	/*
999	* Unfreezing the refcount with 1 rather than 2 effectively	963	* Unfreezing the refcount with 1 rather than 2 effectively
1000	* drops the pagecache ref for us without requiring another	964	* drops the pagecache ref for us without requiring another
@@ -1239,7 +1203,7 @@ static unsigned long shrink_page_list(struct list_head *page_list, · 238 unmodified lines
1239	goto activate_locked;	1203	goto activate_locked;
1240		1204
1241	/* Case 2 above */	1205	/* Case 2 above */
1242	} else if (sane_reclaim(sc) \|\|	1206	} else if (writeback_throttling_sane(sc) \|\|
1243	!PageReclaim(page) \|\| !may_enter_fs) {	1207	!PageReclaim(page) \|\| !may_enter_fs) {
1244	/*	1208	/*
1245	* This is slightly racy - end_page_writeback()	1209	* This is slightly racy - end_page_writeback()
@@ -1394,7 +1358,7 @@ static unsigned long shrink_page_list(struct list_head *page_list, · 148 unmodified lines
1394	* starts and then write it out here.	1358	* starts and then write it out here.
1395	*/	1359	*/
1396	try_to_unmap_flush_dirty();	1360	try_to_unmap_flush_dirty();
1397	switch (pageout(page, mapping, sc)) {	1361	switch (pageout(page, mapping)) {
1398	case PAGE_KEEP:	1362	case PAGE_KEEP:
1399	goto keep_locked;	1363	goto keep_locked;
1400	case PAGE_ACTIVATE:	1364	case PAGE_ACTIVATE:
@@ -1472,7 +1436,8 @@ static unsigned long shrink_page_list(struct list_head *page_list, · 71 unmodified lines
1472		1436
1473	count_vm_event(PGLAZYFREED);	1437	count_vm_event(PGLAZYFREED);
1474	count_memcg_page_event(page, PGLAZYFREED);	1438	count_memcg_page_event(page, PGLAZYFREED);
1475	} else if (!mapping \|\| !__remove_mapping(mapping, page, true))	1439	} else if (!mapping \|\| !__remove_mapping(mapping, page, true,
		1440	sc->target_mem_cgroup))
1476	goto keep_locked;	1441	goto keep_locked;
1477		1442
1478	unlock_page(page);	1443	unlock_page(page);
@@ -1820,7 +1785,7 @@ int isolate_lru_page(struct page *page) · 341 unmodified lines
1820		1785
1821	/*	1786	/*
1822	* A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and	1787	* A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and
1823	* then get resheduled. When there are massive number of tasks doing page	1788	* then get rescheduled. When there are massive number of tasks doing page
1824	* allocation, such sleeping direct reclaimers may keep piling up on each CPU,	1789	* allocation, such sleeping direct reclaimers may keep piling up on each CPU,
1825	* the LRU list will go small and be scanned faster than necessary, leading to	1790	* the LRU list will go small and be scanned faster than necessary, leading to
1826	* unnecessary swapping, thrashing and OOM.	1791	* unnecessary swapping, thrashing and OOM.
@@ -1833,7 +1798,7 @@ static int too_many_isolated(struct pglist_data *pgdat, int file, · 6 unmodified lines
1833	if (current_is_kswapd())	1798	if (current_is_kswapd())
1834	return 0;	1799	return 0;
1835		1800
1836	if (!sane_reclaim(sc))	1801	if (!writeback_throttling_sane(sc))
1837	return 0;	1802	return 0;
1838		1803
1839	if (file) {	1804	if (file) {
@@ -1983,7 +1948,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, · 143 unmodified lines
1983	reclaim_stat->recent_scanned[file] += nr_taken;	1948	reclaim_stat->recent_scanned[file] += nr_taken;
1984		1949
1985	item = current_is_kswapd() ? PGSCAN_KSWAPD : PGSCAN_DIRECT;	1950	item = current_is_kswapd() ? PGSCAN_KSWAPD : PGSCAN_DIRECT;
1986	if (global_reclaim(sc))	1951	if (!cgroup_reclaim(sc))
1987	__count_vm_events(item, nr_scanned);	1952	__count_vm_events(item, nr_scanned);
1988	__count_memcg_events(lruvec_memcg(lruvec), item, nr_scanned);	1953	__count_memcg_events(lruvec_memcg(lruvec), item, nr_scanned);
1989	spin_unlock_irq(&pgdat->lru_lock);	1954	spin_unlock_irq(&pgdat->lru_lock);
@@ -1997,7 +1962,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, · 7 unmodified lines
1997	spin_lock_irq(&pgdat->lru_lock);	1962	spin_lock_irq(&pgdat->lru_lock);
1998		1963
1999	item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT;	1964	item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT;
2000	if (global_reclaim(sc))	1965	if (!cgroup_reclaim(sc))
2001	__count_vm_events(item, nr_reclaimed);	1966	__count_vm_events(item, nr_reclaimed);
2002	__count_memcg_events(lruvec_memcg(lruvec), item, nr_reclaimed);	1967	__count_memcg_events(lruvec_memcg(lruvec), item, nr_reclaimed);
2003	reclaim_stat->recent_rotated[0] += stat.nr_activate[0];	1968	reclaim_stat->recent_rotated[0] += stat.nr_activate[0];
@@ -2199,6 +2164,20 @@ unsigned long reclaim_pages(struct list_head *page_list) · 195 unmodified lines
2199	return nr_reclaimed;	2164	return nr_reclaimed;
2200	}	2165	}
2201		2166
		2167	static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
		2168	struct lruvec lruvec, struct scan_control sc)
		2169	{
		2170	if (is_active_lru(lru)) {
		2171	if (sc->may_deactivate & (1 << is_file_lru(lru)))
		2172	shrink_active_list(nr_to_scan, lruvec, sc, lru);
		2173	else
		2174	sc->skipped_deactivate = 1;
		2175	return 0;
		2176	}
		2177
		2178	return shrink_inactive_list(nr_to_scan, lruvec, sc, lru);
		2179	}
		2180
2202	/*	2181	/*
2203	* The inactive anon list should be small enough that the VM never has	2182	* The inactive anon list should be small enough that the VM never has
2204	* to do too much work.	2183	* to do too much work.
@@ -2227,64 +2206,25 @@ unsigned long reclaim_pages(struct list_head *page_list) · 22 unmodified lines
2227	* 1TB 101 10GB	2206	* 1TB 101 10GB
2228	* 10TB 320 32GB	2207	* 10TB 320 32GB
2229	*/	2208	*/
2230	static bool inactive_list_is_low(struct lruvec *lruvec, bool file,	2209	static bool inactive_is_low(struct lruvec *lruvec, enum lru_list inactive_lru)
2231	struct scan_control *sc, bool trace)
2232	{	2210	{
2233	enum lru_list active_lru = file * LRU_FILE + LRU_ACTIVE;	2211	enum lru_list active_lru = inactive_lru + LRU_ACTIVE;
2234	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2235	enum lru_list inactive_lru = file * LRU_FILE;
2236	unsigned long inactive, active;	2212	unsigned long inactive, active;
2237	unsigned long inactive_ratio;	2213	unsigned long inactive_ratio;
2238	unsigned long refaults;
2239	unsigned long gb;	2214	unsigned long gb;
2240		2215
2241	/*	2216	inactive = lruvec_page_state(lruvec, NR_LRU_BASE + inactive_lru);
2242	* If we don't have swap space, anonymous page deactivation	2217	active = lruvec_page_state(lruvec, NR_LRU_BASE + active_lru);
2243	* is pointless.
2244	*/
2245	if (!file && !total_swap_pages)
2246	return false;
2247
2248	inactive = lruvec_lru_size(lruvec, inactive_lru, sc->reclaim_idx);
2249	active = lruvec_lru_size(lruvec, active_lru, sc->reclaim_idx);
2250
2251	/*
2252	* When refaults are being observed, it means a new workingset
2253	* is being established. Disable active list protection to get
2254	* rid of the stale workingset quickly.
2255	*/
2256	refaults = lruvec_page_state_local(lruvec, WORKINGSET_ACTIVATE);
2257	if (file && lruvec->refaults != refaults) {
2258	inactive_ratio = 0;
2259	} else {
2260	gb = (inactive + active) >> (30 - PAGE_SHIFT);
2261	if (gb)
2262	inactive_ratio = int_sqrt(10 * gb);
2263	else
2264	inactive_ratio = 1;
2265	}
2266		2218
2267	if (trace)	2219	gb = (inactive + active) >> (30 - PAGE_SHIFT);
2268	trace_mm_vmscan_inactive_list_is_low(pgdat->node_id, sc->reclaim_idx,	2220	if (gb)
2269	lruvec_lru_size(lruvec, inactive_lru, MAX_NR_ZONES), inactive,	2221	inactive_ratio = int_sqrt(10 * gb);
2270	lruvec_lru_size(lruvec, active_lru, MAX_NR_ZONES), active,	2222	else
2271	inactive_ratio, file);	2223	inactive_ratio = 1;
2272		2224
2273	return inactive * inactive_ratio < active;	2225	return inactive * inactive_ratio < active;
2274	}	2226	}
2275		2227
2276	static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
2277	struct lruvec lruvec, struct scan_control sc)
2278	{
2279	if (is_active_lru(lru)) {
2280	if (inactive_list_is_low(lruvec, is_file_lru(lru), sc, true))
2281	shrink_active_list(nr_to_scan, lruvec, sc, lru);
2282	return 0;
2283	}
2284
2285	return shrink_inactive_list(nr_to_scan, lruvec, sc, lru);
2286	}
2287
2288	enum scan_balance {	2228	enum scan_balance {
2289	SCAN_EQUAL,	2229	SCAN_EQUAL,
2290	SCAN_FRACT,	2230	SCAN_FRACT,
@@ -2301,10 +2241,10 @@ enum scan_balance { · 10 unmodified lines
2301	* nr[0] = anon inactive pages to scan; nr[1] = anon active pages to scan	2241	* nr[0] = anon inactive pages to scan; nr[1] = anon active pages to scan
2302	* nr[2] = file inactive pages to scan; nr[3] = file active pages to scan	2242	* nr[2] = file inactive pages to scan; nr[3] = file active pages to scan
2303	*/	2243	*/
2304	static void get_scan_count(struct lruvec lruvec, struct mem_cgroup memcg,	2244	static void get_scan_count(struct lruvec lruvec, struct scan_control sc,
2305	struct scan_control sc, unsigned long nr,	2245	unsigned long *nr)
2306	unsigned long *lru_pages)
2307	{	2246	{
		2247	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
2308	int swappiness = mem_cgroup_swappiness(memcg);	2248	int swappiness = mem_cgroup_swappiness(memcg);
2309	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;	2249	struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
2310	u64 fraction[2];	2250	u64 fraction[2];
@@ -2329,7 +2269,7 @@ static void get_scan_count(struct lruvec lruvec, struct mem_cgroup memcg, · 18 unmodified lines
2329	* using the memory controller's swap limit feature would be	2269	* using the memory controller's swap limit feature would be
2330	* too expensive.	2270	* too expensive.
2331	*/	2271	*/
2332	if (!global_reclaim(sc) && !swappiness) {	2272	if (cgroup_reclaim(sc) && !swappiness) {
2333	scan_balance = SCAN_FILE;	2273	scan_balance = SCAN_FILE;
2334	goto out;	2274	goto out;
2335	}	2275	}
@@ -2345,58 +2285,18 @@ static void get_scan_count(struct lruvec lruvec, struct mem_cgroup memcg, · 9 unmodified lines
2345	}	2285	}
2346		2286
2347	/*	2287	/*
2348	* Prevent the reclaimer from falling into the cache trap: as	2288	* If the system is almost out of file pages, force-scan anon.
2349	* cache pages start out inactive, every cache fault will tip
2350	* the scan balance towards the file LRU. And as the file LRU
2351	* shrinks, so does the window for rotation from references.
2352	* This means we have a runaway feedback loop where a tiny
2353	* thrashing file LRU becomes infinitely more attractive than
2354	* anon pages. Try to detect this based on file LRU size.
2355	*/	2289	*/
2356	if (global_reclaim(sc)) {	2290	if (sc->file_is_tiny) {
2357	unsigned long pgdatfile;	2291	scan_balance = SCAN_ANON;
2358	unsigned long pgdatfree;	2292	goto out;
2359	int z;
2360	unsigned long total_high_wmark = 0;
2361
2362	pgdatfree = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES);
2363	pgdatfile = node_page_state(pgdat, NR_ACTIVE_FILE) +
2364	node_page_state(pgdat, NR_INACTIVE_FILE);
2365
2366	for (z = 0; z < MAX_NR_ZONES; z++) {
2367	struct zone *zone = &pgdat->node_zones[z];
2368	if (!managed_zone(zone))
2369	continue;
2370
2371	total_high_wmark += high_wmark_pages(zone);
2372	}
2373
2374	if (unlikely(pgdatfile + pgdatfree <= total_high_wmark)) {
2375	/*
2376	* Force SCAN_ANON if there are enough inactive
2377	* anonymous pages on the LRU in eligible zones.
2378	* Otherwise, the small LRU gets thrashed.
2379	*/
2380	if (!inactive_list_is_low(lruvec, false, sc, false) &&
2381	lruvec_lru_size(lruvec, LRU_INACTIVE_ANON, sc->reclaim_idx)
2382	>> sc->priority) {
2383	scan_balance = SCAN_ANON;
2384	goto out;
2385	}
2386	}
2387	}	2293	}
2388		2294
2389	/*	2295	/*
2390	* If there is enough inactive page cache, i.e. if the size of the	2296	* If there is enough inactive page cache, we do not reclaim
2391	* inactive list is greater than that of the active list and the	2297	* anything from the anonymous working right now.
2392	* inactive list actually has some pages to scan on this priority, we
2393	* do not reclaim anything from the anonymous working set right now.
2394	* Without the second condition we could end up never scanning an
2395	* lruvec even if it has plenty of old anonymous pages unless the
2396	* system is under heavy pressure.
2397	*/	2298	*/
2398	if (!inactive_list_is_low(lruvec, true, sc, false) &&	2299	if (sc->cache_trim_mode) {
2399	lruvec_lru_size(lruvec, LRU_INACTIVE_FILE, sc->reclaim_idx) >> sc->priority) {
2400	scan_balance = SCAN_FILE;	2300	scan_balance = SCAN_FILE;
2401	goto out;	2301	goto out;
2402	}	2302	}
@@ -2454,7 +2354,6 @@ static void get_scan_count(struct lruvec lruvec, struct mem_cgroup memcg, · 51 unmodified lines
2454	fraction[1] = fp;	2354	fraction[1] = fp;
2455	denominator = ap + fp + 1;	2355	denominator = ap + fp + 1;
2456	out:	2356	out:
2457	*lru_pages = 0;
2458	for_each_evictable_lru(lru) {	2357	for_each_evictable_lru(lru) {
2459	int file = is_file_lru(lru);	2358	int file = is_file_lru(lru);
2460	unsigned long lruvec_size;	2359	unsigned long lruvec_size;
@@ -2549,18 +2448,12 @@ out: · 88 unmodified lines
2549	BUG();	2448	BUG();
2550	}	2449	}
2551		2450
2552	*lru_pages += lruvec_size;
2553	nr[lru] = scan;	2451	nr[lru] = scan;
2554	}	2452	}
2555	}	2453	}
2556		2454
2557	/*	2455	static void shrink_lruvec(struct lruvec lruvec, struct scan_control sc)
2558	* This is a basic per-node page freer. Used by both kswapd and direct reclaim.
2559	*/
2560	static void shrink_node_memcg(struct pglist_data pgdat, struct mem_cgroup memcg,
2561	struct scan_control sc, unsigned long lru_pages)
2562	{	2456	{
2563	struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg);
2564	unsigned long nr[NR_LRU_LISTS];	2457	unsigned long nr[NR_LRU_LISTS];
2565	unsigned long targets[NR_LRU_LISTS];	2458	unsigned long targets[NR_LRU_LISTS];
2566	unsigned long nr_to_scan;	2459	unsigned long nr_to_scan;
@@ -2570,7 +2463,7 @@ static void shrink_node_memcg(struct pglist_data pgdat, struct mem_cgroup memc · 3 unmodified lines
2570	struct blk_plug plug;	2463	struct blk_plug plug;
2571	bool scan_adjusted;	2464	bool scan_adjusted;
2572		2465
2573	get_scan_count(lruvec, memcg, sc, nr, lru_pages);	2466	get_scan_count(lruvec, sc, nr);
2574		2467
2575	/* Record the original scan target for proportional adjustments later */	2468	/* Record the original scan target for proportional adjustments later */
2576	memcpy(targets, nr, sizeof(nr));	2469	memcpy(targets, nr, sizeof(nr));
@@ -2586,7 +2479,7 @@ static void shrink_node_memcg(struct pglist_data pgdat, struct mem_cgroup memc · 9 unmodified lines
2586	* abort proportional reclaim if either the file or anon lru has already	2479	* abort proportional reclaim if either the file or anon lru has already
2587	* dropped to zero at the first pass.	2480	* dropped to zero at the first pass.
2588	*/	2481	*/
2589	scan_adjusted = (global_reclaim(sc) && !current_is_kswapd() &&	2482	scan_adjusted = (!cgroup_reclaim(sc) && !current_is_kswapd() &&
2590	sc->priority == DEF_PRIORITY);	2483	sc->priority == DEF_PRIORITY);
2591		2484
2592	blk_start_plug(&plug);	2485	blk_start_plug(&plug);
@@ -2668,7 +2561,7 @@ static void shrink_node_memcg(struct pglist_data pgdat, struct mem_cgroup memc · 75 unmodified lines
2668	* Even if we did not try to evict anon pages at all, we want to	2561	* Even if we did not try to evict anon pages at all, we want to
2669	* rebalance the anon lru active/inactive ratio.	2562	* rebalance the anon lru active/inactive ratio.
2670	*/	2563	*/
2671	if (inactive_list_is_low(lruvec, false, sc, true))	2564	if (total_swap_pages && inactive_is_low(lruvec, LRU_INACTIVE_ANON))
2672	shrink_active_list(SWAP_CLUSTER_MAX, lruvec,	2565	shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
2673	sc, LRU_ACTIVE_ANON);	2566	sc, LRU_ACTIVE_ANON);
2674	}	2567	}
@@ -2744,156 +2637,234 @@ static inline bool should_continue_reclaim(struct pglist_data *pgdat, · 69 unmodified lines
2744	return inactive_lru_pages > pages_for_compaction;	2637	return inactive_lru_pages > pages_for_compaction;
2745	}	2638	}
2746		2639
2747	static bool pgdat_memcg_congested(pg_data_t pgdat, struct mem_cgroup memcg)	2640	static void shrink_node_memcgs(pg_data_t pgdat, struct scan_control sc)
2748	{	2641	{
2749	return test_bit(PGDAT_CONGESTED, &pgdat->flags) \|\|	2642	struct mem_cgroup *target_memcg = sc->target_mem_cgroup;
2750	(memcg && memcg_congested(pgdat, memcg));	2643	struct mem_cgroup *memcg;
		2644
		2645	memcg = mem_cgroup_iter(target_memcg, NULL, NULL);
		2646	do {
		2647	struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
		2648	unsigned long reclaimed;
		2649	unsigned long scanned;
		2650
		2651	switch (mem_cgroup_protected(target_memcg, memcg)) {
		2652	case MEMCG_PROT_MIN:
		2653	/*
		2654	* Hard protection.
		2655	* If there is no reclaimable memory, OOM.
		2656	*/
		2657	continue;
		2658	case MEMCG_PROT_LOW:
		2659	/*
		2660	* Soft protection.
		2661	* Respect the protection only as long as
		2662	* there is an unprotected supply
		2663	* of reclaimable memory from other cgroups.
		2664	*/
		2665	if (!sc->memcg_low_reclaim) {
		2666	sc->memcg_low_skipped = 1;
		2667	continue;
		2668	}
		2669	memcg_memory_event(memcg, MEMCG_LOW);
		2670	break;
		2671	case MEMCG_PROT_NONE:
		2672	/*
		2673	* All protection thresholds breached. We may
		2674	* still choose to vary the scan pressure
		2675	* applied based on by how much the cgroup in
		2676	* question has exceeded its protection
		2677	* thresholds (see get_scan_count).
		2678	*/
		2679	break;
		2680	}
		2681
		2682	reclaimed = sc->nr_reclaimed;
		2683	scanned = sc->nr_scanned;
		2684
		2685	shrink_lruvec(lruvec, sc);
		2686
		2687	shrink_slab(sc->gfp_mask, pgdat->node_id, memcg,
		2688	sc->priority);
		2689
		2690	/* Record the group's reclaim efficiency */
		2691	vmpressure(sc->gfp_mask, memcg, false,
		2692	sc->nr_scanned - scanned,
		2693	sc->nr_reclaimed - reclaimed);
		2694
		2695	} while ((memcg = mem_cgroup_iter(target_memcg, memcg, NULL)));
2751	}	2696	}
2752		2697
2753	static bool shrink_node(pg_data_t pgdat, struct scan_control sc)	2698	static bool shrink_node(pg_data_t pgdat, struct scan_control sc)
2754	{	2699	{
2755	struct reclaim_state *reclaim_state = current->reclaim_state;	2700	struct reclaim_state *reclaim_state = current->reclaim_state;
2756	unsigned long nr_reclaimed, nr_scanned;	2701	unsigned long nr_reclaimed, nr_scanned;
		2702	struct lruvec *target_lruvec;
2757	bool reclaimable = false;	2703	bool reclaimable = false;
		2704	unsigned long file;
2758		2705
2759	do {	2706	target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat);
2760	struct mem_cgroup *root = sc->target_mem_cgroup;
2761	unsigned long node_lru_pages = 0;
2762	struct mem_cgroup *memcg;
2763		2707
2764	memset(&sc->nr, 0, sizeof(sc->nr));	2708	again:
		2709	memset(&sc->nr, 0, sizeof(sc->nr));
2765		2710
2766	nr_reclaimed = sc->nr_reclaimed;	2711	nr_reclaimed = sc->nr_reclaimed;
2767	nr_scanned = sc->nr_scanned;	2712	nr_scanned = sc->nr_scanned;
2768		2713
2769	memcg = mem_cgroup_iter(root, NULL, NULL);	2714	/*
2770	do {	2715	* Target desirable inactive:active list ratios for the anon
2771	unsigned long lru_pages;	2716	* and file LRU lists.
2772	unsigned long reclaimed;	2717	*/
2773	unsigned long scanned;	2718	if (!sc->force_deactivate) {
		2719	unsigned long refaults;
2774		2720
2775	switch (mem_cgroup_protected(root, memcg)) {	2721	if (inactive_is_low(target_lruvec, LRU_INACTIVE_ANON))
2776	case MEMCG_PROT_MIN:	2722	sc->may_deactivate \|= DEACTIVATE_ANON;
2777	/*	2723	else
2778	* Hard protection.	2724	sc->may_deactivate &= ~DEACTIVATE_ANON;
2779	* If there is no reclaimable memory, OOM.
2780	*/
2781	continue;
2782	case MEMCG_PROT_LOW:
2783	/*
2784	* Soft protection.
2785	* Respect the protection only as long as
2786	* there is an unprotected supply
2787	* of reclaimable memory from other cgroups.
2788	*/
2789	if (!sc->memcg_low_reclaim) {
2790	sc->memcg_low_skipped = 1;
2791	continue;
2792	}
2793	memcg_memory_event(memcg, MEMCG_LOW);
2794	break;
2795	case MEMCG_PROT_NONE:
2796	/*
2797	* All protection thresholds breached. We may
2798	* still choose to vary the scan pressure
2799	* applied based on by how much the cgroup in
2800	* question has exceeded its protection
2801	* thresholds (see get_scan_count).
2802	*/
2803	break;
2804	}
2805		2725
2806	reclaimed = sc->nr_reclaimed;	2726	/*
2807	scanned = sc->nr_scanned;	2727	* When refaults are being observed, it means a new
2808	shrink_node_memcg(pgdat, memcg, sc, &lru_pages);	2728	* workingset is being established. Deactivate to get
2809	node_lru_pages += lru_pages;	2729	* rid of any stale active pages quickly.
		2730	*/
		2731	refaults = lruvec_page_state(target_lruvec,
		2732	WORKINGSET_ACTIVATE);
		2733	if (refaults != target_lruvec->refaults \|\|
		2734	inactive_is_low(target_lruvec, LRU_INACTIVE_FILE))
		2735	sc->may_deactivate \|= DEACTIVATE_FILE;
		2736	else
		2737	sc->may_deactivate &= ~DEACTIVATE_FILE;
		2738	} else
		2739	sc->may_deactivate = DEACTIVATE_ANON \| DEACTIVATE_FILE;
2810		2740
2811	shrink_slab(sc->gfp_mask, pgdat->node_id, memcg,	2741	/*
2812	sc->priority);	2742	* If we have plenty of inactive file pages that aren't
		2743	* thrashing, try to reclaim those first before touching
		2744	* anonymous pages.
		2745	*/
		2746	file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE);
		2747	if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE))
		2748	sc->cache_trim_mode = 1;
		2749	else
		2750	sc->cache_trim_mode = 0;
		2751
		2752	/*
		2753	* Prevent the reclaimer from falling into the cache trap: as
		2754	* cache pages start out inactive, every cache fault will tip
		2755	* the scan balance towards the file LRU. And as the file LRU
		2756	* shrinks, so does the window for rotation from references.
		2757	* This means we have a runaway feedback loop where a tiny
		2758	* thrashing file LRU becomes infinitely more attractive than
		2759	* anon pages. Try to detect this based on file LRU size.
		2760	*/
		2761	if (!cgroup_reclaim(sc)) {
		2762	unsigned long total_high_wmark = 0;
		2763	unsigned long free, anon;
		2764	int z;
2813		2765
2814	/* Record the group's reclaim efficiency */	2766	free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES);
2815	vmpressure(sc->gfp_mask, memcg, false,	2767	file = node_page_state(pgdat, NR_ACTIVE_FILE) +
2816	sc->nr_scanned - scanned,	2768	node_page_state(pgdat, NR_INACTIVE_FILE);
2817	sc->nr_reclaimed - reclaimed);
2818		2769
2819	} while ((memcg = mem_cgroup_iter(root, memcg, NULL)));	2770	for (z = 0; z < MAX_NR_ZONES; z++) {
		2771	struct zone *zone = &pgdat->node_zones[z];
		2772	if (!managed_zone(zone))
		2773	continue;
2820		2774
2821	if (reclaim_state) {	2775	total_high_wmark += high_wmark_pages(zone);
2822	sc->nr_reclaimed += reclaim_state->reclaimed_slab;
2823	reclaim_state->reclaimed_slab = 0;
2824	}	2776	}
2825		2777
2826	/* Record the subtree's reclaim efficiency */	2778	/*
2827	vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true,	2779	* Consider anon: if that's low too, this isn't a
2828	sc->nr_scanned - nr_scanned,	2780	* runaway file reclaim problem, but rather just
2829	sc->nr_reclaimed - nr_reclaimed);	2781	* extreme pressure. Reclaim as per usual then.
		2782	*/
		2783	anon = node_page_state(pgdat, NR_INACTIVE_ANON);
2830		2784
2831	if (sc->nr_reclaimed - nr_reclaimed)	2785	sc->file_is_tiny =
2832	reclaimable = true;	2786	file + free <= total_high_wmark &&
		2787	!(sc->may_deactivate & DEACTIVATE_ANON) &&
		2788	anon >> sc->priority;
		2789	}
2833		2790
2834	if (current_is_kswapd()) {	2791	shrink_node_memcgs(pgdat, sc);
2835	/*
2836	* If reclaim is isolating dirty pages under writeback,
2837	* it implies that the long-lived page allocation rate
2838	* is exceeding the page laundering rate. Either the
2839	* global limits are not being effective at throttling
2840	* processes due to the page distribution throughout
2841	* zones or there is heavy usage of a slow backing
2842	* device. The only option is to throttle from reclaim
2843	* context which is not ideal as there is no guarantee
2844	* the dirtying process is throttled in the same way
2845	* balance_dirty_pages() manages.
2846	*
2847	* Once a node is flagged PGDAT_WRITEBACK, kswapd will
2848	* count the number of pages under pages flagged for
2849	* immediate reclaim and stall if any are encountered
2850	* in the nr_immediate check below.
2851	*/
2852	if (sc->nr.writeback && sc->nr.writeback == sc->nr.taken)
2853	set_bit(PGDAT_WRITEBACK, &pgdat->flags);
2854		2792
2855	/*	2793	if (reclaim_state) {
2856	* Tag a node as congested if all the dirty pages	2794	sc->nr_reclaimed += reclaim_state->reclaimed_slab;
2857	* scanned were backed by a congested BDI and	2795	reclaim_state->reclaimed_slab = 0;
2858	* wait_iff_congested will stall.	2796	}
2859	*/
2860	if (sc->nr.dirty && sc->nr.dirty == sc->nr.congested)
2861	set_bit(PGDAT_CONGESTED, &pgdat->flags);
2862		2797
2863	/* Allow kswapd to start writing pages during reclaim.*/	2798	/* Record the subtree's reclaim efficiency */
2864	if (sc->nr.unqueued_dirty == sc->nr.file_taken)	2799	vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true,
2865	set_bit(PGDAT_DIRTY, &pgdat->flags);	2800	sc->nr_scanned - nr_scanned,
		2801	sc->nr_reclaimed - nr_reclaimed);
2866		2802
2867	/*	2803	if (sc->nr_reclaimed - nr_reclaimed)
2868	* If kswapd scans pages marked marked for immediate	2804	reclaimable = true;
2869	* reclaim and under writeback (nr_immediate), it
2870	* implies that pages are cycling through the LRU
2871	* faster than they are written so also forcibly stall.
2872	*/
2873	if (sc->nr.immediate)
2874	congestion_wait(BLK_RW_ASYNC, HZ/10);
2875	}
2876		2805
		2806	if (current_is_kswapd()) {
2877	/*	2807	/*
2878	* Legacy memcg will stall in page writeback so avoid forcibly	2808	* If reclaim is isolating dirty pages under writeback,
2879	* stalling in wait_iff_congested().	2809	* it implies that the long-lived page allocation rate
		2810	* is exceeding the page laundering rate. Either the
		2811	* global limits are not being effective at throttling
		2812	* processes due to the page distribution throughout
		2813	* zones or there is heavy usage of a slow backing
		2814	* device. The only option is to throttle from reclaim
		2815	* context which is not ideal as there is no guarantee
		2816	* the dirtying process is throttled in the same way
		2817	* balance_dirty_pages() manages.
		2818	*
		2819	* Once a node is flagged PGDAT_WRITEBACK, kswapd will
		2820	* count the number of pages under pages flagged for
		2821	* immediate reclaim and stall if any are encountered
		2822	* in the nr_immediate check below.
2880	*/	2823	*/
2881	if (!global_reclaim(sc) && sane_reclaim(sc) &&	2824	if (sc->nr.writeback && sc->nr.writeback == sc->nr.taken)
2882	sc->nr.dirty && sc->nr.dirty == sc->nr.congested)	2825	set_bit(PGDAT_WRITEBACK, &pgdat->flags);
2883	set_memcg_congestion(pgdat, root, true);	2826
		2827	/* Allow kswapd to start writing pages during reclaim.*/
		2828	if (sc->nr.unqueued_dirty == sc->nr.file_taken)
		2829	set_bit(PGDAT_DIRTY, &pgdat->flags);
2884		2830
2885	/*	2831	/*
2886	* Stall direct reclaim for IO completions if underlying BDIs	2832	* If kswapd scans pages marked marked for immediate
2887	* and node is congested. Allow kswapd to continue until it	2833	* reclaim and under writeback (nr_immediate), it
2888	* starts encountering unqueued dirty pages or cycling through	2834	* implies that pages are cycling through the LRU
2889	* the LRU too quickly.	2835	* faster than they are written so also forcibly stall.
2890	*/	2836	*/
2891	if (!sc->hibernation_mode && !current_is_kswapd() &&	2837	if (sc->nr.immediate)
2892	current_may_throttle() && pgdat_memcg_congested(pgdat, root))	2838	congestion_wait(BLK_RW_ASYNC, HZ/10);
2893	wait_iff_congested(BLK_RW_ASYNC, HZ/10);	2839	}
2894		2840
2895	} while (should_continue_reclaim(pgdat, sc->nr_reclaimed - nr_reclaimed,	2841	/*
2896	sc));	2842	* Tag a node/memcg as congested if all the dirty pages
		2843	* scanned were backed by a congested BDI and
		2844	* wait_iff_congested will stall.
		2845	*
		2846	* Legacy memcg will stall in page writeback so avoid forcibly
		2847	* stalling in wait_iff_congested().
		2848	*/
		2849	if ((current_is_kswapd() \|\|
		2850	(cgroup_reclaim(sc) && writeback_throttling_sane(sc))) &&
		2851	sc->nr.dirty && sc->nr.dirty == sc->nr.congested)
		2852	set_bit(LRUVEC_CONGESTED, &target_lruvec->flags);
		2853
		2854	/*
		2855	* Stall direct reclaim for IO completions if underlying BDIs
		2856	* and node is congested. Allow kswapd to continue until it
		2857	* starts encountering unqueued dirty pages or cycling through
		2858	* the LRU too quickly.
		2859	*/
		2860	if (!current_is_kswapd() && current_may_throttle() &&
		2861	!sc->hibernation_mode &&
		2862	test_bit(LRUVEC_CONGESTED, &target_lruvec->flags))
		2863	wait_iff_congested(BLK_RW_ASYNC, HZ/10);
		2864
		2865	if (should_continue_reclaim(pgdat, sc->nr_reclaimed - nr_reclaimed,
		2866	sc))
		2867	goto again;
2897		2868
2898	/*	2869	/*
2899	* Kswapd gives up on balancing particular nodes after too	2870	* Kswapd gives up on balancing particular nodes after too
@@ -2973,7 +2944,7 @@ static void shrink_zones(struct zonelist zonelist, struct scan_control sc) · 73 unmodified lines
2973	* Take care memory controller reclaiming has small influence	2944	* Take care memory controller reclaiming has small influence
2974	* to global LRU.	2945	* to global LRU.
2975	*/	2946	*/
2976	if (global_reclaim(sc)) {	2947	if (!cgroup_reclaim(sc)) {
2977	if (!cpuset_zone_allowed(zone,	2948	if (!cpuset_zone_allowed(zone,
2978	GFP_KERNEL \| __GFP_HARDWALL))	2949	GFP_KERNEL \| __GFP_HARDWALL))
2979	continue;	2950	continue;
@@ -3032,19 +3003,14 @@ static void shrink_zones(struct zonelist zonelist, struct scan_control sc) · 52 unmodified lines
3032	sc->gfp_mask = orig_mask;	3003	sc->gfp_mask = orig_mask;
3033	}	3004	}
3034		3005
3035	static void snapshot_refaults(struct mem_cgroup root_memcg, pg_data_t pgdat)	3006	static void snapshot_refaults(struct mem_cgroup target_memcg, pg_data_t pgdat)
3036	{	3007	{
3037	struct mem_cgroup *memcg;	3008	struct lruvec *target_lruvec;
3038		3009	unsigned long refaults;
3039	memcg = mem_cgroup_iter(root_memcg, NULL, NULL);
3040	do {
3041	unsigned long refaults;
3042	struct lruvec *lruvec;
3043		3010
3044	lruvec = mem_cgroup_lruvec(pgdat, memcg);	3011	target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat);
3045	refaults = lruvec_page_state_local(lruvec, WORKINGSET_ACTIVATE);	3012	refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE);
3046	lruvec->refaults = refaults;	3013	target_lruvec->refaults = refaults;
3047	} while ((memcg = mem_cgroup_iter(root_memcg, memcg, NULL)));
3048	}	3014	}
3049		3015
3050	/*	3016	/*
@@ -3073,7 +3039,7 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist, · 22 unmodified lines
3073	retry:	3039	retry:
3074	delayacct_freepages_start();	3040	delayacct_freepages_start();
3075		3041
3076	if (global_reclaim(sc))	3042	if (!cgroup_reclaim(sc))
3077	__count_zid_vm_events(ALLOCSTALL, sc->reclaim_idx, 1);	3043	__count_zid_vm_events(ALLOCSTALL, sc->reclaim_idx, 1);
3078		3044
3079	do {	3045	do {
@@ -3102,8 +3068,16 @@ retry: · 22 unmodified lines
3102	if (zone->zone_pgdat == last_pgdat)	3068	if (zone->zone_pgdat == last_pgdat)
3103	continue;	3069	continue;
3104	last_pgdat = zone->zone_pgdat;	3070	last_pgdat = zone->zone_pgdat;
		3071
3105	snapshot_refaults(sc->target_mem_cgroup, zone->zone_pgdat);	3072	snapshot_refaults(sc->target_mem_cgroup, zone->zone_pgdat);
3106	set_memcg_congestion(last_pgdat, sc->target_mem_cgroup, false);	3073
		3074	if (cgroup_reclaim(sc)) {
		3075	struct lruvec *lruvec;
		3076
		3077	lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup,
		3078	zone->zone_pgdat);
		3079	clear_bit(LRUVEC_CONGESTED, &lruvec->flags);
		3080	}
3107	}	3081	}
3108		3082
3109	delayacct_freepages_end();	3083	delayacct_freepages_end();
@@ -3115,9 +3089,27 @@ retry: · 5 unmodified lines
3115	if (sc->compaction_ready)	3089	if (sc->compaction_ready)
3116	return 1;	3090	return 1;
3117		3091
		3092	/*
		3093	* We make inactive:active ratio decisions based on the node's
		3094	* composition of memory, but a restrictive reclaim_idx or a
		3095	* memory.low cgroup setting can exempt large amounts of
		3096	* memory from reclaim. Neither of which are very common, so
		3097	* instead of doing costly eligibility calculations of the
		3098	* entire cgroup subtree up front, we assume the estimates are
		3099	* good, and retry with forcible deactivation if that fails.
		3100	*/
		3101	if (sc->skipped_deactivate) {
		3102	sc->priority = initial_priority;
		3103	sc->force_deactivate = 1;
		3104	sc->skipped_deactivate = 0;
		3105	goto retry;
		3106	}
		3107
3118	/* Untapped cgroup reserves? Don't OOM, retry. */	3108	/* Untapped cgroup reserves? Don't OOM, retry. */
3119	if (sc->memcg_low_skipped) {	3109	if (sc->memcg_low_skipped) {
3120	sc->priority = initial_priority;	3110	sc->priority = initial_priority;
		3111	sc->force_deactivate = 0;
		3112	sc->skipped_deactivate = 0;
3121	sc->memcg_low_reclaim = 1;	3113	sc->memcg_low_reclaim = 1;
3122	sc->memcg_low_skipped = 0;	3114	sc->memcg_low_skipped = 0;
3123	goto retry;	3115	goto retry;
@@ -3309,6 +3301,7 @@ unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, · 185 unmodified lines
3309	pg_data_t *pgdat,	3301	pg_data_t *pgdat,
3310	unsigned long *nr_scanned)	3302	unsigned long *nr_scanned)
3311	{	3303	{
		3304	struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
3312	struct scan_control sc = {	3305	struct scan_control sc = {
3313	.nr_to_reclaim = SWAP_CLUSTER_MAX,	3306	.nr_to_reclaim = SWAP_CLUSTER_MAX,
3314	.target_mem_cgroup = memcg,	3307	.target_mem_cgroup = memcg,
@@ -3317,7 +3310,6 @@ unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, · 2 unmodified lines
3317	.reclaim_idx = MAX_NR_ZONES - 1,	3310	.reclaim_idx = MAX_NR_ZONES - 1,
3318	.may_swap = !noswap,	3311	.may_swap = !noswap,
3319	};	3312	};
3320	unsigned long lru_pages;
3321		3313
3322	WARN_ON_ONCE(!current->reclaim_state);	3314	WARN_ON_ONCE(!current->reclaim_state);
3323		3315
@@ -3334,7 +3326,7 @@ unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, · 10 unmodified lines
3334	* will pick up pages from other mem cgroup's as well. We hack	3326	* will pick up pages from other mem cgroup's as well. We hack
3335	* the priority and make it zero.	3327	* the priority and make it zero.
3336	*/	3328	*/
3337	shrink_node_memcg(pgdat, memcg, &sc, &lru_pages);	3329	shrink_lruvec(lruvec, &sc);
3338		3330
3339	trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed);	3331	trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed);
3340		3332
@@ -3348,10 +3340,8 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, · 7 unmodified lines
3348	gfp_t gfp_mask,	3340	gfp_t gfp_mask,
3349	bool may_swap)	3341	bool may_swap)
3350	{	3342	{
3351	struct zonelist *zonelist;
3352	unsigned long nr_reclaimed;	3343	unsigned long nr_reclaimed;
3353	unsigned long pflags;	3344	unsigned long pflags;
3354	int nid;
3355	unsigned int noreclaim_flag;	3345	unsigned int noreclaim_flag;
3356	struct scan_control sc = {	3346	struct scan_control sc = {
3357	.nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),	3347	.nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
@@ -3364,16 +3354,14 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, · 6 unmodified lines
3364	.may_unmap = 1,	3354	.may_unmap = 1,
3365	.may_swap = may_swap,	3355	.may_swap = may_swap,
3366	};	3356	};
3367
3368	set_task_reclaim_state(current, &sc.reclaim_state);
3369	/*	3357	/*
3370	* Unlike direct reclaim via alloc_pages(), memcg's reclaim doesn't	3358	* Traverse the ZONELIST_FALLBACK zonelist of the current node to put
3371	* take care of from where we get pages. So the node where we start the	3359	* equal pressure on all the nodes. This is based on the assumption that
3372	* scan does not need to be the current node.	3360	* the reclaim does not bail out early.
3373	*/	3361	*/
3374	nid = mem_cgroup_select_victim_node(memcg);	3362	struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
3375		3363
3376	zonelist = &NODE_DATA(nid)->node_zonelists[ZONELIST_FALLBACK];	3364	set_task_reclaim_state(current, &sc.reclaim_state);
3377		3365
3378	trace_mm_vmscan_memcg_reclaim_begin(0, sc.gfp_mask);	3366	trace_mm_vmscan_memcg_reclaim_begin(0, sc.gfp_mask);
3379		3367
@@ -3396,18 +3384,20 @@ static void age_active_anon(struct pglist_data *pgdat, · 16 unmodified lines
3396	struct scan_control *sc)	3384	struct scan_control *sc)
3397	{	3385	{
3398	struct mem_cgroup *memcg;	3386	struct mem_cgroup *memcg;
		3387	struct lruvec *lruvec;
3399		3388
3400	if (!total_swap_pages)	3389	if (!total_swap_pages)
3401	return;	3390	return;
3402		3391
		3392	lruvec = mem_cgroup_lruvec(NULL, pgdat);
		3393	if (!inactive_is_low(lruvec, LRU_INACTIVE_ANON))
		3394	return;
		3395
3403	memcg = mem_cgroup_iter(NULL, NULL, NULL);	3396	memcg = mem_cgroup_iter(NULL, NULL, NULL);
3404	do {	3397	do {
3405	struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg);	3398	lruvec = mem_cgroup_lruvec(memcg, pgdat);
3406		3399	shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
3407	if (inactive_list_is_low(lruvec, false, sc, true))	3400	sc, LRU_ACTIVE_ANON);
3408	shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
3409	sc, LRU_ACTIVE_ANON);
3410
3411	memcg = mem_cgroup_iter(NULL, memcg, NULL);	3401	memcg = mem_cgroup_iter(NULL, memcg, NULL);
3412	} while (memcg);	3402	} while (memcg);
3413	}	3403	}
@@ -3475,7 +3465,9 @@ static bool pgdat_balanced(pg_data_t *pgdat, int order, int classzone_idx) · 61 unmodified lines
3475	/* Clear pgdat state for congested, dirty or under writeback. */	3465	/* Clear pgdat state for congested, dirty or under writeback. */
3476	static void clear_pgdat_congested(pg_data_t *pgdat)	3466	static void clear_pgdat_congested(pg_data_t *pgdat)
3477	{	3467	{
3478	clear_bit(PGDAT_CONGESTED, &pgdat->flags);	3468	struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat);
		3469
		3470	clear_bit(LRUVEC_CONGESTED, &lruvec->flags);
3479	clear_bit(PGDAT_DIRTY, &pgdat->flags);	3471	clear_bit(PGDAT_DIRTY, &pgdat->flags);
3480	clear_bit(PGDAT_WRITEBACK, &pgdat->flags);	3472	clear_bit(PGDAT_WRITEBACK, &pgdat->flags);
3481	}	3473	}

mm/workingset.c+53−16

@@ -213,28 +213,53 @@ static void unpack_shadow(void shadow, int memcgidp, pg_data_t **pgdat, · 212 unmodified lines
213	*workingsetp = workingset;	213	*workingsetp = workingset;
214	}	214	}
215		215
		216	static void advance_inactive_age(struct mem_cgroup memcg, pg_data_t pgdat)
		217	{
		218	/*
		219	* Reclaiming a cgroup means reclaiming all its children in a
		220	* round-robin fashion. That means that each cgroup has an LRU
		221	* order that is composed of the LRU orders of its child
		222	* cgroups; and every page has an LRU position not just in the
		223	* cgroup that owns it, but in all of that group's ancestors.
		224	*
		225	* So when the physical inactive list of a leaf cgroup ages,
		226	* the virtual inactive lists of all its parents, including
		227	* the root cgroup's, age as well.
		228	*/
		229	do {
		230	struct lruvec *lruvec;
		231
		232	lruvec = mem_cgroup_lruvec(memcg, pgdat);
		233	atomic_long_inc(&lruvec->inactive_age);
		234	} while (memcg && (memcg = parent_mem_cgroup(memcg)));
		235	}
		236
216	/**	237	/**
217	* workingset_eviction - note the eviction of a page from memory	238	* workingset_eviction - note the eviction of a page from memory
		239	* @target_memcg: the cgroup that is causing the reclaim
218	* @page: the page being evicted	240	* @page: the page being evicted
219	*	241	*
220	* Returns a shadow entry to be stored in @page->mapping->i_pages in place	242	* Returns a shadow entry to be stored in @page->mapping->i_pages in place
221	* of the evicted @page so that a later refault can be detected.	243	* of the evicted @page so that a later refault can be detected.
222	*/	244	*/
223	void workingset_eviction(struct page page)	245	void workingset_eviction(struct page page, struct mem_cgroup *target_memcg)
224	{	246	{
225	struct pglist_data *pgdat = page_pgdat(page);	247	struct pglist_data *pgdat = page_pgdat(page);
226	struct mem_cgroup *memcg = page_memcg(page);
227	int memcgid = mem_cgroup_id(memcg);
228	unsigned long eviction;	248	unsigned long eviction;
229	struct lruvec *lruvec;	249	struct lruvec *lruvec;
		250	int memcgid;
230		251
231	/* Page is fully exclusive and pins page->mem_cgroup */	252	/* Page is fully exclusive and pins page->mem_cgroup */
232	VM_BUG_ON_PAGE(PageLRU(page), page);	253	VM_BUG_ON_PAGE(PageLRU(page), page);
233	VM_BUG_ON_PAGE(page_count(page), page);	254	VM_BUG_ON_PAGE(page_count(page), page);
234	VM_BUG_ON_PAGE(!PageLocked(page), page);	255	VM_BUG_ON_PAGE(!PageLocked(page), page);
235		256
236	lruvec = mem_cgroup_lruvec(pgdat, memcg);	257	advance_inactive_age(page_memcg(page), pgdat);
237	eviction = atomic_long_inc_return(&lruvec->inactive_age);	258
		259	lruvec = mem_cgroup_lruvec(target_memcg, pgdat);
		260	/* XXX: target_memcg can be NULL, go through lruvec */
		261	memcgid = mem_cgroup_id(lruvec_memcg(lruvec));
		262	eviction = atomic_long_read(&lruvec->inactive_age);
238	return pack_shadow(memcgid, pgdat, eviction, PageWorkingset(page));	263	return pack_shadow(memcgid, pgdat, eviction, PageWorkingset(page));
239	}	264	}
240		265
@@ -244,10 +269,13 @@ void workingset_eviction(struct page page) · 3 unmodified lines
244	* @shadow: shadow entry of the evicted page	269	* @shadow: shadow entry of the evicted page
245	*	270	*
246	* Calculates and evaluates the refault distance of the previously	271	* Calculates and evaluates the refault distance of the previously
247	* evicted page in the context of the node it was allocated in.	272	* evicted page in the context of the node and the memcg whose memory
		273	* pressure caused the eviction.
248	*/	274	*/
249	void workingset_refault(struct page page, void shadow)	275	void workingset_refault(struct page page, void shadow)
250	{	276	{
		277	struct mem_cgroup *eviction_memcg;
		278	struct lruvec *eviction_lruvec;
251	unsigned long refault_distance;	279	unsigned long refault_distance;
252	struct pglist_data *pgdat;	280	struct pglist_data *pgdat;
253	unsigned long active_file;	281	unsigned long active_file;
@@ -277,12 +305,12 @@ void workingset_refault(struct page page, void shadow) · 23 unmodified lines
277	* would be better if the root_mem_cgroup existed in all	305	* would be better if the root_mem_cgroup existed in all
278	* configurations instead.	306	* configurations instead.
279	*/	307	*/
280	memcg = mem_cgroup_from_id(memcgid);	308	eviction_memcg = mem_cgroup_from_id(memcgid);
281	if (!mem_cgroup_disabled() && !memcg)	309	if (!mem_cgroup_disabled() && !eviction_memcg)
282	goto out;	310	goto out;
283	lruvec = mem_cgroup_lruvec(pgdat, memcg);	311	eviction_lruvec = mem_cgroup_lruvec(eviction_memcg, pgdat);
284	refault = atomic_long_read(&lruvec->inactive_age);	312	refault = atomic_long_read(&eviction_lruvec->inactive_age);
285	active_file = lruvec_lru_size(lruvec, LRU_ACTIVE_FILE, MAX_NR_ZONES);	313	active_file = lruvec_page_state(eviction_lruvec, NR_ACTIVE_FILE);
286		314
287	/*	315	/*
288	* Calculate the refault distance	316	* Calculate the refault distance
@@ -302,6 +330,17 @@ void workingset_refault(struct page page, void shadow) · 13 unmodified lines
302	*/	330	*/
303	refault_distance = (refault - eviction) & EVICTION_MASK;	331	refault_distance = (refault - eviction) & EVICTION_MASK;
304		332
		333	/*
		334	* The activation decision for this page is made at the level
		335	* where the eviction occurred, as that is where the LRU order
		336	* during page reclaim is being determined.
		337	*
		338	* However, the cgroup that will own the page is the one that
		339	* is actually experiencing the refault event.
		340	*/
		341	memcg = page_memcg(page);
		342	lruvec = mem_cgroup_lruvec(memcg, pgdat);
		343
305	inc_lruvec_state(lruvec, WORKINGSET_REFAULT);	344	inc_lruvec_state(lruvec, WORKINGSET_REFAULT);
306		345
307	/*	346	/*
@@ -313,7 +352,7 @@ void workingset_refault(struct page page, void shadow) · 5 unmodified lines
313	goto out;	352	goto out;
314		353
315	SetPageActive(page);	354	SetPageActive(page);
316	atomic_long_inc(&lruvec->inactive_age);	355	advance_inactive_age(memcg, pgdat);
317	inc_lruvec_state(lruvec, WORKINGSET_ACTIVATE);	356	inc_lruvec_state(lruvec, WORKINGSET_ACTIVATE);
318		357
319	/* Page was active prior to eviction */	358	/* Page was active prior to eviction */
@@ -332,7 +371,6 @@ out: · 12 unmodified lines
332	void workingset_activation(struct page *page)	371	void workingset_activation(struct page *page)
333	{	372	{
334	struct mem_cgroup *memcg;	373	struct mem_cgroup *memcg;
335	struct lruvec *lruvec;
336		374
337	rcu_read_lock();	375	rcu_read_lock();
338	/*	376	/*
@@ -345,8 +383,7 @@ void workingset_activation(struct page *page) · 6 unmodified lines
345	memcg = page_memcg_rcu(page);	383	memcg = page_memcg_rcu(page);
346	if (!mem_cgroup_disabled() && !memcg)	384	if (!mem_cgroup_disabled() && !memcg)
347	goto out;	385	goto out;
348	lruvec = mem_cgroup_lruvec(page_pgdat(page), memcg);	386	advance_inactive_age(memcg, page_pgdat(page));
349	atomic_long_inc(&lruvec->inactive_age);
350	out:	387	out:
351	rcu_read_unlock();	388	rcu_read_unlock();
352	}	389	}
@@ -426,7 +463,7 @@ static unsigned long count_shadow_nodes(struct shrinker *shrinker, · 73 unmodified lines
426	struct lruvec *lruvec;	463	struct lruvec *lruvec;
427	int i;	464	int i;
428		465
429	lruvec = mem_cgroup_lruvec(NODE_DATA(sc->nid), sc->memcg);	466	lruvec = mem_cgroup_lruvec(sc->memcg, NODE_DATA(sc->nid));
430	for (pages = 0, i = 0; i < NR_LRU_LISTS; i++)	467	for (pages = 0, i = 0; i < NR_LRU_LISTS; i++)
431	pages += lruvec_page_state_local(lruvec,	468	pages += lruvec_page_state_local(lruvec,
432	NR_LRU_BASE + i);	469	NR_LRU_BASE + i);

mm/z3fold.c+303−72

 #include <linux/workqueue.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
+#include <linux/rwlock.h>
 #include <linux/zpool.h>
 #include <linux/magic.h>
 	 */
 	unsigned long slot[BUDDY_MASK + 1];
 	unsigned long pool; /* back link + flags */
+	rwlock_t lock;
 };
 #define HANDLE_FLAG_MASK	(0x03)
 	unsigned short start_middle;
 	unsigned short first_num:2;
 	unsigned short mapped_count:2;
+	unsigned short foreign_handles:2;
 };
 /**
 	PAGE_CLAIMED, /* by either reclaim or free */
 };
+/*
+ * handle flags, go under HANDLE_FLAG_MASK
+ */
+enum z3fold_handle_flags {
+	HANDLES_ORPHANED = 0,
+};
+/*
+ * Forward declarations
+ */
+static struct z3fold_header *__z3fold_alloc(struct z3fold_pool *, size_t, bool);
+static void compact_page_work(struct work_struct *w);
 /*****************
  * Helpers
 *****************/
 #define for_each_unbuddied_list(_iter, _begin) \
 	for ((_iter) = (_begin); (_iter) < NCHUNKS; (_iter)++)
-static void compact_page_work(struct work_struct *w);
 static inline struct z3fold_buddy_slots *alloc_slots(struct z3fold_pool *pool,
 							gfp_t gfp)
 {
 	if (slots) {
 		memset(slots->slot, 0, sizeof(slots->slot));
 		slots->pool = (unsigned long)pool;
+		rwlock_init(&slots->lock);
 	}
 	return slots;
 	return (struct z3fold_buddy_slots *)(handle & ~(SLOTS_ALIGN - 1));
 }
+/* Lock a z3fold page */
+static inline void z3fold_page_lock(struct z3fold_header *zhdr)
+{
+	spin_lock(&zhdr->page_lock);
+}
+/* Try to lock a z3fold page */
+static inline int z3fold_page_trylock(struct z3fold_header *zhdr)
+{
+	return spin_trylock(&zhdr->page_lock);
+}
+/* Unlock a z3fold page */
+static inline void z3fold_page_unlock(struct z3fold_header *zhdr)
+{
+	spin_unlock(&zhdr->page_lock);
+}
+static inline struct z3fold_header *__get_z3fold_header(unsigned long handle,
+							bool lock)
+{
+	struct z3fold_buddy_slots *slots;
+	struct z3fold_header *zhdr;
+	int locked = 0;
+	if (!(handle & (1 << PAGE_HEADLESS))) {
+		slots = handle_to_slots(handle);
+		do {
+			unsigned long addr;
+			read_lock(&slots->lock);
+			addr = *(unsigned long *)handle;
+			zhdr = (struct z3fold_header *)(addr & PAGE_MASK);
+			if (lock)
+				locked = z3fold_page_trylock(zhdr);
+			read_unlock(&slots->lock);
+			if (locked)
+				break;
+			cpu_relax();
+		} while (lock);
+	} else {
+		zhdr = (struct z3fold_header *)(handle & PAGE_MASK);
+	}
+	return zhdr;
+}
+/* Returns the z3fold page where a given handle is stored */
+static inline struct z3fold_header *handle_to_z3fold_header(unsigned long h)
+{
+	return __get_z3fold_header(h, false);
+}
+/* return locked z3fold page if it's not headless */
+static inline struct z3fold_header *get_z3fold_header(unsigned long h)
+{
+	return __get_z3fold_header(h, true);
+}
+static inline void put_z3fold_header(struct z3fold_header *zhdr)
+{
+	struct page *page = virt_to_page(zhdr);
+	if (!test_bit(PAGE_HEADLESS, &page->private))
+		z3fold_page_unlock(zhdr);
+}
 static inline void free_handle(unsigned long handle)
 {
 	struct z3fold_buddy_slots *slots;
+	struct z3fold_header *zhdr;
 	int i;
 	bool is_free;
 	if (handle & (1 << PAGE_HEADLESS))
 		return;
-	WARN_ON(*(unsigned long *)handle == 0);
+	if (WARN_ON(*(unsigned long *)handle == 0))
-	*(unsigned long *)handle = 0;
+		return;
+	zhdr = handle_to_z3fold_header(handle);
 	slots = handle_to_slots(handle);
+	write_lock(&slots->lock);
+	*(unsigned long *)handle = 0;
+	write_unlock(&slots->lock);
+	if (zhdr->slots == slots)
+		return; /* simple case, nothing else to do */
+	/* we are freeing a foreign handle if we are here */
+	zhdr->foreign_handles--;
 	is_free = true;
+	read_lock(&slots->lock);
+	if (!test_bit(HANDLES_ORPHANED, &slots->pool)) {
+		read_unlock(&slots->lock);
+		return;
+	}
 	for (i = 0; i <= BUDDY_MASK; i++) {
 		if (slots->slot[i]) {
 			is_free = false;
 			break;
 		}
 	}
+	read_unlock(&slots->lock);
 	if (is_free) {
 		struct z3fold_pool *pool = slots_to_pool(slots);
 	zhdr->first_num = 0;
 	zhdr->start_middle = 0;
 	zhdr->cpu = -1;
+	zhdr->foreign_handles = 0;
 	zhdr->slots = slots;
 	zhdr->pool = pool;
 	INIT_LIST_HEAD(&zhdr->buddy);
 	__free_page(page);
 }
-/* Lock a z3fold page */
-static inline void z3fold_page_lock(struct z3fold_header *zhdr)
-{
-	spin_lock(&zhdr->page_lock);
-}
-/* Try to lock a z3fold page */
-static inline int z3fold_page_trylock(struct z3fold_header *zhdr)
-{
-	return spin_trylock(&zhdr->page_lock);
-}
-/* Unlock a z3fold page */
-static inline void z3fold_page_unlock(struct z3fold_header *zhdr)
-{
-	spin_unlock(&zhdr->page_lock);
-}
 /* Helper function to build the index */
 static inline int __idx(struct z3fold_header *zhdr, enum buddy bud)
 {
 	if (bud == LAST)
 		h |= (zhdr->last_chunks << BUDDY_SHIFT);
+	write_lock(&slots->lock);
 	slots->slot[idx] = h;
+	write_unlock(&slots->lock);
 	return (unsigned long)&slots->slot[idx];
 }
 	return __encode_handle(zhdr, zhdr->slots, bud);
 }
-/* Returns the z3fold page where a given handle is stored */
-static inline struct z3fold_header *handle_to_z3fold_header(unsigned long h)
-{
-	unsigned long addr = h;
-	if (!(addr & (1 << PAGE_HEADLESS)))
-		addr = *(unsigned long *)h;
-	return (struct z3fold_header *)(addr & PAGE_MASK);
-}
 /* only for LAST bud, returns zero otherwise */
 static unsigned short handle_to_chunks(unsigned long handle)
 {
-	unsigned long addr = *(unsigned long *)handle;
+	struct z3fold_buddy_slots *slots = handle_to_slots(handle);
+	unsigned long addr;
+	read_lock(&slots->lock);
+	addr = *(unsigned long *)handle;
+	read_unlock(&slots->lock);
 	return (addr & ~PAGE_MASK) >> BUDDY_SHIFT;
 }
 static enum buddy handle_to_buddy(unsigned long handle)
 {
 	struct z3fold_header *zhdr;
+	struct z3fold_buddy_slots *slots = handle_to_slots(handle);
 	unsigned long addr;
+	read_lock(&slots->lock);
 	WARN_ON(handle & (1 << PAGE_HEADLESS));
 	addr = *(unsigned long *)handle;
+	read_unlock(&slots->lock);
 	zhdr = (struct z3fold_header *)(addr & PAGE_MASK);
 	return (addr - zhdr->first_num) & BUDDY_MASK;
 }
 {
 	struct page *page = virt_to_page(zhdr);
 	struct z3fold_pool *pool = zhdr_to_pool(zhdr);
+	bool is_free = true;
+	int i;
 	WARN_ON(!list_empty(&zhdr->buddy));
 	set_bit(PAGE_STALE, &page->private);
 	if (!list_empty(&page->lru))
 		list_del_init(&page->lru);
 	spin_unlock(&pool->lock);
+	/* If there are no foreign handles, free the handles array */
+	read_lock(&zhdr->slots->lock);
+	for (i = 0; i <= BUDDY_MASK; i++) {
+		if (zhdr->slots->slot[i]) {
+			is_free = false;
+			break;
+		}
+	}
+	if (!is_free)
+		set_bit(HANDLES_ORPHANED, &zhdr->slots->pool);
+	read_unlock(&zhdr->slots->lock);
+	if (is_free)
+		kmem_cache_free(pool->c_handle, zhdr->slots);
 	if (locked)
 		z3fold_page_unlock(zhdr);
 	spin_lock(&pool->stale_lock);
 	list_add(&zhdr->buddy, &pool->stale);
 	queue_work(pool->release_wq, &pool->work);
 	struct z3fold_header *zhdr = container_of(ref, struct z3fold_header,
 					       refcount);
 	struct z3fold_pool *pool = zhdr_to_pool(zhdr);
 	spin_lock(&pool->lock);
 	list_del_init(&zhdr->buddy);
 	spin_unlock(&pool->lock);
 		       zhdr->middle_chunks << CHUNK_SHIFT);
 }
+static inline bool buddy_single(struct z3fold_header *zhdr)
+{
+	return !((zhdr->first_chunks && zhdr->middle_chunks) ||
+			(zhdr->first_chunks && zhdr->last_chunks) ||
+			(zhdr->middle_chunks && zhdr->last_chunks));
+}
+static struct z3fold_header *compact_single_buddy(struct z3fold_header *zhdr)
+{
+	struct z3fold_pool *pool = zhdr_to_pool(zhdr);
+	void *p = zhdr;
+	unsigned long old_handle = 0;
+	size_t sz = 0;
+	struct z3fold_header *new_zhdr = NULL;
+	int first_idx = __idx(zhdr, FIRST);
+	int middle_idx = __idx(zhdr, MIDDLE);
+	int last_idx = __idx(zhdr, LAST);
+	unsigned short *moved_chunks = NULL;
+	/*
+	 * No need to protect slots here -- all the slots are "local" and
+	 * the page lock is already taken
+	 */
+	if (zhdr->first_chunks && zhdr->slots->slot[first_idx]) {
+		p += ZHDR_SIZE_ALIGNED;
+		sz = zhdr->first_chunks << CHUNK_SHIFT;
+		old_handle = (unsigned long)&zhdr->slots->slot[first_idx];
+		moved_chunks = &zhdr->first_chunks;
+	} else if (zhdr->middle_chunks && zhdr->slots->slot[middle_idx]) {
+		p += zhdr->start_middle << CHUNK_SHIFT;
+		sz = zhdr->middle_chunks << CHUNK_SHIFT;
+		old_handle = (unsigned long)&zhdr->slots->slot[middle_idx];
+		moved_chunks = &zhdr->middle_chunks;
+	} else if (zhdr->last_chunks && zhdr->slots->slot[last_idx]) {
+		p += PAGE_SIZE - (zhdr->last_chunks << CHUNK_SHIFT);
+		sz = zhdr->last_chunks << CHUNK_SHIFT;
+		old_handle = (unsigned long)&zhdr->slots->slot[last_idx];
+		moved_chunks = &zhdr->last_chunks;
+	}
+	if (sz > 0) {
+		enum buddy new_bud = HEADLESS;
+		short chunks = size_to_chunks(sz);
+		void *q;
+		new_zhdr = __z3fold_alloc(pool, sz, false);
+		if (!new_zhdr)
+			return NULL;
+		if (WARN_ON(new_zhdr == zhdr))
+			goto out_fail;
+		if (new_zhdr->first_chunks == 0) {
+			if (new_zhdr->middle_chunks != 0 &&
+					chunks >= new_zhdr->start_middle) {
+				new_bud = LAST;
+			} else {
+				new_bud = FIRST;
+			}
+		} else if (new_zhdr->last_chunks == 0) {
+			new_bud = LAST;
+		} else if (new_zhdr->middle_chunks == 0) {
+			new_bud = MIDDLE;
+		}
+		q = new_zhdr;
+		switch (new_bud) {
+		case FIRST:
+			new_zhdr->first_chunks = chunks;
+			q += ZHDR_SIZE_ALIGNED;
+			break;
+		case MIDDLE:
+			new_zhdr->middle_chunks = chunks;
+			new_zhdr->start_middle =
+				new_zhdr->first_chunks + ZHDR_CHUNKS;
+			q += new_zhdr->start_middle << CHUNK_SHIFT;
+			break;
+		case LAST:
+			new_zhdr->last_chunks = chunks;
+			q += PAGE_SIZE - (new_zhdr->last_chunks << CHUNK_SHIFT);
+			break;
+		default:
+			goto out_fail;
+		}
+		new_zhdr->foreign_handles++;
+		memcpy(q, p, sz);
+		write_lock(&zhdr->slots->lock);
+		*(unsigned long *)old_handle = (unsigned long)new_zhdr +
+			__idx(new_zhdr, new_bud);
+		if (new_bud == LAST)
+			*(unsigned long *)old_handle |=
+					(new_zhdr->last_chunks << BUDDY_SHIFT);
+		write_unlock(&zhdr->slots->lock);
+		add_to_unbuddied(pool, new_zhdr);
+		z3fold_page_unlock(new_zhdr);
+		*moved_chunks = 0;
+	}
+	return new_zhdr;
+out_fail:
+	if (new_zhdr) {
+		if (kref_put(&new_zhdr->refcount, release_z3fold_page_locked))
+			atomic64_dec(&pool->pages_nr);
+		else {
+			add_to_unbuddied(pool, new_zhdr);
+			z3fold_page_unlock(new_zhdr);
+		}
+	}
+	return NULL;
+}
 #define BIG_CHUNK_GAP	3
 /* Has to be called with lock held */
 static int z3fold_compact_page(struct z3fold_header *zhdr)
 		return;
 	}
+	if (!zhdr->foreign_handles && buddy_single(zhdr) &&
+	    zhdr->mapped_count == 0 && compact_single_buddy(zhdr)) {
+		if (kref_put(&zhdr->refcount, release_z3fold_page_locked))
+			atomic64_dec(&pool->pages_nr);
+		else
+			z3fold_page_unlock(zhdr);
+		return;
+	}
 	z3fold_compact_page(zhdr);
 	add_to_unbuddied(pool, zhdr);
 	z3fold_page_unlock(zhdr);
 		spin_unlock(&pool->lock);
 		page = virt_to_page(zhdr);
-		if (test_bit(NEEDS_COMPACTING, &page->private)) {
+		if (test_bit(NEEDS_COMPACTING, &page->private) ||
+		    test_bit(PAGE_CLAIMED, &page->private)) {
 			z3fold_page_unlock(zhdr);
 			zhdr = NULL;
 			put_cpu_ptr(pool->unbuddied);
 			spin_unlock(&pool->lock);
 			page = virt_to_page(zhdr);
-			if (test_bit(NEEDS_COMPACTING, &page->private)) {
+			if (test_bit(NEEDS_COMPACTING, &page->private) ||
+			    test_bit(PAGE_CLAIMED, &page->private)) {
 				z3fold_page_unlock(zhdr);
 				zhdr = NULL;
 				if (can_sleep)
 	enum buddy bud;
 	bool page_claimed;
-	zhdr = handle_to_z3fold_header(handle);
+	zhdr = get_z3fold_header(handle);
 	page = virt_to_page(zhdr);
 	page_claimed = test_and_set_bit(PAGE_CLAIMED, &page->private);
 			spin_lock(&pool->lock);
 			list_del(&page->lru);
 			spin_unlock(&pool->lock);
+			put_z3fold_header(zhdr);
 			free_z3fold_page(page, true);
 			atomic64_dec(&pool->pages_nr);
 		}
 	}
 	/* Non-headless case */
-	z3fold_page_lock(zhdr);
 	bud = handle_to_buddy(handle);
 	switch (bud) {
 	default:
 		pr_err("%s: unknown bud %d\n", __func__, bud);
 		WARN_ON(1);
-		z3fold_page_unlock(zhdr);
+		put_z3fold_header(zhdr);
+		clear_bit(PAGE_CLAIMED, &page->private);
 		return;
 	}
-	free_handle(handle);
+	if (!page_claimed)
+		free_handle(handle);
 	if (kref_put(&zhdr->refcount, release_z3fold_page_locked_list)) {
 		atomic64_dec(&pool->pages_nr);
 		return;
 	}
 	if (unlikely(PageIsolated(page)) ||
 	    test_and_set_bit(NEEDS_COMPACTING, &page->private)) {
-		z3fold_page_unlock(zhdr);
+		put_z3fold_header(zhdr);
 		clear_bit(PAGE_CLAIMED, &page->private);
 		return;
 	}
 		spin_unlock(&pool->lock);
 		zhdr->cpu = -1;
 		kref_get(&zhdr->refcount);
-		do_compact_page(zhdr, true);
 		clear_bit(PAGE_CLAIMED, &page->private);
+		do_compact_page(zhdr, true);
 		return;
 	}
 	kref_get(&zhdr->refcount);
-	queue_work_on(zhdr->cpu, pool->compact_wq, &zhdr->work);
 	clear_bit(PAGE_CLAIMED, &page->private);
-	z3fold_page_unlock(zhdr);
+	queue_work_on(zhdr->cpu, pool->compact_wq, &zhdr->work);
+	put_z3fold_header(zhdr);
 }
 /**
  */
 static int z3fold_reclaim_page(struct z3fold_pool *pool, unsigned int retries)
 {
-	int i, ret = 0;
+	int i, ret = -1;
 	struct z3fold_header *zhdr = NULL;
 	struct page *page = NULL;
 	struct list_head *pos;
-	struct z3fold_buddy_slots slots;
 	unsigned long first_handle = 0, middle_handle = 0, last_handle = 0;
 	spin_lock(&pool->lock);
 				zhdr = NULL;
 				continue; /* can't evict at this point */
 			}
+			if (zhdr->foreign_handles) {
+				clear_bit(PAGE_CLAIMED, &page->private);
+				z3fold_page_unlock(zhdr);
+				zhdr = NULL;
+				continue; /* can't evict such page */
+			}
 			kref_get(&zhdr->refcount);
 			list_del_init(&zhdr->buddy);
 			zhdr->cpu = -1;
 			last_handle = 0;
 			middle_handle = 0;
 			if (zhdr->first_chunks)
-				first_handle = __encode_handle(zhdr, &slots,
+				first_handle = encode_handle(zhdr, FIRST);
-								FIRST);
 			if (zhdr->middle_chunks)
-				middle_handle = __encode_handle(zhdr, &slots,
+				middle_handle = encode_handle(zhdr, MIDDLE);
-								MIDDLE);
 			if (zhdr->last_chunks)
-				last_handle = __encode_handle(zhdr, &slots,
+				last_handle = encode_handle(zhdr, LAST);
-								LAST);
 			/*
 			 * it's safe to unlock here because we hold a
 			 * reference to this page
 			 */
 			z3fold_page_unlock(zhdr);
 		} else {
-			first_handle = __encode_handle(zhdr, &slots, HEADLESS);
+			first_handle = encode_handle(zhdr, HEADLESS);
 			last_handle = middle_handle = 0;
 		}
 		/* Issue the eviction callback(s) */
 		if (middle_handle) {
 			ret = pool->ops->evict(pool, middle_handle);
 			if (ret)
 				goto next;
+			free_handle(middle_handle);
 		}
 		if (first_handle) {
 			ret = pool->ops->evict(pool, first_handle);
 			if (ret)
 				goto next;
+			free_handle(first_handle);
 		}
 		if (last_handle) {
 			ret = pool->ops->evict(pool, last_handle);
 			if (ret)
 				goto next;
+			free_handle(last_handle);
 		}
 next:
 		if (test_bit(PAGE_HEADLESS, &page->private)) {
 	void *addr;
 	enum buddy buddy;
-	zhdr = handle_to_z3fold_header(handle);
+	zhdr = get_z3fold_header(handle);
 	addr = zhdr;
 	page = virt_to_page(zhdr);
 	if (test_bit(PAGE_HEADLESS, &page->private))
 		goto out;
-	z3fold_page_lock(zhdr);
 	buddy = handle_to_buddy(handle);
 	switch (buddy) {
 	case FIRST:
 	if (addr)
 		zhdr->mapped_count++;
-	z3fold_page_unlock(zhdr);
 out:
+	put_z3fold_header(zhdr);
 	return addr;
 }
 	struct page *page;
 	enum buddy buddy;
-	zhdr = handle_to_z3fold_header(handle);
+	zhdr = get_z3fold_header(handle);
 	page = virt_to_page(zhdr);
 	if (test_bit(PAGE_HEADLESS, &page->private))
 		return;
-	z3fold_page_lock(zhdr);
 	buddy = handle_to_buddy(handle);
 	if (buddy == MIDDLE)
 		clear_bit(MIDDLE_CHUNK_MAPPED, &page->private);
 	zhdr->mapped_count--;
-	z3fold_page_unlock(zhdr);
+	put_z3fold_header(zhdr);
 }
 /**
 	    test_bit(PAGE_STALE, &page->private))
 		goto out;
+	if (zhdr->mapped_count != 0 || zhdr->foreign_handles != 0)
+		goto out;
 	pool = zhdr_to_pool(zhdr);
+	spin_lock(&pool->lock);
+	if (!list_empty(&zhdr->buddy))
+		list_del_init(&zhdr->buddy);
+	if (!list_empty(&page->lru))
+		list_del_init(&page->lru);
+	spin_unlock(&pool->lock);
+	kref_get(&zhdr->refcount);
+	z3fold_page_unlock(zhdr);
+	return true;
-	if (zhdr->mapped_count == 0) {
-		kref_get(&zhdr->refcount);
-		if (!list_empty(&zhdr->buddy))
-			list_del_init(&zhdr->buddy);
-		spin_lock(&pool->lock);
-		if (!list_empty(&page->lru))
-			list_del(&page->lru);
-		spin_unlock(&pool->lock);
-		z3fold_page_unlock(zhdr);
-		return true;
-	}
 out:
 	z3fold_page_unlock(zhdr);
 	return false;
 	if (!z3fold_page_trylock(zhdr)) {
 		return -EAGAIN;
 	}
-	if (zhdr->mapped_count != 0) {
+	if (zhdr->mapped_count != 0 || zhdr->foreign_handles != 0) {
 		z3fold_page_unlock(zhdr);
 		return -EBUSY;
 	}

scripts/spelling.txt+28−0

@@ -87,6 +87,7 @@ algorith\|\|algorithm · 86 unmodified lines
87	algorithmical\|\|algorithmically	87	algorithmical\|\|algorithmically
88	algoritm\|\|algorithm	88	algoritm\|\|algorithm
89	algoritms\|\|algorithms	89	algoritms\|\|algorithms
		90	algorithmn\|\|algorithm
90	algorrithm\|\|algorithm	91	algorrithm\|\|algorithm
91	algorritm\|\|algorithm	92	algorritm\|\|algorithm
92	aligment\|\|alignment	93	aligment\|\|alignment
@@ -109,6 +110,7 @@ alredy\|\|already · 16 unmodified lines
109	altough\|\|although	110	altough\|\|although
110	alue\|\|value	111	alue\|\|value
111	ambigious\|\|ambiguous	112	ambigious\|\|ambiguous
		113	ambigous\|\|ambiguous
112	amoung\|\|among	114	amoung\|\|among
113	amout\|\|amount	115	amout\|\|amount
114	amplifer\|\|amplifier	116	amplifer\|\|amplifier
@@ -179,6 +181,7 @@ attepmpt\|\|attempt · 64 unmodified lines
179	attnetion\|\|attention	181	attnetion\|\|attention
180	attruibutes\|\|attributes	182	attruibutes\|\|attributes
181	authentification\|\|authentication	183	authentification\|\|authentication
		184	authenicated\|\|authenticated
182	automaticaly\|\|automatically	185	automaticaly\|\|automatically
183	automaticly\|\|automatically	186	automaticly\|\|automatically
184	automatize\|\|automate	187	automatize\|\|automate
@@ -286,6 +289,7 @@ claread\|\|cleared · 101 unmodified lines
286	clared\|\|cleared	289	clared\|\|cleared
287	closeing\|\|closing	290	closeing\|\|closing
288	clustred\|\|clustered	291	clustred\|\|clustered
		292	cnfiguration\|\|configuration
289	coexistance\|\|coexistence	293	coexistance\|\|coexistence
290	colescing\|\|coalescing	294	colescing\|\|coalescing
291	collapsable\|\|collapsible	295	collapsable\|\|collapsible
@@ -325,9 +329,11 @@ comression\|\|compression · 33 unmodified lines
325	comunication\|\|communication	329	comunication\|\|communication
326	conbination\|\|combination	330	conbination\|\|combination
327	conditionaly\|\|conditionally	331	conditionaly\|\|conditionally
		332	conditon\|\|condition
328	conected\|\|connected	333	conected\|\|connected
329	conector\|\|connector	334	conector\|\|connector
330	connecetd\|\|connected	335	connecetd\|\|connected
		336	configration\|\|configuration
331	configuartion\|\|configuration	337	configuartion\|\|configuration
332	configuation\|\|configuration	338	configuation\|\|configuration
333	configued\|\|configured	339	configued\|\|configured
@@ -347,6 +353,7 @@ containts\|\|contains · 13 unmodified lines
347	contaisn\|\|contains	353	contaisn\|\|contains
348	contant\|\|contact	354	contant\|\|contact
349	contence\|\|contents	355	contence\|\|contents
		356	contiguos\|\|contiguous
350	continious\|\|continuous	357	continious\|\|continuous
351	continous\|\|continuous	358	continous\|\|continuous
352	continously\|\|continuously	359	continously\|\|continuously
@@ -380,6 +387,7 @@ cylic\|\|cyclic · 27 unmodified lines
380	dafault\|\|default	387	dafault\|\|default
381	deafult\|\|default	388	deafult\|\|default
382	deamon\|\|daemon	389	deamon\|\|daemon
		390	debouce\|\|debounce
383	decompres\|\|decompress	391	decompres\|\|decompress
384	decsribed\|\|described	392	decsribed\|\|described
385	decription\|\|description	393	decription\|\|description
@@ -448,6 +456,7 @@ diffrent\|\|different · 62 unmodified lines
448	differenciate\|\|differentiate	456	differenciate\|\|differentiate
449	diffrentiate\|\|differentiate	457	diffrentiate\|\|differentiate
450	difinition\|\|definition	458	difinition\|\|definition
		459	digial\|\|digital
451	dimention\|\|dimension	460	dimention\|\|dimension
452	dimesions\|\|dimensions	461	dimesions\|\|dimensions
453	dispalying\|\|displaying	462	dispalying\|\|displaying
@@ -489,6 +498,7 @@ droput\|\|dropout · 35 unmodified lines
489	druing\|\|during	498	druing\|\|during
490	dynmaic\|\|dynamic	499	dynmaic\|\|dynamic
491	eanable\|\|enable	500	eanable\|\|enable
		501	eanble\|\|enable
492	easilly\|\|easily	502	easilly\|\|easily
493	ecspecially\|\|especially	503	ecspecially\|\|especially
494	edditable\|\|editable	504	edditable\|\|editable
@@ -502,6 +512,7 @@ elementry\|\|elementary · 7 unmodified lines
502	eletronic\|\|electronic	512	eletronic\|\|electronic
503	embeded\|\|embedded	513	embeded\|\|embedded
504	enabledi\|\|enabled	514	enabledi\|\|enabled
		515	enbale\|\|enable
505	enble\|\|enable	516	enble\|\|enable
506	enchanced\|\|enhanced	517	enchanced\|\|enhanced
507	encorporating\|\|incorporating	518	encorporating\|\|incorporating
@@ -536,6 +547,7 @@ excellant\|\|excellent · 28 unmodified lines
536	execeeded\|\|exceeded	547	execeeded\|\|exceeded
537	execeeds\|\|exceeds	548	execeeds\|\|exceeds
538	exeed\|\|exceed	549	exeed\|\|exceed
		550	exeuction\|\|execution
539	existance\|\|existence	551	existance\|\|existence
540	existant\|\|existent	552	existant\|\|existent
541	exixt\|\|exist	553	exixt\|\|exist
@@ -601,10 +613,12 @@ frambuffer\|\|framebuffer · 59 unmodified lines
601	framming\|\|framing	613	framming\|\|framing
602	framwork\|\|framework	614	framwork\|\|framework
603	frequncy\|\|frequency	615	frequncy\|\|frequency
		616	frequancy\|\|frequency
604	frome\|\|from	617	frome\|\|from
605	fucntion\|\|function	618	fucntion\|\|function
606	fuction\|\|function	619	fuction\|\|function
607	fuctions\|\|functions	620	fuctions\|\|functions
		621	fullill\|\|fulfill
608	funcation\|\|function	622	funcation\|\|function
609	funcion\|\|function	623	funcion\|\|function
610	functionallity\|\|functionality	624	functionallity\|\|functionality
@@ -642,6 +656,7 @@ happend\|\|happened · 31 unmodified lines
642	harware\|\|hardware	656	harware\|\|hardware
643	heirarchically\|\|hierarchically	657	heirarchically\|\|hierarchically
644	helpfull\|\|helpful	658	helpfull\|\|helpful
		659	hexdecimal\|\|hexadecimal
645	hybernate\|\|hibernate	660	hybernate\|\|hibernate
646	hierachy\|\|hierarchy	661	hierachy\|\|hierarchy
647	hierarchie\|\|hierarchy	662	hierarchie\|\|hierarchy
@@ -709,12 +724,14 @@ initalize\|\|initialize · 61 unmodified lines
709	initation\|\|initiation	724	initation\|\|initiation
710	initators\|\|initiators	725	initators\|\|initiators
711	initialiazation\|\|initialization	726	initialiazation\|\|initialization
		727	initializationg\|\|initialization
712	initializiation\|\|initialization	728	initializiation\|\|initialization
713	initialze\|\|initialize	729	initialze\|\|initialize
714	initialzed\|\|initialized	730	initialzed\|\|initialized
715	initialzing\|\|initializing	731	initialzing\|\|initializing
716	initilization\|\|initialization	732	initilization\|\|initialization
717	initilize\|\|initialize	733	initilize\|\|initialize
		734	initliaze\|\|initialize
718	inofficial\|\|unofficial	735	inofficial\|\|unofficial
719	inrerface\|\|interface	736	inrerface\|\|interface
720	insititute\|\|institute	737	insititute\|\|institute
@@ -779,6 +796,7 @@ itertation\|\|iteration · 58 unmodified lines
779	itslef\|\|itself	796	itslef\|\|itself
780	jave\|\|java	797	jave\|\|java
781	jeffies\|\|jiffies	798	jeffies\|\|jiffies
		799	jumpimng\|\|jumping
782	juse\|\|just	800	juse\|\|just
783	jus\|\|just	801	jus\|\|just
784	kown\|\|known	802	kown\|\|known
@@ -839,6 +857,7 @@ messags\|\|messages · 54 unmodified lines
839	messgaes\|\|messages	857	messgaes\|\|messages
840	messsage\|\|message	858	messsage\|\|message
841	messsages\|\|messages	859	messsages\|\|messages
		860	metdata\|\|metadata
842	micropone\|\|microphone	861	micropone\|\|microphone
843	microprocesspr\|\|microprocessor	862	microprocesspr\|\|microprocessor
844	migrateable\|\|migratable	863	migrateable\|\|migratable
@@ -857,6 +876,7 @@ mismactch\|\|mismatch · 12 unmodified lines
857	missign\|\|missing	876	missign\|\|missing
858	missmanaged\|\|mismanaged	877	missmanaged\|\|mismanaged
859	missmatch\|\|mismatch	878	missmatch\|\|mismatch
		879	misssing\|\|missing
860	miximum\|\|maximum	880	miximum\|\|maximum
861	mmnemonic\|\|mnemonic	881	mmnemonic\|\|mnemonic
862	mnay\|\|many	882	mnay\|\|many
@@ -912,6 +932,7 @@ occured\|\|occurred · 49 unmodified lines
912	occuring\|\|occurring	932	occuring\|\|occurring
913	offser\|\|offset	933	offser\|\|offset
914	offet\|\|offset	934	offet\|\|offset
		935	offlaod\|\|offload
915	offloded\|\|offloaded	936	offloded\|\|offloaded
916	offseting\|\|offsetting	937	offseting\|\|offsetting
917	omited\|\|omitted	938	omited\|\|omitted
@@ -993,6 +1014,7 @@ poiter\|\|pointer · 75 unmodified lines
993	posible\|\|possible	1014	posible\|\|possible
994	positon\|\|position	1015	positon\|\|position
995	possibilites\|\|possibilities	1016	possibilites\|\|possibilities
		1017	potocol\|\|protocol
996	powerfull\|\|powerful	1018	powerfull\|\|powerful
997	pramater\|\|parameter	1019	pramater\|\|parameter
998	preamle\|\|preamble	1020	preamle\|\|preamble
@@ -1061,11 +1083,13 @@ psychadelic\|\|psychedelic · 62 unmodified lines
1061	pwoer\|\|power	1083	pwoer\|\|power
1062	queing\|\|queuing	1084	queing\|\|queuing
1063	quering\|\|querying	1085	quering\|\|querying
		1086	queus\|\|queues
1064	randomally\|\|randomly	1087	randomally\|\|randomly
1065	raoming\|\|roaming	1088	raoming\|\|roaming
1066	reasearcher\|\|researcher	1089	reasearcher\|\|researcher
1067	reasearchers\|\|researchers	1090	reasearchers\|\|researchers
1068	reasearch\|\|research	1091	reasearch\|\|research
		1092	receieve\|\|receive
1069	recepient\|\|recipient	1093	recepient\|\|recipient
1070	recevied\|\|received	1094	recevied\|\|received
1071	receving\|\|receiving	1095	receving\|\|receiving
@@ -1166,6 +1190,7 @@ scaleing\|\|scaling · 94 unmodified lines
1166	scaned\|\|scanned	1190	scaned\|\|scanned
1167	scaning\|\|scanning	1191	scaning\|\|scanning
1168	scarch\|\|search	1192	scarch\|\|search
		1193	schdule\|\|schedule
1169	seach\|\|search	1194	seach\|\|search
1170	searchs\|\|searches	1195	searchs\|\|searches
1171	secquence\|\|sequence	1196	secquence\|\|sequence
@@ -1308,6 +1333,7 @@ taskelt\|\|tasklet · 136 unmodified lines
1308	teh\|\|the	1333	teh\|\|the
1309	temorary\|\|temporary	1334	temorary\|\|temporary
1310	temproarily\|\|temporarily	1335	temproarily\|\|temporarily
		1336	temperture\|\|temperature
1311	thead\|\|thread	1337	thead\|\|thread
1312	therfore\|\|therefore	1338	therfore\|\|therefore
1313	thier\|\|their	1339	thier\|\|their
@@ -1354,6 +1380,7 @@ uknown\|\|unknown · 40 unmodified lines
1354	usupported\|\|unsupported	1380	usupported\|\|unsupported
1355	uncommited\|\|uncommitted	1381	uncommited\|\|uncommitted
1356	unconditionaly\|\|unconditionally	1382	unconditionaly\|\|unconditionally
		1383	undeflow\|\|underflow
1357	underun\|\|underrun	1384	underun\|\|underrun
1358	unecessary\|\|unnecessary	1385	unecessary\|\|unnecessary
1359	unexecpted\|\|unexpected	1386	unexecpted\|\|unexpected
@@ -1414,6 +1441,7 @@ varible\|\|variable · 54 unmodified lines
1414	varient\|\|variant	1441	varient\|\|variant
1415	vaule\|\|value	1442	vaule\|\|value
1416	verbse\|\|verbose	1443	verbse\|\|verbose
		1444	veify\|\|verify
1417	verisons\|\|versions	1445	verisons\|\|versions
1418	verison\|\|version	1446	verison\|\|version
1419	verson\|\|version	1447	verson\|\|version

tools/testing/selftests/memfd/memfd_test.c+36−0

@@ -290,6 +290,40 @@ static void mfd_assert_read_shared(int fd) · 289 unmodified lines
290	munmap(p, mfd_def_size);	290	munmap(p, mfd_def_size);
291	}	291	}
292		292
		293	static void mfd_assert_fork_private_write(int fd)
		294	{
		295	int *p;
		296	pid_t pid;
		297
		298	p = mmap(NULL,
		299	mfd_def_size,
		300	PROT_READ \| PROT_WRITE,
		301	MAP_PRIVATE,
		302	fd,
		303	0);
		304	if (p == MAP_FAILED) {
		305	printf("mmap() failed: %m\n");
		306	abort();
		307	}
		308
		309	p[0] = 22;
		310
		311	pid = fork();
		312	if (pid == 0) {
		313	p[0] = 33;
		314	exit(0);
		315	} else {
		316	waitpid(pid, NULL, 0);
		317
		318	if (p[0] != 22) {
		319	printf("MAP_PRIVATE copy-on-write failed: %m\n");
		320	abort();
		321	}
		322	}
		323
		324	munmap(p, mfd_def_size);
		325	}
		326
293	static void mfd_assert_write(int fd)	327	static void mfd_assert_write(int fd)
294	{	328	{
295	ssize_t l;	329	ssize_t l;
@@ -760,6 +794,8 @@ static void test_seal_future_write(void) · 464 unmodified lines
760	mfd_assert_read_shared(fd2);	794	mfd_assert_read_shared(fd2);
761	mfd_fail_write(fd2);	795	mfd_fail_write(fd2);
762		796
		797	mfd_assert_fork_private_write(fd);
		798
763	munmap(p, mfd_def_size);	799	munmap(p, mfd_def_size);
764	close(fd2);	800	close(fd2);
765	close(fd);	801	close(fd);

tools/testing/selftests/vm/config+1−0

@@ -1,2 +1,3 @@
1	CONFIG_SYSVIPC=y	1	CONFIG_SYSVIPC=y
2	CONFIG_USERFAULTFD=y	2	CONFIG_USERFAULTFD=y
		3	CONFIG_TEST_VMALLOC=m

Merge branch 'akpm' (patches from Andrew)

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code

@@ -29,7 +29,7 @@ static int cma_debugfs_get(void data, u64 val) · 28 unmodified lines
29		29
30	return 0;	30	return 0;
31	}	31	}
32	DEFINE_SIMPLE_ATTRIBUTE(cma_debugfs_fops, cma_debugfs_get, NULL, "%llu\n");	32	DEFINE_DEBUGFS_ATTRIBUTE(cma_debugfs_fops, cma_debugfs_get, NULL, "%llu\n");
33		33
34	static int cma_used_get(void data, u64 val)	34	static int cma_used_get(void data, u64 val)
35	{	35	{
@@ -44,7 +44,7 @@ static int cma_used_get(void data, u64 val) · 8 unmodified lines
44		44
45	return 0;	45	return 0;
46	}	46	}
47	DEFINE_SIMPLE_ATTRIBUTE(cma_used_fops, cma_used_get, NULL, "%llu\n");	47	DEFINE_DEBUGFS_ATTRIBUTE(cma_used_fops, cma_used_get, NULL, "%llu\n");
48		48
49	static int cma_maxchunk_get(void data, u64 val)	49	static int cma_maxchunk_get(void data, u64 val)
50	{	50	{
@@ -66,7 +66,7 @@ static int cma_maxchunk_get(void data, u64 val) · 15 unmodified lines
66		66
67	return 0;	67	return 0;
68	}	68	}
69	DEFINE_SIMPLE_ATTRIBUTE(cma_maxchunk_fops, cma_maxchunk_get, NULL, "%llu\n");	69	DEFINE_DEBUGFS_ATTRIBUTE(cma_maxchunk_fops, cma_maxchunk_get, NULL, "%llu\n");
70		70
71	static void cma_add_to_cma_mem_list(struct cma cma, struct cma_mem mem)	71	static void cma_add_to_cma_mem_list(struct cma cma, struct cma_mem mem)
72	{	72	{
@@ -126,7 +126,7 @@ static int cma_free_write(void *data, u64 val) · 53 unmodified lines
126		126
127	return cma_free_mem(cma, pages);	127	return cma_free_mem(cma, pages);
128	}	128	}
129	DEFINE_SIMPLE_ATTRIBUTE(cma_free_fops, NULL, cma_free_write, "%llu\n");	129	DEFINE_DEBUGFS_ATTRIBUTE(cma_free_fops, NULL, cma_free_write, "%llu\n");
130		130
131	static int cma_alloc_mem(struct cma *cma, int count)	131	static int cma_alloc_mem(struct cma *cma, int count)
132	{	132	{
@@ -158,7 +158,7 @@ static int cma_alloc_write(void *data, u64 val) · 25 unmodified lines
158		158
159	return cma_alloc_mem(cma, pages);	159	return cma_alloc_mem(cma, pages);
160	}	160	}
161	DEFINE_SIMPLE_ATTRIBUTE(cma_alloc_fops, NULL, cma_alloc_write, "%llu\n");	161	DEFINE_DEBUGFS_ATTRIBUTE(cma_alloc_fops, NULL, cma_alloc_write, "%llu\n");
162		162
163	static void cma_debugfs_add_one(struct cma cma, struct dentry root_dentry)	163	static void cma_debugfs_add_one(struct cma cma, struct dentry root_dentry)
164	{	164	{